diff options
| author | robot-piglet <[email protected]> | 2025-01-13 22:57:12 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2025-01-13 23:12:28 +0300 |
| commit | f35fe00dcc2af8b9605460b0eba29304694c7d22 (patch) | |
| tree | 5a70bd13e7037e2fdbabc25b49e4b9ef7af5d106 /contrib/python/setuptools | |
| parent | d952d8354362c04f11ca60d229dcaf0709fab9da (diff) | |
Intermediate changes
commit_hash:d3483365e53236dc94c949b30c45470fa72387a7
Diffstat (limited to 'contrib/python/setuptools')
149 files changed, 665 insertions, 37974 deletions
diff --git a/contrib/python/setuptools/py3/.dist-info/METADATA b/contrib/python/setuptools/py3/.dist-info/METADATA index 90eca520d03..55b94d8d5a0 100644 --- a/contrib/python/setuptools/py3/.dist-info/METADATA +++ b/contrib/python/setuptools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: setuptools -Version: 70.3.0 +Version: 71.1.0 Summary: Easily download, build, install, upgrade, and uninstall Python packages Author-email: Python Packaging Authority <[email protected]> Project-URL: Source, https://github.com/pypa/setuptools @@ -20,6 +20,16 @@ Requires-Python: >=3.8 Description-Content-Type: text/x-rst License-File: LICENSE Provides-Extra: certs +Provides-Extra: core +Requires-Dist: packaging >=24 ; extra == 'core' +Requires-Dist: ordered-set >=3.1.1 ; extra == 'core' +Requires-Dist: more-itertools >=8.8 ; extra == 'core' +Requires-Dist: jaraco.text >=3.7 ; extra == 'core' +Requires-Dist: wheel >=0.43.0 ; extra == 'core' +Requires-Dist: platformdirs >=2.6.2 ; extra == 'core' +Requires-Dist: importlib-metadata >=6 ; (python_version < "3.10") and extra == 'core' +Requires-Dist: tomli >=2.0.1 ; (python_version < "3.11") and extra == 'core' +Requires-Dist: importlib-resources >=5.10.2 ; (python_version < "3.9") and extra == 'core' Provides-Extra: doc Requires-Dist: sphinx >=3.5 ; extra == 'doc' Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' @@ -54,15 +64,17 @@ Requires-Dist: ini2toml[lite] >=0.14 ; extra == 'test' Requires-Dist: tomli-w >=1.0.0 ; extra == 'test' Requires-Dist: pytest-timeout ; extra == 'test' Requires-Dist: pytest-home >=0.5 ; extra == 'test' -Requires-Dist: mypy ==1.10.0 ; extra == 'test' +Requires-Dist: mypy ==1.11.* ; extra == 'test' Requires-Dist: tomli ; extra == 'test' Requires-Dist: importlib-metadata ; extra == 'test' Requires-Dist: pytest-subprocess ; extra == 'test' Requires-Dist: pyproject-hooks !=1.1 ; extra == 'test' Requires-Dist: jaraco.test ; extra == 'test' +Requires-Dist: pytest-ruff <0.4 ; (platform_system == "Windows") and extra == 'test' Requires-Dist: jaraco.develop >=7.21 ; (python_version >= "3.9" and sys_platform != "cygwin") and extra == 'test' -Requires-Dist: pytest-ruff >=0.3.2 ; (sys_platform != "cygwin") and extra == 'test' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'test' Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'test' +Requires-Dist: pytest-ruff >=0.3.2 ; (sys_platform != "cygwin") and extra == 'test' .. |pypi-version| image:: https://img.shields.io/pypi/v/setuptools.svg :target: https://pypi.org/project/setuptools diff --git a/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml b/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml index 82e5cdb5fff..ffe47f851aa 100644 --- a/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml +++ b/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml @@ -1,4 +1,16 @@ requirements: + - jaraco.context + - jaraco.functools + - jaraco.text + - ordered-set + - more-itertools + - packaging + - platformdirs + - typing-extensions + - typeguard + - wheel - library/python/resource mark_as_sources: - setuptools/command/test.py +exclude: + - setuptools/_vendor/* diff --git a/contrib/python/setuptools/py3/pkg_resources/__init__.py b/contrib/python/setuptools/py3/pkg_resources/__init__.py index bb8cbf88fdb..3f3956b6144 100644 --- a/contrib/python/setuptools/py3/pkg_resources/__init__.py +++ b/contrib/python/setuptools/py3/pkg_resources/__init__.py @@ -34,6 +34,7 @@ import re import types from typing import ( Any, + BinaryIO, Literal, Dict, Iterator, @@ -74,6 +75,10 @@ from pkgutil import get_importer import _imp +sys.path.extend(((vendor_path := os.path.join(os.path.dirname(os.path.dirname(__file__)), 'setuptools', '_vendor')) not in sys.path) * [vendor_path]) # fmt: skip +# workaround for #4476 +sys.modules.pop('backports', None) + # capture these to bypass sandboxing from os import utime from os import open as os_open @@ -87,16 +92,17 @@ except ImportError: # no write support, probably under GAE WRITE_SUPPORT = False -from pkg_resources.extern.jaraco.text import ( +import packaging.specifiers +from jaraco.text import ( yield_lines, drop_comment, join_continuation, ) -from pkg_resources.extern.packaging import markers as _packaging_markers -from pkg_resources.extern.packaging import requirements as _packaging_requirements -from pkg_resources.extern.packaging import utils as _packaging_utils -from pkg_resources.extern.packaging import version as _packaging_version -from pkg_resources.extern.platformdirs import user_cache_dir as _user_cache_dir +from packaging import markers as _packaging_markers +from packaging import requirements as _packaging_requirements +from packaging import utils as _packaging_utils +from packaging import version as _packaging_version +from platformdirs import user_cache_dir as _user_cache_dir if TYPE_CHECKING: from _typeshed import BytesPath, StrPath, StrOrBytesPath @@ -109,7 +115,6 @@ warnings.warn( stacklevel=2, ) - _T = TypeVar("_T") _DistributionT = TypeVar("_DistributionT", bound="Distribution") # Type aliases @@ -153,7 +158,6 @@ class PEP440Warning(RuntimeWarning): parse_version = _packaging_version.Version - _state_vars: dict[str, str] = {} @@ -335,7 +339,9 @@ class VersionConflict(ResolutionError): def report(self): return self._template.format(**locals()) - def with_context(self, required_by: set[Distribution | str]): + def with_context( + self, required_by: set[Distribution | str] + ) -> Self | ContextualVersionConflict: """ If required_by is non-empty, return a version of self that is a ContextualVersionConflict. @@ -404,7 +410,7 @@ DEVELOP_DIST = -1 def register_loader_type( loader_type: type[_ModuleLike], provider_factory: _ProviderFactoryType -): +) -> None: """Register `provider_factory` to make providers for `loader_type` `loader_type` is the type or class of a PEP 302 ``module.__loader__``, @@ -480,7 +486,7 @@ darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") get_platform = get_build_platform -def compatible_platforms(provided: str | None, required: str | None): +def compatible_platforms(provided: str | None, required: str | None) -> bool: """Can code for the `provided` platform run on the `required` platform? Returns true if either platform is ``None``, or the platforms are equal. @@ -538,8 +544,7 @@ def get_distribution(dist: Distribution | _PkgReqType) -> Distribution: if isinstance(dist, str): dist = Requirement.parse(dist) if isinstance(dist, Requirement): - # Bad type narrowing, dist has to be a Requirement here, so get_provider has to return Distribution - dist = get_provider(dist) # type: ignore[assignment] + dist = get_provider(dist) if not isinstance(dist, Distribution): raise TypeError("Expected str, Requirement, or Distribution", dist) return dist @@ -561,7 +566,7 @@ def get_entry_map(dist: _EPDistType, group: str | None = None): return get_distribution(dist).get_entry_map(group) -def get_entry_info(dist: _EPDistType, group: str, name: str): +def get_entry_info(dist: _EPDistType, group: str, name: str) -> EntryPoint | None: """Return the EntryPoint object for `group`+`name`, or ``None``""" return get_distribution(dist).get_entry_info(group, name) @@ -682,7 +687,7 @@ class WorkingSet: sys.path[:] = ws.entries return ws - def add_entry(self, entry: str): + def add_entry(self, entry: str) -> None: """Add a path item to ``.entries``, finding any distributions on it ``find_distributions(entry, True)`` is used to find distributions @@ -725,7 +730,9 @@ class WorkingSet: raise VersionConflict(dist, req) return dist - def iter_entry_points(self, group: str, name: str | None = None): + def iter_entry_points( + self, group: str, name: str | None = None + ) -> Iterator[EntryPoint]: """Yield entry point objects from `group` matching `name` If `name` is None, yields all entry points in `group` from all @@ -739,7 +746,7 @@ class WorkingSet: if name is None or name == entry.name ) - def run_script(self, requires: str, script_name: str): + def run_script(self, requires: str, script_name: str) -> None: """Locate distribution for `requires` and run `script_name` script""" ns = sys._getframe(1).f_globals name = ns['__name__'] @@ -770,7 +777,7 @@ class WorkingSet: entry: str | None = None, insert: bool = True, replace: bool = False, - ): + ) -> None: """Add `dist` to working set, associated with `entry` If `entry` is unspecified, it defaults to the ``.location`` of `dist`. @@ -1050,7 +1057,7 @@ class WorkingSet: return sorted_distributions, error_info - def require(self, *requirements: _NestedStr): + def require(self, *requirements: _NestedStr) -> list[Distribution]: """Ensure that distributions matching `requirements` are activated `requirements` must be a string or a (possibly-nested) sequence @@ -1068,7 +1075,7 @@ class WorkingSet: def subscribe( self, callback: Callable[[Distribution], object], existing: bool = True - ): + ) -> None: """Invoke `callback` for all distributions If `existing=True` (default), @@ -1117,11 +1124,10 @@ class _ReqExtras(Dict["Requirement", Tuple[str, ...]]): Return False if the req has a marker and fails evaluation. Otherwise, return True. """ - extra_evals = ( + return not req.marker or any( req.marker.evaluate({'extra': extra}) - for extra in self.get(req, ()) + (extras or (None,)) + for extra in self.get(req, ()) + (extras or ("",)) ) - return not req.marker or any(extra_evals) class Environment: @@ -1154,7 +1160,7 @@ class Environment: self.python = python self.scan(search_path) - def can_add(self, dist: Distribution): + def can_add(self, dist: Distribution) -> bool: """Is distribution `dist` acceptable for this environment? The distribution must match the platform and python version @@ -1168,11 +1174,11 @@ class Environment: ) return py_compat and compatible_platforms(dist.platform, self.platform) - def remove(self, dist: Distribution): + def remove(self, dist: Distribution) -> None: """Remove `dist` from the environment""" self._distmap[dist.key].remove(dist) - def scan(self, search_path: Iterable[str] | None = None): + def scan(self, search_path: Iterable[str] | None = None) -> None: """Scan `search_path` for distributions usable in this environment Any distributions found are added to the environment. @@ -1198,7 +1204,7 @@ class Environment: distribution_key = project_name.lower() return self._distmap.get(distribution_key, []) - def add(self, dist: Distribution): + def add(self, dist: Distribution) -> None: """Add `dist` if we ``can_add()`` it and it has not already been added""" if self.can_add(dist) and dist.has_version(): dists = self._distmap.setdefault(dist.key, []) @@ -1349,23 +1355,29 @@ class ResourceManager: def __init__(self): self.cached_files = {} - def resource_exists(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_exists( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> bool: """Does the named resource exist?""" return get_provider(package_or_requirement).has_resource(resource_name) - def resource_isdir(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_isdir( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> bool: """Is the named resource an existing directory?""" return get_provider(package_or_requirement).resource_isdir(resource_name) def resource_filename( self, package_or_requirement: _PkgReqType, resource_name: str - ): + ) -> str: """Return a true filesystem path for specified resource""" return get_provider(package_or_requirement).get_resource_filename( self, resource_name ) - def resource_stream(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_stream( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> _ResourceStream: """Return a readable file-like object for specified resource""" return get_provider(package_or_requirement).get_resource_stream( self, resource_name @@ -1379,7 +1391,9 @@ class ResourceManager: self, resource_name ) - def resource_listdir(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_listdir( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> list[str]: """List the contents of the named resource directory""" return get_provider(package_or_requirement).resource_listdir(resource_name) @@ -1413,7 +1427,7 @@ class ResourceManager: err.original_error = old_exc raise err - def get_cache_path(self, archive_name: str, names: Iterable[StrPath] = ()): + def get_cache_path(self, archive_name: str, names: Iterable[StrPath] = ()) -> str: """Return absolute location in cache for `archive_name` and `names` The parent directory of the resulting path will be created if it does @@ -1465,7 +1479,7 @@ class ResourceManager: ).format(**locals()) warnings.warn(msg, UserWarning) - def postprocess(self, tempname: StrOrBytesPath, filename: StrOrBytesPath): + def postprocess(self, tempname: StrOrBytesPath, filename: StrOrBytesPath) -> None: """Perform any platform-specific postprocessing of `tempname` This is where Mac header rewrites should be done; other platforms don't @@ -1485,7 +1499,7 @@ class ResourceManager: mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777 os.chmod(tempname, mode) - def set_extraction_path(self, path: str): + def set_extraction_path(self, path: str) -> None: """Set the base path where resources will be extracted to, if needed. If you do not call this routine before any extractions take place, the @@ -1533,7 +1547,7 @@ def get_default_cache() -> str: return os.environ.get('PYTHON_EGG_CACHE') or _user_cache_dir(appname='Python-Eggs') -def safe_name(name: str): +def safe_name(name: str) -> str: """Convert an arbitrary string to a standard distribution name Any runs of non-alphanumeric/. characters are replaced with a single '-'. @@ -1541,7 +1555,7 @@ def safe_name(name: str): return re.sub('[^A-Za-z0-9.]+', '-', name) -def safe_version(version: str): +def safe_version(version: str) -> str: """ Convert an arbitrary string to a standard version string """ @@ -1585,7 +1599,7 @@ def _safe_segment(segment): return re.sub(r'\.[^A-Za-z0-9]+', '.', segment).strip(".-") -def safe_extra(extra: str): +def safe_extra(extra: str) -> str: """Convert an arbitrary string to a standard 'extra' name Any runs of non-alphanumeric characters are replaced with a single '_', @@ -1594,7 +1608,7 @@ def safe_extra(extra: str): return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() -def to_filename(name: str): +def to_filename(name: str) -> str: """Convert a project or version name to its filename-escaped form Any '-' characters are currently replaced with '_'. @@ -1602,7 +1616,7 @@ def to_filename(name: str): return name.replace('-', '_') -def invalid_marker(text: str): +def invalid_marker(text: str) -> SyntaxError | Literal[False]: """ Validate text as a PEP 508 environment marker; return an exception if invalid or False otherwise. @@ -1642,10 +1656,14 @@ class NullProvider: self.loader = getattr(module, '__loader__', None) self.module_path = os.path.dirname(getattr(module, '__file__', '')) - def get_resource_filename(self, manager: ResourceManager, resource_name: str): + def get_resource_filename( + self, manager: ResourceManager, resource_name: str + ) -> str: return self._fn(self.module_path, resource_name) - def get_resource_stream(self, manager: ResourceManager, resource_name: str): + def get_resource_stream( + self, manager: ResourceManager, resource_name: str + ) -> BinaryIO: return io.BytesIO(self.get_resource_string(manager, resource_name)) def get_resource_string( @@ -1653,7 +1671,7 @@ class NullProvider: ) -> bytes: return self._get(self._fn(self.module_path, resource_name)) - def has_resource(self, resource_name: str): + def has_resource(self, resource_name: str) -> bool: return self._has(self._fn(self.module_path, resource_name)) def _get_metadata_path(self, name): @@ -1666,7 +1684,7 @@ class NullProvider: path = self._get_metadata_path(name) return self._has(path) - def get_metadata(self, name: str): + def get_metadata(self, name: str) -> str: if not self.egg_info: return "" path = self._get_metadata_path(name) @@ -1682,13 +1700,13 @@ class NullProvider: def get_metadata_lines(self, name: str) -> Iterator[str]: return yield_lines(self.get_metadata(name)) - def resource_isdir(self, resource_name: str): + def resource_isdir(self, resource_name: str) -> bool: return self._isdir(self._fn(self.module_path, resource_name)) def metadata_isdir(self, name: str) -> bool: return bool(self.egg_info and self._isdir(self._fn(self.egg_info, name))) - def resource_listdir(self, resource_name: str): + def resource_listdir(self, resource_name: str) -> list[str]: return self._listdir(self._fn(self.module_path, resource_name)) def metadata_listdir(self, name: str) -> list[str]: @@ -1696,7 +1714,7 @@ class NullProvider: return self._listdir(self._fn(self.egg_info, name)) return [] - def run_script(self, script_name: str, namespace: dict[str, Any]): + def run_script(self, script_name: str, namespace: dict[str, Any]) -> None: script = 'scripts/' + script_name if not self.has_metadata(script): raise ResolutionError( @@ -1880,7 +1898,9 @@ class DefaultProvider(EggProvider): def _listdir(self, path): return os.listdir(path) - def get_resource_stream(self, manager: object, resource_name: str): + def get_resource_stream( + self, manager: object, resource_name: str + ) -> io.BufferedReader: return open(self._fn(self.module_path, resource_name), 'rb') def _get(self, path) -> bytes: @@ -1929,7 +1949,7 @@ class ZipManifests(Dict[str, "MemoizedZipManifests.manifest_mod"]): # `path` could be `StrPath | IO[bytes]` but that violates the LSP for `MemoizedZipManifests.load` @classmethod - def build(cls, path: str): + def build(cls, path: str) -> dict[str, zipfile.ZipInfo]: """ Build a dictionary similar to the zipimport directory caches, except instead of tuples, store ZipInfo objects. @@ -2007,7 +2027,9 @@ class ZipProvider(EggProvider): def zipinfo(self): return self._zip_manifests.load(self.loader.archive) - def get_resource_filename(self, manager: ResourceManager, resource_name: str): + def get_resource_filename( + self, manager: ResourceManager, resource_name: str + ) -> str: if not self.egg_name: raise NotImplementedError( "resource_filename() only supported for .egg, not .zip" @@ -2167,7 +2189,7 @@ class FileMetadata(EmptyProvider): def has_metadata(self, name: str) -> bool: return name == 'PKG-INFO' and os.path.isfile(self.path) - def get_metadata(self, name: str): + def get_metadata(self, name: str) -> str: if name != 'PKG-INFO': raise KeyError("No metadata except PKG-INFO is available") @@ -2232,7 +2254,9 @@ _distribution_finders: dict[type, _DistFinderType[Any]] = _declare_state( ) -def register_finder(importer_type: type[_T], distribution_finder: _DistFinderType[_T]): +def register_finder( + importer_type: type[_T], distribution_finder: _DistFinderType[_T] +) -> None: """Register `distribution_finder` to find distributions in sys.path items `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item @@ -2242,7 +2266,7 @@ def register_finder(importer_type: type[_T], distribution_finder: _DistFinderTyp _distribution_finders[importer_type] = distribution_finder -def find_distributions(path_item: str, only: bool = False): +def find_distributions(path_item: str, only: bool = False) -> Iterable[Distribution]: """Yield distributions accessible via `path_item`""" importer = get_importer(path_item) finder = _find_adapter(_distribution_finders, importer) @@ -2416,7 +2440,7 @@ _namespace_packages: dict[str | None, list[str]] = _declare_state( def register_namespace_handler( importer_type: type[_T], namespace_handler: _NSHandlerType[_T] -): +) -> None: """Register `namespace_handler` to declare namespace packages `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item @@ -2505,7 +2529,7 @@ def _rebuild_mod_path(orig_path, package_name, module: types.ModuleType): module.__path__ = new_path -def declare_namespace(packageName: str): +def declare_namespace(packageName: str) -> None: """Declare that package 'packageName' is a namespace package""" msg = ( @@ -2548,7 +2572,7 @@ def declare_namespace(packageName: str): _imp.release_lock() -def fixup_namespace_packages(path_item: str, parent: str | None = None): +def fixup_namespace_packages(path_item: str, parent: str | None = None) -> None: """Ensure that previously-declared namespace packages include path_item""" _imp.acquire_lock() try: @@ -2625,6 +2649,7 @@ if TYPE_CHECKING: @overload def _normalize_cached(filename: BytesPath) -> bytes: ... def _normalize_cached(filename: StrOrBytesPath) -> str | bytes: ... + else: @functools.lru_cache(maxsize=None) @@ -2759,7 +2784,7 @@ class EntryPoint: self, env: Environment | None = None, installer: _InstallerType | None = None, - ): + ) -> None: if not self.dist: error_cls = UnknownExtra if self.extras else AttributeError raise error_cls("Can't require() without a distribution", self) @@ -2783,7 +2808,7 @@ class EntryPoint: ) @classmethod - def parse(cls, src: str, dist: Distribution | None = None): + def parse(cls, src: str, dist: Distribution | None = None) -> Self: """Parse a single entry point from string `src` Entry point syntax follows the form:: @@ -2817,7 +2842,7 @@ class EntryPoint: group: str, lines: _NestedStr, dist: Distribution | None = None, - ): + ) -> dict[str, Self]: """Parse an entry point group""" if not MODULE(group): raise ValueError("Invalid group name", group) @@ -2834,7 +2859,7 @@ class EntryPoint: cls, data: str | Iterable[str] | dict[str, str | Iterable[str]], dist: Distribution | None = None, - ): + ) -> dict[str, dict[str, Self]]: """Parse a map of entry point groups""" _data: Iterable[tuple[str | None, str | Iterable[str]]] if isinstance(data, dict): @@ -3039,7 +3064,9 @@ class Distribution: return self.__dep_map @staticmethod - def _filter_extras(dm: dict[str | None, list[Requirement]]): + def _filter_extras( + dm: dict[str | None, list[Requirement]], + ) -> dict[str | None, list[Requirement]]: """ Given a mapping of extras to dependencies, strip off environment markers and filter out any dependencies @@ -3066,7 +3093,7 @@ class Distribution: dm.setdefault(extra, []).extend(parse_requirements(reqs)) return dm - def requires(self, extras: Iterable[str] = ()): + def requires(self, extras: Iterable[str] = ()) -> list[Requirement]: """List of Requirements needed for this distro if `extras` are used""" dm = self._dep_map deps: list[Requirement] = [] @@ -3105,7 +3132,7 @@ class Distribution: lines = self._get_metadata(self.PKG_INFO) return _version_from_file(lines) - def activate(self, path: list[str] | None = None, replace: bool = False): + def activate(self, path: list[str] | None = None, replace: bool = False) -> None: """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: path = sys.path @@ -3160,7 +3187,7 @@ class Distribution: filename: StrPath, metadata: _MetadataType = None, **kw: int, # We could set `precedence` explicitly, but keeping this as `**kw` for full backwards and subclassing compatibility - ): + ) -> Distribution: return cls.from_location( _normalize_cached(filename), os.path.basename(filename), metadata, **kw ) @@ -3195,7 +3222,7 @@ class Distribution: return self._ep_map.get(group, {}) return self._ep_map - def get_entry_info(self, group: str, name: str): + def get_entry_info(self, group: str, name: str) -> EntryPoint | None: """Return the EntryPoint object for `group`+`name`, or ``None``""" return self.get_entry_map(group).get(name) @@ -3205,7 +3232,7 @@ class Distribution: path: list[str], loc=None, replace: bool = False, - ): + ) -> None: """Ensure self.location is on path If replace=False (default): @@ -3310,7 +3337,7 @@ class Distribution: return False return True - def clone(self, **kw: str | int | IResourceProvider | None): + def clone(self, **kw: str | int | IResourceProvider | None) -> Self: """Copy this distribution, substituting in any changed keyword args""" names = 'project_name version py_version platform location precedence' for attr in names.split(): @@ -3416,7 +3443,7 @@ def issue_warning(*args, **kw): warnings.warn(stacklevel=level + 1, *args, **kw) -def parse_requirements(strs: _NestedStr): +def parse_requirements(strs: _NestedStr) -> map[Requirement]: """ Yield ``Requirement`` objects for each specification in `strs`. @@ -3430,6 +3457,10 @@ class RequirementParseError(_packaging_requirements.InvalidRequirement): class Requirement(_packaging_requirements.Requirement): + # prefer variable length tuple to set (as found in + # packaging.requirements.Requirement) + extras: tuple[str, ...] # type: ignore[assignment] + def __init__(self, requirement_string: str): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" super().__init__(requirement_string) @@ -3437,8 +3468,7 @@ class Requirement(_packaging_requirements.Requirement): project_name = safe_name(self.name) self.project_name, self.key = project_name, project_name.lower() self.specs = [(spec.operator, spec.version) for spec in self.specifier] - # packaging.requirements.Requirement uses a set for its extras. We use a variable-length tuple - self.extras: tuple[str] = tuple(map(safe_extra, self.extras)) + self.extras = tuple(map(safe_extra, self.extras)) self.hashCmp = ( self.key, self.url, @@ -3454,17 +3484,24 @@ class Requirement(_packaging_requirements.Requirement): def __ne__(self, other): return not self == other - def __contains__(self, item: Distribution | str | tuple[str, ...]) -> bool: + def __contains__( + self, item: Distribution | packaging.specifiers.UnparsedVersion + ) -> bool: if isinstance(item, Distribution): if item.key != self.key: return False - item = item.version + version = item.version + else: + version = item # Allow prereleases always in order to match the previous behavior of # this method. In the future this should be smarter and follow PEP 440 # more accurately. - return self.specifier.contains(item, prereleases=True) + return self.specifier.contains( + version, + prereleases=True, + ) def __hash__(self): return self.__hash @@ -3473,7 +3510,7 @@ class Requirement(_packaging_requirements.Requirement): return "Requirement.parse(%r)" % str(self) @staticmethod - def parse(s: str | Iterable[str]): + def parse(s: str | Iterable[str]) -> Requirement: (req,) = parse_requirements(s) return req @@ -3499,7 +3536,7 @@ def _find_adapter(registry: Mapping[type, _AdapterT], ob: object) -> _AdapterT: raise TypeError(f"Could not find adapter for {registry} and {ob}") -def ensure_directory(path: StrOrBytesPath): +def ensure_directory(path: StrOrBytesPath) -> None: """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) os.makedirs(dirname, exist_ok=True) @@ -3701,6 +3738,7 @@ class PkgResourcesDeprecationWarning(Warning): _LOCALE_ENCODING = "locale" if sys.version_info >= (3, 10) else None +# This must go before calls to `_call_aside`. See https://github.com/pypa/setuptools/pull/4422 def _read_utf8_with_fallback(file: str, fallback_encoding=_LOCALE_ENCODING) -> str: """See setuptools.unicode_utils._read_utf8_with_fallback""" try: diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py deleted file mode 100644 index a7a9a6e7b94..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py +++ /dev/null @@ -1,2900 +0,0 @@ -#!/usr/bin/env python3 -#------------------------------------------------------------------- -# tarfile.py -#------------------------------------------------------------------- -# Copyright (C) 2002 Lars Gustaebel <[email protected]> -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -"""Read from and write to tar format archives. -""" - -version = "0.9.0" -__author__ = "Lars Gust\u00e4bel ([email protected])" -__credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." - -#--------- -# Imports -#--------- -from builtins import open as bltn_open -import sys -import os -import io -import shutil -import stat -import time -import struct -import copy -import re -import warnings - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -# os.symlink on Windows prior to 6.0 raises NotImplementedError -# OSError (winerror=1314) will be raised if the caller does not hold the -# SeCreateSymbolicLinkPrivilege privilege -symlink_exception = (AttributeError, NotImplementedError, OSError) - -# from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", - "CompressionError", "StreamError", "ExtractError", "HeaderError", - "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", - "tar_filter", "FilterError", "AbsoluteLinkError", - "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] - - -#--------------------------------------------------------- -# tar constants -#--------------------------------------------------------- -NUL = b"\0" # the null character -BLOCKSIZE = 512 # length of processing blocks -RECORDSIZE = BLOCKSIZE * 20 # length of records -GNU_MAGIC = b"ustar \0" # magic gnu tar string -POSIX_MAGIC = b"ustar\x0000" # magic posix tar string - -LENGTH_NAME = 100 # maximum length of a filename -LENGTH_LINK = 100 # maximum length of a linkname -LENGTH_PREFIX = 155 # maximum length of the prefix field - -REGTYPE = b"0" # regular file -AREGTYPE = b"\0" # regular file -LNKTYPE = b"1" # link (inside tarfile) -SYMTYPE = b"2" # symbolic link -CHRTYPE = b"3" # character special device -BLKTYPE = b"4" # block special device -DIRTYPE = b"5" # directory -FIFOTYPE = b"6" # fifo special device -CONTTYPE = b"7" # contiguous file - -GNUTYPE_LONGNAME = b"L" # GNU tar longname -GNUTYPE_LONGLINK = b"K" # GNU tar longlink -GNUTYPE_SPARSE = b"S" # GNU tar sparse file - -XHDTYPE = b"x" # POSIX.1-2001 extended header -XGLTYPE = b"g" # POSIX.1-2001 global header -SOLARIS_XHDTYPE = b"X" # Solaris extended header - -USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format -GNU_FORMAT = 1 # GNU tar format -PAX_FORMAT = 2 # POSIX.1-2001 (pax) format -DEFAULT_FORMAT = PAX_FORMAT - -#--------------------------------------------------------- -# tarfile constants -#--------------------------------------------------------- -# File types that tarfile supports: -SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, - SYMTYPE, DIRTYPE, FIFOTYPE, - CONTTYPE, CHRTYPE, BLKTYPE, - GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# File types that will be treated as a regular file. -REGULAR_TYPES = (REGTYPE, AREGTYPE, - CONTTYPE, GNUTYPE_SPARSE) - -# File types that are part of the GNU tar format. -GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# Fields from a pax header that override a TarInfo attribute. -PAX_FIELDS = ("path", "linkpath", "size", "mtime", - "uid", "gid", "uname", "gname") - -# Fields from a pax header that are affected by hdrcharset. -PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"} - -# Fields in a pax header that are numbers, all other fields -# are treated as strings. -PAX_NUMBER_FIELDS = { - "atime": float, - "ctime": float, - "mtime": float, - "uid": int, - "gid": int, - "size": int -} - -#--------------------------------------------------------- -# initialization -#--------------------------------------------------------- -if os.name == "nt": - ENCODING = "utf-8" -else: - ENCODING = sys.getfilesystemencoding() - -#--------------------------------------------------------- -# Some useful functions -#--------------------------------------------------------- - -def stn(s, length, encoding, errors): - """Convert a string to a null-terminated bytes object. - """ - if s is None: - raise ValueError("metadata cannot contain None") - s = s.encode(encoding, errors) - return s[:length] + (length - len(s)) * NUL - -def nts(s, encoding, errors): - """Convert a null-terminated bytes object to a string. - """ - p = s.find(b"\0") - if p != -1: - s = s[:p] - return s.decode(encoding, errors) - -def nti(s): - """Convert a number field to a python number. - """ - # There are two possible encodings for a number field, see - # itn() below. - if s[0] in (0o200, 0o377): - n = 0 - for i in range(len(s) - 1): - n <<= 8 - n += s[i + 1] - if s[0] == 0o377: - n = -(256 ** (len(s) - 1) - n) - else: - try: - s = nts(s, "ascii", "strict") - n = int(s.strip() or "0", 8) - except ValueError: - raise InvalidHeaderError("invalid header") - return n - -def itn(n, digits=8, format=DEFAULT_FORMAT): - """Convert a python number to a number field. - """ - # POSIX 1003.1-1988 requires numbers to be encoded as a string of - # octal digits followed by a null-byte, this allows values up to - # (8**(digits-1))-1. GNU tar allows storing numbers greater than - # that if necessary. A leading 0o200 or 0o377 byte indicate this - # particular encoding, the following digits-1 bytes are a big-endian - # base-256 representation. This allows values up to (256**(digits-1))-1. - # A 0o200 byte indicates a positive number, a 0o377 byte a negative - # number. - original_n = n - n = int(n) - if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL - elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): - if n >= 0: - s = bytearray([0o200]) - else: - s = bytearray([0o377]) - n = 256 ** digits + n - - for i in range(digits - 1): - s.insert(1, n & 0o377) - n >>= 8 - else: - raise ValueError("overflow in number field") - - return s - -def calc_chksums(buf): - """Calculate the checksum for a member's header by summing up all - characters except for the chksum field which is treated as if - it was filled with spaces. According to the GNU tar sources, - some tars (Sun and NeXT) calculate chksum with signed char, - which will be different if there are chars in the buffer with - the high bit set. So we calculate two checksums, unsigned and - signed. - """ - unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf)) - signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf)) - return unsigned_chksum, signed_chksum - -def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): - """Copy length bytes from fileobj src to fileobj dst. - If length is None, copy the entire content. - """ - bufsize = bufsize or 16 * 1024 - if length == 0: - return - if length is None: - shutil.copyfileobj(src, dst, bufsize) - return - - blocks, remainder = divmod(length, bufsize) - for b in range(blocks): - buf = src.read(bufsize) - if len(buf) < bufsize: - raise exception("unexpected end of data") - dst.write(buf) - - if remainder != 0: - buf = src.read(remainder) - if len(buf) < remainder: - raise exception("unexpected end of data") - dst.write(buf) - return - -def _safe_print(s): - encoding = getattr(sys.stdout, 'encoding', None) - if encoding is not None: - s = s.encode(encoding, 'backslashreplace').decode(encoding) - print(s, end=' ') - - -class TarError(Exception): - """Base exception.""" - pass -class ExtractError(TarError): - """General exception for extract errors.""" - pass -class ReadError(TarError): - """Exception for unreadable tar archives.""" - pass -class CompressionError(TarError): - """Exception for unavailable compression methods.""" - pass -class StreamError(TarError): - """Exception for unsupported operations on stream-like TarFiles.""" - pass -class HeaderError(TarError): - """Base exception for header errors.""" - pass -class EmptyHeaderError(HeaderError): - """Exception for empty headers.""" - pass -class TruncatedHeaderError(HeaderError): - """Exception for truncated headers.""" - pass -class EOFHeaderError(HeaderError): - """Exception for end of file headers.""" - pass -class InvalidHeaderError(HeaderError): - """Exception for invalid headers.""" - pass -class SubsequentHeaderError(HeaderError): - """Exception for missing and invalid extended headers.""" - pass - -#--------------------------- -# internal stream interface -#--------------------------- -class _LowLevelFile: - """Low-level file object. Supports reading and writing. - It is used instead of a regular file object for streaming - access. - """ - - def __init__(self, name, mode): - mode = { - "r": os.O_RDONLY, - "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - }[mode] - if hasattr(os, "O_BINARY"): - mode |= os.O_BINARY - self.fd = os.open(name, mode, 0o666) - - def close(self): - os.close(self.fd) - - def read(self, size): - return os.read(self.fd, size) - - def write(self, s): - os.write(self.fd, s) - -class _Stream: - """Class that serves as an adapter between TarFile and - a stream-like object. The stream-like object only - needs to have a read() or write() method that works with bytes, - and the method is accessed blockwise. - Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin.buffer, - sys.stdout.buffer, a socket, a tape device etc. - - _Stream is intended to be used only internally. - """ - - def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel): - """Construct a _Stream object. - """ - self._extfileobj = True - if fileobj is None: - fileobj = _LowLevelFile(name, mode) - self._extfileobj = False - - if comptype == '*': - # Enable transparent compression detection for the - # stream interface - fileobj = _StreamProxy(fileobj) - comptype = fileobj.getcomptype() - - self.name = name or "" - self.mode = mode - self.comptype = comptype - self.fileobj = fileobj - self.bufsize = bufsize - self.buf = b"" - self.pos = 0 - self.closed = False - - try: - if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") from None - self.zlib = zlib - self.crc = zlib.crc32(b"") - if mode == "r": - self.exception = zlib.error - self._init_read_gz() - else: - self._init_write_gz(compresslevel) - - elif comptype == "bz2": - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = bz2.BZ2Decompressor() - self.exception = OSError - else: - self.cmp = bz2.BZ2Compressor(compresslevel) - - elif comptype == "xz": - try: - import lzma - except ImportError: - raise CompressionError("lzma module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = lzma.LZMADecompressor() - self.exception = lzma.LZMAError - else: - self.cmp = lzma.LZMACompressor() - - elif comptype != "tar": - raise CompressionError("unknown compression type %r" % comptype) - - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - def __del__(self): - if hasattr(self, "closed") and not self.closed: - self.close() - - def _init_write_gz(self, compresslevel): - """Initialize for writing with gzip compression. - """ - self.cmp = self.zlib.compressobj(compresslevel, - self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) - timestamp = struct.pack("<L", int(time.time())) - self.__write(b"\037\213\010\010" + timestamp + b"\002\377") - if self.name.endswith(".gz"): - self.name = self.name[:-3] - # Honor "directory components removed" from RFC1952 - self.name = os.path.basename(self.name) - # RFC1952 says we must use ISO-8859-1 for the FNAME field. - self.__write(self.name.encode("iso-8859-1", "replace") + NUL) - - def write(self, s): - """Write string s to the stream. - """ - if self.comptype == "gz": - self.crc = self.zlib.crc32(s, self.crc) - self.pos += len(s) - if self.comptype != "tar": - s = self.cmp.compress(s) - self.__write(s) - - def __write(self, s): - """Write string s to the stream if a whole new block - is ready to be written. - """ - self.buf += s - while len(self.buf) > self.bufsize: - self.fileobj.write(self.buf[:self.bufsize]) - self.buf = self.buf[self.bufsize:] - - def close(self): - """Close the _Stream object. No operation should be - done on it afterwards. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode == "w" and self.comptype != "tar": - self.buf += self.cmp.flush() - - if self.mode == "w" and self.buf: - self.fileobj.write(self.buf) - self.buf = b"" - if self.comptype == "gz": - self.fileobj.write(struct.pack("<L", self.crc)) - self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def _init_read_gz(self): - """Initialize for reading a gzip compressed fileobj. - """ - self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" - - # taken from gzip.GzipFile with some alterations - if self.__read(2) != b"\037\213": - raise ReadError("not a gzip file") - if self.__read(1) != b"\010": - raise CompressionError("unsupported compression method") - - flag = ord(self.__read(1)) - self.__read(6) - - if flag & 4: - xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) - if flag & 8: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 16: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 2: - self.__read(2) - - def tell(self): - """Return the stream's file pointer position. - """ - return self.pos - - def seek(self, pos=0): - """Set the stream's file pointer to pos. Negative seeking - is forbidden. - """ - if pos - self.pos >= 0: - blocks, remainder = divmod(pos - self.pos, self.bufsize) - for i in range(blocks): - self.read(self.bufsize) - self.read(remainder) - else: - raise StreamError("seeking backwards is not allowed") - return self.pos - - def read(self, size): - """Return the next size number of bytes from the stream.""" - assert size is not None - buf = self._read(size) - self.pos += len(buf) - return buf - - def _read(self, size): - """Return size bytes from the stream. - """ - if self.comptype == "tar": - return self.__read(size) - - c = len(self.dbuf) - t = [self.dbuf] - while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" - else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] - - def __read(self, size): - """Return size bytes from stream. If internal buffer is empty, - read another block from the stream. - """ - c = len(self.buf) - t = [self.buf] - while c < size: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - t.append(buf) - c += len(buf) - t = b"".join(t) - self.buf = t[size:] - return t[:size] -# class _Stream - -class _StreamProxy(object): - """Small proxy class that enables transparent compression - detection for the Stream interface (mode 'r|*'). - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - self.buf = self.fileobj.read(BLOCKSIZE) - - def read(self, size): - self.read = self.fileobj.read - return self.buf - - def getcomptype(self): - if self.buf.startswith(b"\x1f\x8b\x08"): - return "gz" - elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY": - return "bz2" - elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): - return "xz" - else: - return "tar" - - def close(self): - self.fileobj.close() -# class StreamProxy - -#------------------------ -# Extraction file object -#------------------------ -class _FileInFile(object): - """A thin wrapper around an existing file object that - provides a part of its data as an individual file - object. - """ - - def __init__(self, fileobj, offset, size, name, blockinfo=None): - self.fileobj = fileobj - self.offset = offset - self.size = size - self.position = 0 - self.name = name - self.closed = False - - if blockinfo is None: - blockinfo = [(0, size)] - - # Construct a map with data and zero blocks. - self.map_index = 0 - self.map = [] - lastpos = 0 - realpos = self.offset - for offset, size in blockinfo: - if offset > lastpos: - self.map.append((False, lastpos, offset, None)) - self.map.append((True, offset, offset + size, realpos)) - realpos += size - lastpos = offset + size - if lastpos < self.size: - self.map.append((False, lastpos, self.size, None)) - - def flush(self): - pass - - def readable(self): - return True - - def writable(self): - return False - - def seekable(self): - return self.fileobj.seekable() - - def tell(self): - """Return the current file position. - """ - return self.position - - def seek(self, position, whence=io.SEEK_SET): - """Seek to a position in the file. - """ - if whence == io.SEEK_SET: - self.position = min(max(position, 0), self.size) - elif whence == io.SEEK_CUR: - if position < 0: - self.position = max(self.position + position, 0) - else: - self.position = min(self.position + position, self.size) - elif whence == io.SEEK_END: - self.position = max(min(self.size + position, self.size), 0) - else: - raise ValueError("Invalid argument") - return self.position - - def read(self, size=None): - """Read data from the file. - """ - if size is None: - size = self.size - self.position - else: - size = min(size, self.size - self.position) - - buf = b"" - while size > 0: - while True: - data, start, stop, offset = self.map[self.map_index] - if start <= self.position < stop: - break - else: - self.map_index += 1 - if self.map_index == len(self.map): - self.map_index = 0 - length = min(size, stop - self.position) - if data: - self.fileobj.seek(offset + (self.position - start)) - b = self.fileobj.read(length) - if len(b) != length: - raise ReadError("unexpected end of data") - buf += b - else: - buf += NUL * length - size -= length - self.position += length - return buf - - def readinto(self, b): - buf = self.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def close(self): - self.closed = True -#class _FileInFile - -class ExFileObject(io.BufferedReader): - - def __init__(self, tarfile, tarinfo): - fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data, - tarinfo.size, tarinfo.name, tarinfo.sparse) - super().__init__(fileobj) -#class ExFileObject - - -#----------------------------- -# extraction filters (PEP 706) -#----------------------------- - -class FilterError(TarError): - pass - -class AbsolutePathError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'member {tarinfo.name!r} has an absolute path') - -class OutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, ' - + 'which is outside the destination') - -class SpecialFileError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a special file') - -class AbsoluteLinkError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a link to an absolute path') - -class LinkOutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' - + 'which is outside the destination') - -def _get_filtered_attrs(member, dest_path, for_data=True): - new_attrs = {} - name = member.name - dest_path = os.path.realpath(dest_path) - # Strip leading / (tar's directory separator) from filenames. - # Include os.sep (target OS directory separator) as well. - if name.startswith(('/', os.sep)): - name = new_attrs['name'] = member.path.lstrip('/' + os.sep) - if os.path.isabs(name): - # Path is absolute even after stripping. - # For example, 'C:/foo' on Windows. - raise AbsolutePathError(member) - # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise OutsideDestinationError(member, target_path) - # Limit permissions (no high bits, and go-w) - mode = member.mode - if mode is not None: - # Strip high bits & group/other write bits - mode = mode & 0o755 - if for_data: - # For data, handle permissions & file types - if member.isreg() or member.islnk(): - if not mode & 0o100: - # Clear executable bits if not executable by user - mode &= ~0o111 - # Ensure owner can read & write - mode |= 0o600 - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Reject special files - raise SpecialFileError(member) - if mode != member.mode: - new_attrs['mode'] = mode - if for_data: - # Ignore ownership for 'data' - if member.uid is not None: - new_attrs['uid'] = None - if member.gid is not None: - new_attrs['gid'] = None - if member.uname is not None: - new_attrs['uname'] = None - if member.gname is not None: - new_attrs['gname'] = None - # Check link destination for 'data' - if member.islnk() or member.issym(): - if os.path.isabs(member.linkname): - raise AbsoluteLinkError(member) - if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) - else: - target_path = os.path.join(dest_path, - member.linkname) - target_path = os.path.realpath(target_path) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise LinkOutsideDestinationError(member, target_path) - return new_attrs - -def fully_trusted_filter(member, dest_path): - return member - -def tar_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, False) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -def data_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, True) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -_NAMED_FILTERS = { - "fully_trusted": fully_trusted_filter, - "tar": tar_filter, - "data": data_filter, -} - -#------------------ -# Exported Classes -#------------------ - -# Sentinel for replace() defaults, meaning "don't change the attribute" -_KEEP = object() - -class TarInfo(object): - """Informational class which holds the details about an - archive member given by a tar header block. - TarInfo objects are returned by TarFile.getmember(), - TarFile.getmembers() and TarFile.gettarinfo() and are - usually created internally. - """ - - __slots__ = dict( - name = 'Name of the archive member.', - mode = 'Permission bits.', - uid = 'User ID of the user who originally stored this member.', - gid = 'Group ID of the user who originally stored this member.', - size = 'Size in bytes.', - mtime = 'Time of last modification.', - chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), - linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), - uname = 'User name.', - gname = 'Group name.', - devmajor = 'Device major number.', - devminor = 'Device minor number.', - offset = 'The tar header starts here.', - offset_data = "The file's data starts here.", - pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), - sparse = 'Sparse member information.', - tarfile = None, - _sparse_structs = None, - _link_target = None, - ) - - def __init__(self, name=""): - """Construct a TarInfo object. name is the optional name - of the member. - """ - self.name = name # member name - self.mode = 0o644 # file permissions - self.uid = 0 # user id - self.gid = 0 # group id - self.size = 0 # file size - self.mtime = 0 # modification time - self.chksum = 0 # header checksum - self.type = REGTYPE # member type - self.linkname = "" # link name - self.uname = "" # user name - self.gname = "" # group name - self.devmajor = 0 # device major number - self.devminor = 0 # device minor number - - self.offset = 0 # the tar header starts here - self.offset_data = 0 # the file's data starts here - - self.sparse = None # sparse member information - self.pax_headers = {} # pax header information - - @property - def path(self): - 'In pax headers, "name" is called "path".' - return self.name - - @path.setter - def path(self, name): - self.name = name - - @property - def linkpath(self): - 'In pax headers, "linkname" is called "linkpath".' - return self.linkname - - @linkpath.setter - def linkpath(self, linkname): - self.linkname = linkname - - def __repr__(self): - return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) - - def replace(self, *, - name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, - uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, - deep=True, _KEEP=_KEEP): - """Return a deep copy of self with the given attributes replaced. - """ - if deep: - result = copy.deepcopy(self) - else: - result = copy.copy(self) - if name is not _KEEP: - result.name = name - if mtime is not _KEEP: - result.mtime = mtime - if mode is not _KEEP: - result.mode = mode - if linkname is not _KEEP: - result.linkname = linkname - if uid is not _KEEP: - result.uid = uid - if gid is not _KEEP: - result.gid = gid - if uname is not _KEEP: - result.uname = uname - if gname is not _KEEP: - result.gname = gname - return result - - def get_info(self): - """Return the TarInfo's attributes as a dictionary. - """ - if self.mode is None: - mode = None - else: - mode = self.mode & 0o7777 - info = { - "name": self.name, - "mode": mode, - "uid": self.uid, - "gid": self.gid, - "size": self.size, - "mtime": self.mtime, - "chksum": self.chksum, - "type": self.type, - "linkname": self.linkname, - "uname": self.uname, - "gname": self.gname, - "devmajor": self.devmajor, - "devminor": self.devminor - } - - if info["type"] == DIRTYPE and not info["name"].endswith("/"): - info["name"] += "/" - - return info - - def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): - """Return a tar header as a string of 512 byte blocks. - """ - info = self.get_info() - for name, value in info.items(): - if value is None: - raise ValueError("%s may not be None" % name) - - if format == USTAR_FORMAT: - return self.create_ustar_header(info, encoding, errors) - elif format == GNU_FORMAT: - return self.create_gnu_header(info, encoding, errors) - elif format == PAX_FORMAT: - return self.create_pax_header(info, encoding) - else: - raise ValueError("invalid format") - - def create_ustar_header(self, info, encoding, errors): - """Return the object as a ustar header block. - """ - info["magic"] = POSIX_MAGIC - - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - raise ValueError("linkname is too long") - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - info["prefix"], info["name"] = self._posix_split_name(info["name"], encoding, errors) - - return self._create_header(info, USTAR_FORMAT, encoding, errors) - - def create_gnu_header(self, info, encoding, errors): - """Return the object as a GNU header block sequence. - """ - info["magic"] = GNU_MAGIC - - buf = b"" - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) - - return buf + self._create_header(info, GNU_FORMAT, encoding, errors) - - def create_pax_header(self, info, encoding): - """Return the object as a ustar header block. If it cannot be - represented this way, prepend a pax extended header sequence - with supplement information. - """ - info["magic"] = POSIX_MAGIC - pax_headers = self.pax_headers.copy() - - # Test string fields for values that exceed the field length or cannot - # be represented in ASCII encoding. - for name, hname, length in ( - ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), - ("uname", "uname", 32), ("gname", "gname", 32)): - - if hname in pax_headers: - # The pax header has priority. - continue - - # Try to encode the string as ASCII. - try: - info[name].encode("ascii", "strict") - except UnicodeEncodeError: - pax_headers[hname] = info[name] - continue - - if len(info[name]) > length: - pax_headers[hname] = info[name] - - # Test number fields for values that exceed the field limit or values - # that like to be stored as float. - for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): - needs_pax = False - - val = info[name] - val_is_float = isinstance(val, float) - val_int = round(val) if val_is_float else val - if not 0 <= val_int < 8 ** (digits - 1): - # Avoid overflow. - info[name] = 0 - needs_pax = True - elif val_is_float: - # Put rounded value in ustar header, and full - # precision value in pax header. - info[name] = val_int - needs_pax = True - - # The existing pax header has priority. - if needs_pax and name not in pax_headers: - pax_headers[name] = str(val) - - # Create a pax extended header if necessary. - if pax_headers: - buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) - else: - buf = b"" - - return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") - - @classmethod - def create_pax_global_header(cls, pax_headers): - """Return the object as a pax global header block sequence. - """ - return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") - - def _posix_split_name(self, name, encoding, errors): - """Split a name longer than 100 chars into a prefix - and a name part. - """ - components = name.split("/") - for i in range(1, len(components)): - prefix = "/".join(components[:i]) - name = "/".join(components[i:]) - if len(prefix.encode(encoding, errors)) <= LENGTH_PREFIX and \ - len(name.encode(encoding, errors)) <= LENGTH_NAME: - break - else: - raise ValueError("name is too long") - - return prefix, name - - @staticmethod - def _create_header(info, format, encoding, errors): - """Return a header block. info is a dictionary with file - information, format must be one of the *_FORMAT constants. - """ - has_device_fields = info.get("type") in (CHRTYPE, BLKTYPE) - if has_device_fields: - devmajor = itn(info.get("devmajor", 0), 8, format) - devminor = itn(info.get("devminor", 0), 8, format) - else: - devmajor = stn("", 8, encoding, errors) - devminor = stn("", 8, encoding, errors) - - # None values in metadata should cause ValueError. - # itn()/stn() do this for all fields except type. - filetype = info.get("type", REGTYPE) - if filetype is None: - raise ValueError("TarInfo.type must not be None") - - parts = [ - stn(info.get("name", ""), 100, encoding, errors), - itn(info.get("mode", 0) & 0o7777, 8, format), - itn(info.get("uid", 0), 8, format), - itn(info.get("gid", 0), 8, format), - itn(info.get("size", 0), 12, format), - itn(info.get("mtime", 0), 12, format), - b" ", # checksum field - filetype, - stn(info.get("linkname", ""), 100, encoding, errors), - info.get("magic", POSIX_MAGIC), - stn(info.get("uname", ""), 32, encoding, errors), - stn(info.get("gname", ""), 32, encoding, errors), - devmajor, - devminor, - stn(info.get("prefix", ""), 155, encoding, errors) - ] - - buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) - chksum = calc_chksums(buf[-BLOCKSIZE:])[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - - @staticmethod - def _create_payload(payload): - """Return the string payload filled with zero bytes - up to the next 512 byte border. - """ - blocks, remainder = divmod(len(payload), BLOCKSIZE) - if remainder > 0: - payload += (BLOCKSIZE - remainder) * NUL - return payload - - @classmethod - def _create_gnu_long_header(cls, name, type, encoding, errors): - """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence - for name. - """ - name = name.encode(encoding, errors) + NUL - - info = {} - info["name"] = "././@LongLink" - info["type"] = type - info["size"] = len(name) - info["magic"] = GNU_MAGIC - - # create extended header + name blocks. - return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ - cls._create_payload(name) - - @classmethod - def _create_pax_generic_header(cls, pax_headers, type, encoding): - """Return a POSIX.1-2008 extended or global header sequence - that contains a list of keyword, value pairs. The values - must be strings. - """ - # Check if one of the fields contains surrogate characters and thereby - # forces hdrcharset=BINARY, see _proc_pax() for more information. - binary = False - for keyword, value in pax_headers.items(): - try: - value.encode("utf-8", "strict") - except UnicodeEncodeError: - binary = True - break - - records = b"" - if binary: - # Put the hdrcharset field at the beginning of the header. - records += b"21 hdrcharset=BINARY\n" - - for keyword, value in pax_headers.items(): - keyword = keyword.encode("utf-8") - if binary: - # Try to restore the original byte representation of `value'. - # Needless to say, that the encoding must match the string. - value = value.encode(encoding, "surrogateescape") - else: - value = value.encode("utf-8") - - l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' - n = p = 0 - while True: - n = l + len(str(p)) - if n == p: - break - p = n - records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" - - # We use a hardcoded "././@PaxHeader" name like star does - # instead of the one that POSIX recommends. - info = {} - info["name"] = "././@PaxHeader" - info["type"] = type - info["size"] = len(records) - info["magic"] = POSIX_MAGIC - - # Create pax header + record blocks. - return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ - cls._create_payload(records) - - @classmethod - def frombuf(cls, buf, encoding, errors): - """Construct a TarInfo object from a 512 byte bytes object. - """ - if len(buf) == 0: - raise EmptyHeaderError("empty header") - if len(buf) != BLOCKSIZE: - raise TruncatedHeaderError("truncated header") - if buf.count(NUL) == BLOCKSIZE: - raise EOFHeaderError("end of file header") - - chksum = nti(buf[148:156]) - if chksum not in calc_chksums(buf): - raise InvalidHeaderError("bad checksum") - - obj = cls() - obj.name = nts(buf[0:100], encoding, errors) - obj.mode = nti(buf[100:108]) - obj.uid = nti(buf[108:116]) - obj.gid = nti(buf[116:124]) - obj.size = nti(buf[124:136]) - obj.mtime = nti(buf[136:148]) - obj.chksum = chksum - obj.type = buf[156:157] - obj.linkname = nts(buf[157:257], encoding, errors) - obj.uname = nts(buf[265:297], encoding, errors) - obj.gname = nts(buf[297:329], encoding, errors) - obj.devmajor = nti(buf[329:337]) - obj.devminor = nti(buf[337:345]) - prefix = nts(buf[345:500], encoding, errors) - - # Old V7 tar format represents a directory as a regular - # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): - obj.type = DIRTYPE - - # The old GNU sparse format occupies some of the unused - # space in the buffer for up to 4 sparse structures. - # Save them for later processing in _proc_sparse(). - if obj.type == GNUTYPE_SPARSE: - pos = 386 - structs = [] - for i in range(4): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[482]) - origsize = nti(buf[483:495]) - obj._sparse_structs = (structs, isextended, origsize) - - # Remove redundant slashes from directories. - if obj.isdir(): - obj.name = obj.name.rstrip("/") - - # Reconstruct a ustar longname. - if prefix and obj.type not in GNU_TYPES: - obj.name = prefix + "/" + obj.name - return obj - - @classmethod - def fromtarfile(cls, tarfile): - """Return the next TarInfo object from TarFile object - tarfile. - """ - buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) - obj.offset = tarfile.fileobj.tell() - BLOCKSIZE - return obj._proc_member(tarfile) - - #-------------------------------------------------------------------------- - # The following are methods that are called depending on the type of a - # member. The entry point is _proc_member() which can be overridden in a - # subclass to add custom _proc_*() methods. A _proc_*() method MUST - # implement the following - # operations: - # 1. Set self.offset_data to the position where the data blocks begin, - # if there is data that follows. - # 2. Set tarfile.offset to the position where the next member's header will - # begin. - # 3. Return self or another valid TarInfo object. - def _proc_member(self, tarfile): - """Choose the right processing method depending on - the type and call it. - """ - if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): - return self._proc_gnulong(tarfile) - elif self.type == GNUTYPE_SPARSE: - return self._proc_sparse(tarfile) - elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): - return self._proc_pax(tarfile) - else: - return self._proc_builtin(tarfile) - - def _proc_builtin(self, tarfile): - """Process a builtin type or an unknown type which - will be treated as a regular file. - """ - self.offset_data = tarfile.fileobj.tell() - offset = self.offset_data - if self.isreg() or self.type not in SUPPORTED_TYPES: - # Skip the following data blocks. - offset += self._block(self.size) - tarfile.offset = offset - - # Patch the TarInfo object with saved global - # header information. - self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if self.isdir(): - self.name = self.name.rstrip("/") - - return self - - def _proc_gnulong(self, tarfile): - """Process the blocks that hold a GNU longname - or longlink member. - """ - buf = tarfile.fileobj.read(self._block(self.size)) - - # Fetch the next header and process it. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Patch the TarInfo object from the next header with - # the longname information. - next.offset = self.offset - if self.type == GNUTYPE_LONGNAME: - next.name = nts(buf, tarfile.encoding, tarfile.errors) - elif self.type == GNUTYPE_LONGLINK: - next.linkname = nts(buf, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if next.isdir(): - next.name = next.name.removesuffix("/") - - return next - - def _proc_sparse(self, tarfile): - """Process a GNU sparse header plus extra headers. - """ - # We already collected some sparse structures in frombuf(). - structs, isextended, origsize = self._sparse_structs - del self._sparse_structs - - # Collect sparse structures from extended header blocks. - while isextended: - buf = tarfile.fileobj.read(BLOCKSIZE) - pos = 0 - for i in range(21): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset and numbytes: - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[504]) - self.sparse = structs - - self.offset_data = tarfile.fileobj.tell() - tarfile.offset = self.offset_data + self._block(self.size) - self.size = origsize - return self - - def _proc_pax(self, tarfile): - """Process an extended or global header as described in - POSIX.1-2008. - """ - # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) - - # A pax header stores supplemental information for either - # the following file (extended) or all following files - # (global). - if self.type == XGLTYPE: - pax_headers = tarfile.pax_headers - else: - pax_headers = tarfile.pax_headers.copy() - - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - - # Parse pax header information. A record looks like that: - # "%d %s=%s\n" % (length, keyword, value). length is the size - # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") - pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: - raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] - - # Normally, we could just use "utf-8" as the encoding and "strict" - # as the error handler, but we better not take the risk. For - # example, GNU tar <= 1.23 is known to store filenames it cannot - # translate to UTF-8 as raw strings (unfortunately without a - # hdrcharset=BINARY header). - # We first try the strict standard encoding, and if that fails we - # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", - tarfile.errors) - if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, - tarfile.errors) - else: - value = self._decode_pax_field(value, "utf-8", "utf-8", - tarfile.errors) - - pax_headers[keyword] = value - pos += length - - # Fetch the next header. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Process GNU sparse information. - if "GNU.sparse.map" in pax_headers: - # GNU extended sparse format version 0.1. - self._proc_gnusparse_01(next, pax_headers) - - elif "GNU.sparse.size" in pax_headers: - # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) - - elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": - # GNU extended sparse format version 1.0. - self._proc_gnusparse_10(next, pax_headers, tarfile) - - if self.type in (XHDTYPE, SOLARIS_XHDTYPE): - # Patch the TarInfo object with the extended header info. - next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset - - if "size" in pax_headers: - # If the extended header replaces the size field, - # we need to recalculate the offset where the next - # header starts. - offset = next.offset_data - if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) - tarfile.offset = offset - - return next - - def _proc_gnusparse_00(self, next, pax_headers, buf): - """Process a GNU tar extended sparse header, version 0.0. - """ - offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) - numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) - next.sparse = list(zip(offsets, numbytes)) - - def _proc_gnusparse_01(self, next, pax_headers): - """Process a GNU tar extended sparse header, version 0.1. - """ - sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _proc_gnusparse_10(self, next, pax_headers, tarfile): - """Process a GNU tar extended sparse header, version 1.0. - """ - fields = None - sparse = [] - buf = tarfile.fileobj.read(BLOCKSIZE) - fields, buf = buf.split(b"\n", 1) - fields = int(fields) - while len(sparse) < fields * 2: - if b"\n" not in buf: - buf += tarfile.fileobj.read(BLOCKSIZE) - number, buf = buf.split(b"\n", 1) - sparse.append(int(number)) - next.offset_data = tarfile.fileobj.tell() - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _apply_pax_info(self, pax_headers, encoding, errors): - """Replace fields with supplemental information from a previous - pax extended or global header. - """ - for keyword, value in pax_headers.items(): - if keyword == "GNU.sparse.name": - setattr(self, "path", value) - elif keyword == "GNU.sparse.size": - setattr(self, "size", int(value)) - elif keyword == "GNU.sparse.realsize": - setattr(self, "size", int(value)) - elif keyword in PAX_FIELDS: - if keyword in PAX_NUMBER_FIELDS: - try: - value = PAX_NUMBER_FIELDS[keyword](value) - except ValueError: - value = 0 - if keyword == "path": - value = value.rstrip("/") - setattr(self, keyword, value) - - self.pax_headers = pax_headers.copy() - - def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): - """Decode a single field from a pax record. - """ - try: - return value.decode(encoding, "strict") - except UnicodeDecodeError: - return value.decode(fallback_encoding, fallback_errors) - - def _block(self, count): - """Round up a byte count by BLOCKSIZE and return it, - e.g. _block(834) => 1024. - """ - blocks, remainder = divmod(count, BLOCKSIZE) - if remainder: - blocks += 1 - return blocks * BLOCKSIZE - - def isreg(self): - 'Return True if the Tarinfo object is a regular file.' - return self.type in REGULAR_TYPES - - def isfile(self): - 'Return True if the Tarinfo object is a regular file.' - return self.isreg() - - def isdir(self): - 'Return True if it is a directory.' - return self.type == DIRTYPE - - def issym(self): - 'Return True if it is a symbolic link.' - return self.type == SYMTYPE - - def islnk(self): - 'Return True if it is a hard link.' - return self.type == LNKTYPE - - def ischr(self): - 'Return True if it is a character device.' - return self.type == CHRTYPE - - def isblk(self): - 'Return True if it is a block device.' - return self.type == BLKTYPE - - def isfifo(self): - 'Return True if it is a FIFO.' - return self.type == FIFOTYPE - - def issparse(self): - return self.sparse is not None - - def isdev(self): - 'Return True if it is one of character device, block device or FIFO.' - return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) -# class TarInfo - -class TarFile(object): - """The TarFile Class provides an interface to tar archives. - """ - - debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) - - dereference = False # If true, add content of linked file to the - # tar file, else the link. - - ignore_zeros = False # If true, skips empty or invalid blocks and - # continues processing. - - errorlevel = 1 # If 0, fatal errors only appear in debug - # messages (if debug >= 0). If > 0, errors - # are passed to the caller as exceptions. - - format = DEFAULT_FORMAT # The format to use when creating an archive. - - encoding = ENCODING # Encoding for 8-bit character strings. - - errors = None # Error handler for unicode conversion. - - tarinfo = TarInfo # The default TarInfo class to use. - - fileobject = ExFileObject # The file-object for extractfile(). - - extraction_filter = None # The default filter for extraction. - - def __init__(self, name=None, mode="r", fileobj=None, format=None, - tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, - errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to - read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' - defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. - """ - modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} - if mode not in modes: - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - self.mode = mode - self._mode = modes[mode] - - if not fileobj: - if self.mode == "a" and not os.path.exists(name): - # Create nonexistent files in append mode. - self.mode = "w" - self._mode = "wb" - fileobj = bltn_open(name, self._mode) - self._extfileobj = False - else: - if (name is None and hasattr(fileobj, "name") and - isinstance(fileobj.name, (str, bytes))): - name = fileobj.name - if hasattr(fileobj, "mode"): - self._mode = fileobj.mode - self._extfileobj = True - self.name = os.path.abspath(name) if name else None - self.fileobj = fileobj - - # Init attributes. - if format is not None: - self.format = format - if tarinfo is not None: - self.tarinfo = tarinfo - if dereference is not None: - self.dereference = dereference - if ignore_zeros is not None: - self.ignore_zeros = ignore_zeros - if encoding is not None: - self.encoding = encoding - self.errors = errors - - if pax_headers is not None and self.format == PAX_FORMAT: - self.pax_headers = pax_headers - else: - self.pax_headers = {} - - if debug is not None: - self.debug = debug - if errorlevel is not None: - self.errorlevel = errorlevel - - # Init datastructures. - self.copybufsize = copybufsize - self.closed = False - self.members = [] # list of members as TarInfo objects - self._loaded = False # flag if all members have been read - self.offset = self.fileobj.tell() - # current position in the archive file - self.inodes = {} # dictionary caching the inodes of - # archive members already added - - try: - if self.mode == "r": - self.firstmember = None - self.firstmember = self.next() - - if self.mode == "a": - # Move to the end of the archive, - # before the first empty block. - while True: - self.fileobj.seek(self.offset) - try: - tarinfo = self.tarinfo.fromtarfile(self) - self.members.append(tarinfo) - except EOFHeaderError: - self.fileobj.seek(self.offset) - break - except HeaderError as e: - raise ReadError(str(e)) from None - - if self.mode in ("a", "w", "x"): - self._loaded = True - - if self.pax_headers: - buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) - self.fileobj.write(buf) - self.offset += len(buf) - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - #-------------------------------------------------------------------------- - # Below are the classmethods which act as alternate constructors to the - # TarFile class. The open() method is the only one that is needed for - # public use; it is the "super"-constructor and is able to select an - # adequate "sub"-constructor for a particular compression using the mapping - # from OPEN_METH. - # - # This concept allows one to subclass TarFile without losing the comfort of - # the super-constructor. A sub-constructor is registered and made available - # by adding it to the mapping in OPEN_METH. - - @classmethod - def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): - r"""Open a tar archive for reading, writing or appending. Return - an appropriate TarFile class. - - mode: - 'r' or 'r:\*' open for reading with transparent compression - 'r:' open for reading exclusively uncompressed - 'r:gz' open for reading with gzip compression - 'r:bz2' open for reading with bzip2 compression - 'r:xz' open for reading with lzma compression - 'a' or 'a:' open for appending, creating the file if necessary - 'w' or 'w:' open for writing without compression - 'w:gz' open for writing with gzip compression - 'w:bz2' open for writing with bzip2 compression - 'w:xz' open for writing with lzma compression - - 'x' or 'x:' create a tarfile exclusively without compression, raise - an exception if the file is already created - 'x:gz' create a gzip compressed tarfile, raise an exception - if the file is already created - 'x:bz2' create a bzip2 compressed tarfile, raise an exception - if the file is already created - 'x:xz' create an lzma compressed tarfile, raise an exception - if the file is already created - - 'r|\*' open a stream of tar blocks with transparent compression - 'r|' open an uncompressed stream of tar blocks for reading - 'r|gz' open a gzip compressed stream of tar blocks - 'r|bz2' open a bzip2 compressed stream of tar blocks - 'r|xz' open an lzma compressed stream of tar blocks - 'w|' open an uncompressed stream for writing - 'w|gz' open a gzip compressed stream for writing - 'w|bz2' open a bzip2 compressed stream for writing - 'w|xz' open an lzma compressed stream for writing - """ - - if not name and not fileobj: - raise ValueError("nothing to open") - - if mode in ("r", "r:*"): - # Find out which *open() is appropriate for opening the file. - def not_compressed(comptype): - return cls.OPEN_METH[comptype] == 'taropen' - error_msgs = [] - for comptype in sorted(cls.OPEN_METH, key=not_compressed): - func = getattr(cls, cls.OPEN_METH[comptype]) - if fileobj is not None: - saved_pos = fileobj.tell() - try: - return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError) as e: - error_msgs.append(f'- method {comptype}: {e!r}') - if fileobj is not None: - fileobj.seek(saved_pos) - continue - error_msgs_summary = '\n'.join(error_msgs) - raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}") - - elif ":" in mode: - filemode, comptype = mode.split(":", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - # Select the *open() function according to - # given compression. - if comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - else: - raise CompressionError("unknown compression type %r" % comptype) - return func(name, filemode, fileobj, **kwargs) - - elif "|" in mode: - filemode, comptype = mode.split("|", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - if filemode not in ("r", "w"): - raise ValueError("mode must be 'r' or 'w'") - - compresslevel = kwargs.pop("compresslevel", 9) - stream = _Stream(name, filemode, comptype, fileobj, bufsize, - compresslevel) - try: - t = cls(name, filemode, stream, **kwargs) - except: - stream.close() - raise - t._extfileobj = False - return t - - elif mode in ("a", "w", "x"): - return cls.taropen(name, mode, fileobj, **kwargs) - - raise ValueError("undiscernible mode") - - @classmethod - def taropen(cls, name, mode="r", fileobj=None, **kwargs): - """Open uncompressed tar archive name for reading or writing. - """ - if mode not in ("r", "a", "w", "x"): - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - return cls(name, mode, fileobj, **kwargs) - - @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open gzip compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from gzip import GzipFile - except ImportError: - raise CompressionError("gzip module is not available") from None - - try: - fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) - except OSError as e: - if fileobj is not None and mode == 'r': - raise ReadError("not a gzip file") from e - raise - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except OSError as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a gzip file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open bzip2 compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from bz2 import BZ2File - except ImportError: - raise CompressionError("bz2 module is not available") from None - - fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (OSError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a bzip2 file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): - """Open lzma compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from lzma import LZMAFile, LZMAError - except ImportError: - raise CompressionError("lzma module is not available") from None - - fileobj = LZMAFile(fileobj or name, mode, preset=preset) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (LZMAError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not an lzma file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - # All *open() methods are registered here. - OPEN_METH = { - "tar": "taropen", # uncompressed tar - "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar - } - - #-------------------------------------------------------------------------- - # The public methods which TarFile provides: - - def close(self): - """Close the TarFile. In write-mode, two finishing zero blocks are - appended to the archive. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode in ("a", "w", "x"): - self.fileobj.write(NUL * (BLOCKSIZE * 2)) - self.offset += (BLOCKSIZE * 2) - # fill up the end with zero-blocks - # (like option -b20 for tar does) - blocks, remainder = divmod(self.offset, RECORDSIZE) - if remainder > 0: - self.fileobj.write(NUL * (RECORDSIZE - remainder)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def getmember(self, name): - """Return a TarInfo object for member ``name``. If ``name`` can not be - found in the archive, KeyError is raised. If a member occurs more - than once in the archive, its last occurrence is assumed to be the - most up-to-date version. - """ - tarinfo = self._getmember(name.rstrip('/')) - if tarinfo is None: - raise KeyError("filename %r not found" % name) - return tarinfo - - def getmembers(self): - """Return the members of the archive as a list of TarInfo objects. The - list has the same order as the members in the archive. - """ - self._check() - if not self._loaded: # if we want to obtain a list of - self._load() # all members, we first have to - # scan the whole archive. - return self.members - - def getnames(self): - """Return the members of the archive as a list of their names. It has - the same order as the list returned by getmembers(). - """ - return [tarinfo.name for tarinfo in self.getmembers()] - - def gettarinfo(self, name=None, arcname=None, fileobj=None): - """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by ``name``, or - specified as a file object ``fileobj`` with a file descriptor. If - given, ``arcname`` specifies an alternative name for the file in the - archive, otherwise, the name is taken from the 'name' attribute of - 'fileobj', or the 'name' argument. The name should be a text - string. - """ - self._check("awx") - - # When fileobj is given, replace name by - # fileobj's real name. - if fileobj is not None: - name = fileobj.name - - # Building the name of the member in the archive. - # Backward slashes are converted to forward slashes, - # Absolute paths are turned to relative paths. - if arcname is None: - arcname = name - drv, arcname = os.path.splitdrive(arcname) - arcname = arcname.replace(os.sep, "/") - arcname = arcname.lstrip("/") - - # Now, fill the TarInfo object with - # information specific for the file. - tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed - - # Use os.stat or os.lstat, depending on if symlinks shall be resolved. - if fileobj is None: - if not self.dereference: - statres = os.lstat(name) - else: - statres = os.stat(name) - else: - statres = os.fstat(fileobj.fileno()) - linkname = "" - - stmd = statres.st_mode - if stat.S_ISREG(stmd): - inode = (statres.st_ino, statres.st_dev) - if not self.dereference and statres.st_nlink > 1 and \ - inode in self.inodes and arcname != self.inodes[inode]: - # Is it a hardlink to an already - # archived file? - type = LNKTYPE - linkname = self.inodes[inode] - else: - # The inode is added only if its valid. - # For win32 it is always 0. - type = REGTYPE - if inode[0]: - self.inodes[inode] = arcname - elif stat.S_ISDIR(stmd): - type = DIRTYPE - elif stat.S_ISFIFO(stmd): - type = FIFOTYPE - elif stat.S_ISLNK(stmd): - type = SYMTYPE - linkname = os.readlink(name) - elif stat.S_ISCHR(stmd): - type = CHRTYPE - elif stat.S_ISBLK(stmd): - type = BLKTYPE - else: - return None - - # Fill the TarInfo object with all - # information we can get. - tarinfo.name = arcname - tarinfo.mode = stmd - tarinfo.uid = statres.st_uid - tarinfo.gid = statres.st_gid - if type == REGTYPE: - tarinfo.size = statres.st_size - else: - tarinfo.size = 0 - tarinfo.mtime = statres.st_mtime - tarinfo.type = type - tarinfo.linkname = linkname - if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if type in (CHRTYPE, BLKTYPE): - if hasattr(os, "major") and hasattr(os, "minor"): - tarinfo.devmajor = os.major(statres.st_rdev) - tarinfo.devminor = os.minor(statres.st_rdev) - return tarinfo - - def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If ``verbose`` is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. ``members`` is optional and must be a subset of the - list returned by getmembers(). - """ - self._check() - - if members is None: - members = self - for tarinfo in members: - if verbose: - if tarinfo.mode is None: - _safe_print("??????????") - else: - _safe_print(stat.filemode(tarinfo.mode)) - _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid)) - if tarinfo.ischr() or tarinfo.isblk(): - _safe_print("%10s" % - ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) - else: - _safe_print("%10d" % tarinfo.size) - if tarinfo.mtime is None: - _safe_print("????-??-?? ??:??:??") - else: - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) - - _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) - - if verbose: - if tarinfo.issym(): - _safe_print("-> " + tarinfo.linkname) - if tarinfo.islnk(): - _safe_print("link to " + tarinfo.linkname) - print() - - def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file ``name`` to the archive. ``name`` may be any type of file - (directory, fifo, symbolic link, etc.). If given, ``arcname`` - specifies an alternative name for the file in the archive. - Directories are added recursively by default. This can be avoided by - setting ``recursive`` to False. ``filter`` is a function - that expects a TarInfo object argument and returns the changed - TarInfo object, if it returns None the TarInfo object will be - excluded from the archive. - """ - self._check("awx") - - if arcname is None: - arcname = name - - # Skip if somebody tries to archive the archive... - if self.name is not None and os.path.abspath(name) == self.name: - self._dbg(2, "tarfile: Skipped %r" % name) - return - - self._dbg(1, name) - - # Create a TarInfo object from the file. - tarinfo = self.gettarinfo(name, arcname) - - if tarinfo is None: - self._dbg(1, "tarfile: Unsupported type %r" % name) - return - - # Change or exclude the TarInfo object. - if filter is not None: - tarinfo = filter(tarinfo) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Append the tar header and data to the archive. - if tarinfo.isreg(): - with bltn_open(name, "rb") as f: - self.addfile(tarinfo, f) - - elif tarinfo.isdir(): - self.addfile(tarinfo) - if recursive: - for f in sorted(os.listdir(name)): - self.add(os.path.join(name, f), os.path.join(arcname, f), - recursive, filter=filter) - - else: - self.addfile(tarinfo) - - def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object ``tarinfo`` to the archive. If ``fileobj`` is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). - """ - self._check("awx") - - tarinfo = copy.copy(tarinfo) - - buf = tarinfo.tobuf(self.format, self.encoding, self.errors) - self.fileobj.write(buf) - self.offset += len(buf) - bufsize=self.copybufsize - # If there's data to follow, append it. - if fileobj is not None: - copyfileobj(fileobj, self.fileobj, tarinfo.size, bufsize=bufsize) - blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) - if remainder > 0: - self.fileobj.write(NUL * (BLOCKSIZE - remainder)) - blocks += 1 - self.offset += blocks * BLOCKSIZE - - self.members.append(tarinfo) - - def _get_filter_function(self, filter): - if filter is None: - filter = self.extraction_filter - if filter is None: - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter - if isinstance(filter, str): - raise TypeError( - 'String names are not supported for ' - + 'TarFile.extraction_filter. Use a function such as ' - + 'tarfile.data_filter directly.') - return filter - if callable(filter): - return filter - try: - return _NAMED_FILTERS[filter] - except KeyError: - raise ValueError(f"filter {filter!r} not found") from None - - def extractall(self, path=".", members=None, *, numeric_owner=False, - filter=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only - the numbers for user/group names are used and not the names. - - The `filter` function will be called on each member just - before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - directories = [] - - filter_function = self._get_filter_function(filter) - if members is None: - members = self - - for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is None: - continue - if tarinfo.isdir(): - # For directories, delay setting attributes until later, - # since permissions can interfere with extraction and - # extracting contents can reset mtime. - directories.append(tarinfo) - self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) - - # Reverse sort directories. - directories.sort(key=lambda a: a.name, reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, - filter=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` - is True, only the numbers for user/group names are used and not - the names. - - The `filter` function will be called before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) - - def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - unfiltered = tarinfo - try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo - - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" - self._check("r") - - try: - self._extract_member(tarinfo, os.path.join(path, tarinfo.name), - set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def _handle_nonfatal_error(self, e): - """Handle non-fatal error (ExtractError) according to errorlevel""" - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def _handle_fatal_error(self, e): - """Handle "fatal" error according to self.errorlevel""" - if self.errorlevel > 0: - raise - elif isinstance(e, OSError): - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) - else: - self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) - - def extractfile(self, member): - """Extract a member from the archive as a file object. ``member`` may be - a filename or a TarInfo object. If ``member`` is a regular file or - a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If ``member`` does not appear - in the archive, KeyError is raised. - """ - self._check("r") - - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES: - # Members with unknown types are treated as regular files. - return self.fileobject(self, tarinfo) - - elif tarinfo.islnk() or tarinfo.issym(): - if isinstance(self.fileobj, _Stream): - # A small but ugly workaround for the case that someone tries - # to extract a (sym)link as a file-object from a non-seekable - # stream of tar blocks. - raise StreamError("cannot extract (sym)link as file object") - else: - # A (sym)link's file object is its target's file object. - return self.extractfile(self._find_link_target(tarinfo)) - else: - # If there's no data associated with the member (directory, chrdev, - # blkdev, etc.), return None instead of a file object. - return None - - def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical - file called targetpath. - """ - # Fetch the TarInfo object for the given name - # and build the destination pathname, replacing - # forward slashes to platform specific separators. - targetpath = targetpath.rstrip("/") - targetpath = targetpath.replace("/", os.sep) - - # Create all upper directories. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - # Create directories that are not part of the archive with - # default permissions. - os.makedirs(upperdirs) - - if tarinfo.islnk() or tarinfo.issym(): - self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) - else: - self._dbg(1, tarinfo.name) - - if tarinfo.isreg(): - self.makefile(tarinfo, targetpath) - elif tarinfo.isdir(): - self.makedir(tarinfo, targetpath) - elif tarinfo.isfifo(): - self.makefifo(tarinfo, targetpath) - elif tarinfo.ischr() or tarinfo.isblk(): - self.makedev(tarinfo, targetpath) - elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) - elif tarinfo.type not in SUPPORTED_TYPES: - self.makeunknown(tarinfo, targetpath) - else: - self.makefile(tarinfo, targetpath) - - if set_attrs: - self.chown(tarinfo, targetpath, numeric_owner) - if not tarinfo.issym(): - self.chmod(tarinfo, targetpath) - self.utime(tarinfo, targetpath) - - #-------------------------------------------------------------------------- - # Below are the different file methods. They are called via - # _extract_member() when extract() is called. They can be replaced in a - # subclass to implement other functionality. - - def makedir(self, tarinfo, targetpath): - """Make a directory called targetpath. - """ - try: - if tarinfo.mode is None: - # Use the system's default mode - os.mkdir(targetpath) - else: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) - except FileExistsError: - if not os.path.isdir(targetpath): - raise - - def makefile(self, tarinfo, targetpath): - """Make a file called targetpath. - """ - source = self.fileobj - source.seek(tarinfo.offset_data) - bufsize = self.copybufsize - with bltn_open(targetpath, "wb") as target: - if tarinfo.sparse is not None: - for offset, size in tarinfo.sparse: - target.seek(offset) - copyfileobj(source, target, size, ReadError, bufsize) - target.seek(tarinfo.size) - target.truncate() - else: - copyfileobj(source, target, tarinfo.size, ReadError, bufsize) - - def makeunknown(self, tarinfo, targetpath): - """Make a file from a TarInfo object with an unknown type - at targetpath. - """ - self.makefile(tarinfo, targetpath) - self._dbg(1, "tarfile: Unknown file type %r, " \ - "extracted as regular file." % tarinfo.type) - - def makefifo(self, tarinfo, targetpath): - """Make a fifo called targetpath. - """ - if hasattr(os, "mkfifo"): - os.mkfifo(targetpath) - else: - raise ExtractError("fifo not supported by system") - - def makedev(self, tarinfo, targetpath): - """Make a character or block device called targetpath. - """ - if not hasattr(os, "mknod") or not hasattr(os, "makedev"): - raise ExtractError("special devices not supported by system") - - mode = tarinfo.mode - if mode is None: - # Use mknod's default - mode = 0o600 - if tarinfo.isblk(): - mode |= stat.S_IFBLK - else: - mode |= stat.S_IFCHR - - os.mknod(targetpath, mode, - os.makedev(tarinfo.devmajor, tarinfo.devminor)) - - def makelink(self, tarinfo, targetpath): - """Make a (symbolic) link called targetpath. If it cannot be created - (platform limitation), we try to make a copy of the referenced file - instead of a link. - """ - try: - # For systems that support symbolic and hard links. - if tarinfo.issym(): - if os.path.lexists(targetpath): - # Avoid FileExistsError on following os.symlink. - os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) - else: - if os.path.exists(tarinfo._link_target): - os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except symlink_exception: - try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None - - def chown(self, tarinfo, targetpath, numeric_owner): - """Set owner of targetpath according to tarinfo. If numeric_owner - is True, use .gid/.uid instead of .gname/.uname. If numeric_owner - is False, fall back to .gid/.uid when the search based on name - fails. - """ - if hasattr(os, "geteuid") and os.geteuid() == 0: - # We have to be root to do so. - g = tarinfo.gid - u = tarinfo.uid - if not numeric_owner: - try: - if grp and tarinfo.gname: - g = grp.getgrnam(tarinfo.gname)[2] - except KeyError: - pass - try: - if pwd and tarinfo.uname: - u = pwd.getpwnam(tarinfo.uname)[2] - except KeyError: - pass - if g is None: - g = -1 - if u is None: - u = -1 - try: - if tarinfo.issym() and hasattr(os, "lchown"): - os.lchown(targetpath, u, g) - else: - os.chown(targetpath, u, g) - except OSError as e: - raise ExtractError("could not change owner") from e - - def chmod(self, tarinfo, targetpath): - """Set file permissions of targetpath according to tarinfo. - """ - if tarinfo.mode is None: - return - try: - os.chmod(targetpath, tarinfo.mode) - except OSError as e: - raise ExtractError("could not change mode") from e - - def utime(self, tarinfo, targetpath): - """Set modification time of targetpath according to tarinfo. - """ - mtime = tarinfo.mtime - if mtime is None: - return - if not hasattr(os, 'utime'): - return - try: - os.utime(targetpath, (mtime, mtime)) - except OSError as e: - raise ExtractError("could not change modification time") from e - - #-------------------------------------------------------------------------- - def next(self): - """Return the next member of the archive as a TarInfo object, when - TarFile is opened for reading. Return None if there is no more - available. - """ - self._check("ra") - if self.firstmember is not None: - m = self.firstmember - self.firstmember = None - return m - - # Advance the file pointer. - if self.offset != self.fileobj.tell(): - if self.offset == 0: - return None - self.fileobj.seek(self.offset - 1) - if not self.fileobj.read(1): - raise ReadError("unexpected end of data") - - # Read the next block. - tarinfo = None - while True: - try: - tarinfo = self.tarinfo.fromtarfile(self) - except EOFHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - except InvalidHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - elif self.offset == 0: - raise ReadError(str(e)) from None - except EmptyHeaderError: - if self.offset == 0: - raise ReadError("empty file") from None - except TruncatedHeaderError as e: - if self.offset == 0: - raise ReadError(str(e)) from None - except SubsequentHeaderError as e: - raise ReadError(str(e)) from None - except Exception as e: - try: - import zlib - if isinstance(e, zlib.error): - raise ReadError(f'zlib error: {e}') from None - else: - raise e - except ImportError: - raise e - break - - if tarinfo is not None: - self.members.append(tarinfo) - else: - self._loaded = True - - return tarinfo - - #-------------------------------------------------------------------------- - # Little helper methods: - - def _getmember(self, name, tarinfo=None, normalize=False): - """Find an archive member by name from bottom to top. - If tarinfo is given, it is used as the starting point. - """ - # Ensure that all members have been loaded. - members = self.getmembers() - - # Limit the member search list up to tarinfo. - skipping = False - if tarinfo is not None: - try: - index = members.index(tarinfo) - except ValueError: - # The given starting point might be a (modified) copy. - # We'll later skip members until we find an equivalent. - skipping = True - else: - # Happy fast path - members = members[:index] - - if normalize: - name = os.path.normpath(name) - - for member in reversed(members): - if skipping: - if tarinfo.offset == member.offset: - skipping = False - continue - if normalize: - member_name = os.path.normpath(member.name) - else: - member_name = member.name - - if name == member_name: - return member - - if skipping: - # Starting point was not found - raise ValueError(tarinfo) - - def _load(self): - """Read through the entire archive file and look for readable - members. - """ - while self.next() is not None: - pass - self._loaded = True - - def _check(self, mode=None): - """Check if TarFile is still open, and if the operation's mode - corresponds to TarFile's mode. - """ - if self.closed: - raise OSError("%s is closed" % self.__class__.__name__) - if mode is not None and self.mode not in mode: - raise OSError("bad operation for mode %r" % self.mode) - - def _find_link_target(self, tarinfo): - """Find the target member of a symlink or hardlink member in the - archive. - """ - if tarinfo.issym(): - # Always search the entire archive. - linkname = "/".join(filter(None, (os.path.dirname(tarinfo.name), tarinfo.linkname))) - limit = None - else: - # Search the archive before the link, because a hard link is - # just a reference to an already archived file. - linkname = tarinfo.linkname - limit = tarinfo - - member = self._getmember(linkname, tarinfo=limit, normalize=True) - if member is None: - raise KeyError("linkname %r not found" % linkname) - return member - - def __iter__(self): - """Provide an iterator object. - """ - if self._loaded: - yield from self.members - return - - # Yield items using TarFile's next() method. - # When all members have been read, set TarFile as _loaded. - index = 0 - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will have already exhausted the next() method. - if self.firstmember is not None: - tarinfo = self.next() - index += 1 - yield tarinfo - - while True: - if index < len(self.members): - tarinfo = self.members[index] - elif not self._loaded: - tarinfo = self.next() - if not tarinfo: - self._loaded = True - return - else: - return - index += 1 - yield tarinfo - - def _dbg(self, level, msg): - """Write debugging output to sys.stderr. - """ - if level <= self.debug: - print(msg, file=sys.stderr) - - def __enter__(self): - self._check() - return self - - def __exit__(self, type, value, traceback): - if type is None: - self.close() - else: - # An exception occurred. We must not call close() because - # it would try to write end-of-archive blocks and padding. - if not self._extfileobj: - self.fileobj.close() - self.closed = True - -#-------------------- -# exported functions -#-------------------- - -def is_tarfile(name): - """Return True if name points to a tar archive that we - are able to handle, else return False. - - 'name' should be a string, file, or file-like object. - """ - try: - if hasattr(name, "read"): - pos = name.tell() - t = open(fileobj=name) - name.seek(pos) - else: - t = open(name) - t.close() - return True - except TarError: - return False - -open = TarFile.open - - -def main(): - import argparse - - description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='Verbose output') - parser.add_argument('--filter', metavar='<filtername>', - choices=_NAMED_FILTERS, - help='Filter for extraction') - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='<tarfile>', - help='Show listing of a tarfile') - group.add_argument('-e', '--extract', nargs='+', - metavar=('<tarfile>', '<output_dir>'), - help='Extract tarfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('<name>', '<file>'), - help='Create tarfile from sources') - group.add_argument('-t', '--test', metavar='<tarfile>', - help='Test if a tarfile is valid') - - args = parser.parse_args() - - if args.filter and args.extract is None: - parser.exit(1, '--filter is only valid for extraction\n') - - if args.test is not None: - src = args.test - if is_tarfile(src): - with open(src, 'r') as tar: - tar.getmembers() - print(tar.getmembers(), file=sys.stderr) - if args.verbose: - print('{!r} is a tar archive.'.format(src)) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.list is not None: - src = args.list - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.list(verbose=args.verbose) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.extract is not None: - if len(args.extract) == 1: - src = args.extract[0] - curdir = os.curdir - elif len(args.extract) == 2: - src, curdir = args.extract - else: - parser.exit(1, parser.format_help()) - - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir, filter=args.filter) - if args.verbose: - if curdir == '.': - msg = '{!r} file is extracted.'.format(src) - else: - msg = ('{!r} file is extracted ' - 'into {!r} directory.').format(src, curdir) - print(msg) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.create is not None: - tar_name = args.create.pop(0) - _, ext = os.path.splitext(tar_name) - compressions = { - # gz - '.gz': 'gz', - '.tgz': 'gz', - # xz - '.xz': 'xz', - '.txz': 'xz', - # bz2 - '.bz2': 'bz2', - '.tbz': 'bz2', - '.tbz2': 'bz2', - '.tb2': 'bz2', - } - tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' - tar_files = args.create - - with TarFile.open(tar_name, tar_mode) as tf: - for file_name in tar_files: - tf.add(file_name) - - if args.verbose: - print('{!r} file created.'.format(tar_name)) - -if __name__ == '__main__': - main() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py deleted file mode 100644 index 34e3a9950cc..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Read resources contained within a package.""" - -from ._common import ( - as_file, - files, - Package, -) - -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - -from .abc import ResourceReader - - -__all__ = [ - 'Package', - 'Resource', - 'ResourceReader', - 'as_file', - 'contents', - 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py deleted file mode 100644 index ea363d86a56..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py +++ /dev/null @@ -1,170 +0,0 @@ -from contextlib import suppress -from io import TextIOWrapper - -from . import abc - - -class SpecLoaderAdapter: - """ - Adapt a package spec to adapt the underlying loader. - """ - - def __init__(self, spec, adapter=lambda spec: spec.loader): - self.spec = spec - self.loader = adapter(spec) - - def __getattr__(self, name): - return getattr(self.spec, name) - - -class TraversableResourcesLoader: - """ - Adapt a loader to provide TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - def get_resource_reader(self, name): - return CompatibilityFiles(self.spec)._native() - - -def _io_wrapper(file, mode='r', *args, **kwargs): - if mode == 'r': - return TextIOWrapper(file, *args, **kwargs) - elif mode == 'rb': - return file - raise ValueError( - "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode) - ) - - -class CompatibilityFiles: - """ - Adapter for an existing or non-existent resource reader - to provide a compatibility .files(). - """ - - class SpecPath(abc.Traversable): - """ - Path tied to a module spec. - Can be read and exposes the resource reader children. - """ - - def __init__(self, spec, reader): - self._spec = spec - self._reader = reader - - def iterdir(self): - if not self._reader: - return iter(()) - return iter( - CompatibilityFiles.ChildPath(self._reader, path) - for path in self._reader.contents() - ) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - if not self._reader: - return CompatibilityFiles.OrphanPath(other) - return CompatibilityFiles.ChildPath(self._reader, other) - - @property - def name(self): - return self._spec.name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) - - class ChildPath(abc.Traversable): - """ - Path tied to a resource reader child. - Can be read but doesn't expose any meaningful children. - """ - - def __init__(self, reader, name): - self._reader = reader - self._name = name - - def iterdir(self): - return iter(()) - - def is_file(self): - return self._reader.is_resource(self.name) - - def is_dir(self): - return not self.is_file() - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(self.name, other) - - @property - def name(self): - return self._name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper( - self._reader.open_resource(self.name), mode, *args, **kwargs - ) - - class OrphanPath(abc.Traversable): - """ - Orphan path, not tied to a module spec or resource reader. - Can't be read and doesn't expose any meaningful children. - """ - - def __init__(self, *path_parts): - if len(path_parts) < 1: - raise ValueError('Need at least one path part to construct a path') - self._path = path_parts - - def iterdir(self): - return iter(()) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(*self._path, other) - - @property - def name(self): - return self._path[-1] - - def open(self, mode='r', *args, **kwargs): - raise FileNotFoundError("Can't open orphan path") - - def __init__(self, spec): - self.spec = spec - - @property - def _reader(self): - with suppress(AttributeError): - return self.spec.loader.get_resource_reader(self.spec.name) - - def _native(self): - """ - Return the native reader if it supports files(). - """ - reader = self._reader - return reader if hasattr(reader, 'files') else self - - def __getattr__(self, attr): - return getattr(self._reader, attr) - - def files(self): - return CompatibilityFiles.SpecPath(self.spec, self._reader) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - """ - return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py deleted file mode 100644 index 3c6de1cfb2e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -import pathlib -import tempfile -import functools -import contextlib -import types -import importlib -import inspect -import warnings -import itertools - -from typing import Union, Optional, cast -from .abc import ResourceReader, Traversable - -from ._compat import wrap_spec - -Package = Union[types.ModuleType, str] -Anchor = Package - - -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor -def files(anchor: Optional[Anchor] = None) -> Traversable: - """ - Get a Traversable resource for an anchor. - """ - return from_package(resolve(anchor)) - - -def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: - """ - Return the package's loader if it's a ResourceReader. - """ - # We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore - if reader is None: - return None - return reader(spec.name) # type: ignore - - -def resolve(cand: Optional[Anchor]) -> types.ModuleType: - return cast(types.ModuleType, cand) - - -def _(cand: str) -> types.ModuleType: - return importlib.import_module(cand) - - -def _(cand: None) -> types.ModuleType: - return resolve(_infer_caller().f_globals['__name__']) - - -def _infer_caller(): - """ - Walk the stack and find the frame of the first caller not in this module. - """ - - def is_this_file(frame_info): - return frame_info.filename == __file__ - - def is_wrapper(frame_info): - return frame_info.function == 'wrapper' - - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) - # also exclude 'wrapper' due to singledispatch in the call stack - callers = itertools.filterfalse(is_wrapper, not_this_file) - return next(callers).frame - - -def from_package(package: types.ModuleType): - """ - Return a Traversable object for the given package. - - """ - spec = wrap_spec(package) - reader = spec.loader.get_resource_reader(spec.name) - return reader.files() - - -def _tempfile( - reader, - suffix='', - # gh-93353: Keep a reference to call os.remove() in late Python - # finalization. - *, - _os_remove=os.remove, -): - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on Windows - # properly. - fd, raw_path = tempfile.mkstemp(suffix=suffix) - try: - try: - os.write(fd, reader()) - finally: - os.close(fd) - del reader - yield pathlib.Path(raw_path) - finally: - try: - _os_remove(raw_path) - except FileNotFoundError: - pass - - -def _temp_file(path): - return _tempfile(path.read_bytes, suffix=path.name) - - -def _is_present_dir(path: Traversable) -> bool: - """ - Some Traversables implement ``is_dir()`` to raise an - exception (i.e. ``FileNotFoundError``) when the - directory doesn't exist. This function wraps that call - to always return a boolean and only return True - if there's a dir and it exists. - """ - with contextlib.suppress(FileNotFoundError): - return path.is_dir() - return False - - -def as_file(path): - """ - Given a Traversable object, return that object as a - path on the local file system in a context manager. - """ - return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) - - -@as_file.register(pathlib.Path) -def _(path): - """ - Degenerate behavior for pathlib.Path objects. - """ - yield path - - -def _temp_path(dir: tempfile.TemporaryDirectory): - """ - Wrap tempfile.TemporyDirectory to return a pathlib object. - """ - with dir as result: - yield pathlib.Path(result) - - -def _temp_dir(path): - """ - Given a traversable dir, recursively replicate the whole tree - to the file system in a context manager. - """ - assert path.is_dir() - with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: - yield _write_contents(temp_dir, path) - - -def _write_contents(target, source): - child = target.joinpath(source.name) - if source.is_dir(): - child.mkdir() - for item in source.iterdir(): - _write_contents(child, item) - else: - child.write_bytes(source.read_bytes()) - return child diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py deleted file mode 100644 index 8b5b1d280f3..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py +++ /dev/null @@ -1,108 +0,0 @@ -# flake8: noqa - -import abc -import os -import sys -import pathlib -from contextlib import suppress -from typing import Union - - -if sys.version_info >= (3, 10): - from zipfile import Path as ZipPath # type: ignore -else: - from ..zipp import Path as ZipPath # type: ignore - - -try: - from typing import runtime_checkable # type: ignore -except ImportError: - - def runtime_checkable(cls): # type: ignore - return cls - - -try: - from typing import Protocol # type: ignore -except ImportError: - Protocol = abc.ABC # type: ignore - - -class TraversableResourcesLoader: - """ - Adapt loaders to provide TraversableResources and other - compatibility. - - Used primarily for Python 3.9 and earlier where the native - loaders do not yet implement TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - @property - def path(self): - return self.spec.origin - - def get_resource_reader(self, name): - from . import readers, _adapters - - def _zip_reader(spec): - with suppress(AttributeError): - return readers.ZipReader(spec.loader, spec.name) - - def _namespace_reader(spec): - with suppress(AttributeError, ValueError): - return readers.NamespaceReader(spec.submodule_search_locations) - - def _available_reader(spec): - with suppress(AttributeError): - return spec.loader.get_resource_reader(spec.name) - - def _native_reader(spec): - reader = _available_reader(spec) - return reader if hasattr(reader, 'files') else None - - def _file_reader(spec): - try: - path = pathlib.Path(self.path) - except TypeError: - return None - if path.exists(): - return readers.FileReader(self) - - return ( - # native reader if it supplies 'files' - _native_reader(self.spec) - or - # local ZipReader if a zip module - _zip_reader(self.spec) - or - # local NamespaceReader if a namespace module - _namespace_reader(self.spec) - or - # local FileReader - _file_reader(self.spec) - # fallback - adapt the spec ResourceReader to TraversableReader - or _adapters.CompatibilityFiles(self.spec) - ) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - - Supersedes _adapters.wrap_spec to use TraversableResourcesLoader - from above for older Python compatibility (<3.10). - """ - from . import _adapters - - return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py deleted file mode 100644 index cce05582ffc..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py +++ /dev/null @@ -1,35 +0,0 @@ -from itertools import filterfalse - -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py deleted file mode 100644 index 23b6aeafe4f..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py +++ /dev/null @@ -1,170 +0,0 @@ -import abc -import io -import itertools -import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional - -from ._compat import runtime_checkable, Protocol, StrPath - - -__all__ = ["ResourceReader", "Traversable", "TraversableResources"] - - -class ResourceReader(metaclass=abc.ABCMeta): - """Abstract base class for loaders to provide resource reading support.""" - - @abc.abstractmethod - def open_resource(self, resource: Text) -> BinaryIO: - """Return an opened, file-like object for binary reading. - - The 'resource' argument is expected to represent only a file name. - If the resource cannot be found, FileNotFoundError is raised. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def resource_path(self, resource: Text) -> Text: - """Return the file system path to the specified resource. - - The 'resource' argument is expected to represent only a file name. - If the resource does not exist on the file system, raise - FileNotFoundError. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def is_resource(self, path: Text) -> bool: - """Return True if the named 'path' is a resource. - - Files are resources, directories are not. - """ - raise FileNotFoundError - - @abc.abstractmethod - def contents(self) -> Iterable[str]: - """Return an iterable of entries in `package`.""" - raise FileNotFoundError - - -class TraversalError(Exception): - pass - - -@runtime_checkable -class Traversable(Protocol): - """ - An object with a subset of pathlib.Path methods suitable for - traversing directories and opening files. - - Any exceptions that occur when accessing the backing resource - may propagate unaltered. - """ - - @abc.abstractmethod - def iterdir(self) -> Iterator["Traversable"]: - """ - Yield Traversable objects in self - """ - - def read_bytes(self) -> bytes: - """ - Read contents of self as bytes - """ - with self.open('rb') as strm: - return strm.read() - - def read_text(self, encoding: Optional[str] = None) -> str: - """ - Read contents of self as text - """ - with self.open(encoding=encoding) as strm: - return strm.read() - - @abc.abstractmethod - def is_dir(self) -> bool: - """ - Return True if self is a directory - """ - - @abc.abstractmethod - def is_file(self) -> bool: - """ - Return True if self is a file - """ - - def joinpath(self, *descendants: StrPath) -> "Traversable": - """ - Return Traversable resolved with any descendants applied. - - Each descendant should be a path segment relative to self - and each may contain multiple levels separated by - ``posixpath.sep`` (``/``). - """ - if not descendants: - return self - names = itertools.chain.from_iterable( - path.parts for path in map(pathlib.PurePosixPath, descendants) - ) - target = next(names) - matches = ( - traversable for traversable in self.iterdir() if traversable.name == target - ) - try: - match = next(matches) - except StopIteration: - raise TraversalError( - "Target not found during traversal.", target, list(names) - ) - return match.joinpath(*names) - - def __truediv__(self, child: StrPath) -> "Traversable": - """ - Return Traversable child in self - """ - return self.joinpath(child) - - @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): - """ - mode may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as pathlib.Path.open). - - When opening as text, accepts encoding parameters such as those - accepted by io.TextIOWrapper. - """ - - @property - @abc.abstractmethod - def name(self) -> str: - """ - The base name of this object without any parent references. - """ - - -class TraversableResources(ResourceReader): - """ - The required interface for providing traversable - resources. - """ - - @abc.abstractmethod - def files(self) -> "Traversable": - """Return a Traversable object for the loaded package.""" - - def open_resource(self, resource: StrPath) -> io.BufferedReader: - return self.files().joinpath(resource).open('rb') - - def resource_path(self, resource: Any) -> NoReturn: - raise FileNotFoundError(resource) - - def is_resource(self, path: StrPath) -> bool: - return self.files().joinpath(path).is_file() - - def contents(self) -> Iterator[str]: - return (item.name for item in self.files().iterdir()) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py deleted file mode 100644 index ab34db74091..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py +++ /dev/null @@ -1,120 +0,0 @@ -import collections -import pathlib -import operator - -from . import abc - -from ._itertools import unique_everseen -from ._compat import ZipPath - - -def remove_duplicates(items): - return iter(collections.OrderedDict.fromkeys(items)) - - -class FileReader(abc.TraversableResources): - def __init__(self, loader): - self.path = pathlib.Path(loader.path).parent - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path - - -class ZipReader(abc.TraversableResources): - def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' - self.archive = loader.archive - - def open_resource(self, resource): - try: - return super().open_resource(resource) - except KeyError as exc: - raise FileNotFoundError(exc.args[0]) - - def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. - target = self.files().joinpath(path) - return target.is_file() and target.exists() - - def files(self): - return ZipPath(self.archive, self.prefix) - - -class MultiplexedPath(abc.Traversable): - """ - Given a series of Traversable objects, implement a merged - version of the interface across all objects. Useful for - namespace packages which may be multihomed at a single - name. - """ - - def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) - if not self._paths: - message = 'MultiplexedPath must contain at least one path' - raise FileNotFoundError(message) - if not all(path.is_dir() for path in self._paths): - raise NotADirectoryError('MultiplexedPath only supports directories') - - def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) - - def read_bytes(self): - raise FileNotFoundError(f'{self} is not a file') - - def read_text(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - def is_dir(self): - return True - - def is_file(self): - return False - - def joinpath(self, *descendants): - try: - return super().joinpath(*descendants) - except abc.TraversalError: - # One of the paths did not resolve (a directory does not exist). - # Just return something that will not exist. - return self._paths[0].joinpath(*descendants) - - def open(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - @property - def name(self): - return self._paths[0].name - - def __repr__(self): - paths = ', '.join(f"'{path}'" for path in self._paths) - return f'MultiplexedPath({paths})' - - -class NamespaceReader(abc.TraversableResources): - def __init__(self, namespace_path): - if 'NamespacePath' not in str(namespace_path): - raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py deleted file mode 100644 index 7770c922c84..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Interface adapters for low-level readers. -""" - -import abc -import io -import itertools -from typing import BinaryIO, List - -from .abc import Traversable, TraversableResources - - -class SimpleReader(abc.ABC): - """ - The minimum, low-level interface required from a resource - provider. - """ - - @property - @abc.abstractmethod - def package(self) -> str: - """ - The name of the package for which this reader loads resources. - """ - - @abc.abstractmethod - def children(self) -> List['SimpleReader']: - """ - Obtain an iterable of SimpleReader for available - child containers (e.g. directories). - """ - - @abc.abstractmethod - def resources(self) -> List[str]: - """ - Obtain available named resources for this virtual package. - """ - - @abc.abstractmethod - def open_binary(self, resource: str) -> BinaryIO: - """ - Obtain a File-like for a named resource. - """ - - @property - def name(self): - return self.package.split('.')[-1] - - -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader: SimpleReader): - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - -class ResourceHandle(Traversable): - """ - Handle to a named resource in a ResourceReader. - """ - - def __init__(self, parent: ResourceContainer, name: str): - self.parent = parent - self.name = name # type: ignore - - def is_file(self): - return True - - def is_dir(self): - return False - - def open(self, mode='r', *args, **kwargs): - stream = self.parent.reader.open_binary(self.name) - if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) - return stream - - def joinpath(self, name): - raise RuntimeError("Cannot traverse into a resource") - - -class TraversableReader(TraversableResources, SimpleReader): - """ - A TraversableResources based on SimpleReader. Resource providers - may derive from this class to provide the TraversableResources - interface by supplying the SimpleReader interface. - """ - - def files(self): - return ResourceContainer(self) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py deleted file mode 100644 index c42f6135d55..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import annotations - -import contextlib -import functools -import operator -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.request -import warnings -from typing import Iterator - - -if sys.version_info < (3, 12): - from pkg_resources.extern.backports import tarfile -else: - import tarfile - - -def pushd(dir: str | os.PathLike) -> Iterator[str | os.PathLike]: - """ - >>> tmp_path = getfixture('tmp_path') - >>> with pushd(tmp_path): - ... assert os.getcwd() == os.fspath(tmp_path) - >>> assert os.getcwd() != os.fspath(tmp_path) - """ - - orig = os.getcwd() - os.chdir(dir) - try: - yield dir - finally: - os.chdir(orig) - - -def tarball( - url, target_dir: str | os.PathLike | None = None -) -> Iterator[str | os.PathLike]: - """ - Get a tarball, extract it, yield, then clean up. - - >>> import urllib.request - >>> url = getfixture('tarfile_served') - >>> target = getfixture('tmp_path') / 'out' - >>> tb = tarball(url, target_dir=target) - >>> import pathlib - >>> with tb as extracted: - ... contents = pathlib.Path(extracted, 'contents.txt').read_text(encoding='utf-8') - >>> assert not os.path.exists(extracted) - """ - if target_dir is None: - target_dir = os.path.basename(url).replace('.tar.gz', '').replace('.tgz', '') - # In the tar command, use --strip-components=1 to strip the first path and - # then - # use -C to cause the files to be extracted to {target_dir}. This ensures - # that we always know where the files were extracted. - os.mkdir(target_dir) - try: - req = urllib.request.urlopen(url) - with tarfile.open(fileobj=req, mode='r|*') as tf: - tf.extractall(path=target_dir, filter=strip_first_component) - yield target_dir - finally: - shutil.rmtree(target_dir) - - -def strip_first_component( - member: tarfile.TarInfo, - path, -) -> tarfile.TarInfo: - _, member.name = member.name.split('/', 1) - return member - - -def _compose(*cmgrs): - """ - Compose any number of dependent context managers into a single one. - - The last, innermost context manager may take arbitrary arguments, but - each successive context manager should accept the result from the - previous as a single parameter. - - Like :func:`jaraco.functools.compose`, behavior works from right to - left, so the context manager should be indicated from outermost to - innermost. - - Example, to create a context manager to change to a temporary - directory: - - >>> temp_dir_as_cwd = _compose(pushd, temp_dir) - >>> with temp_dir_as_cwd() as dir: - ... assert os.path.samefile(os.getcwd(), dir) - """ - - def compose_two(inner, outer): - def composed(*args, **kwargs): - with inner(*args, **kwargs) as saved, outer(saved) as res: - yield res - - return contextlib.contextmanager(composed) - - return functools.reduce(compose_two, reversed(cmgrs)) - - -tarball_cwd = _compose(pushd, tarball) - - -def tarball_context(*args, **kwargs): - warnings.warn( - "tarball_context is deprecated. Use tarball or tarball_cwd instead.", - DeprecationWarning, - stacklevel=2, - ) - pushd_ctx = kwargs.pop('pushd', pushd) - with tarball(*args, **kwargs) as tball, pushd_ctx(tball) as dir: - yield dir - - -def infer_compression(url): - """ - Given a URL or filename, infer the compression code for tar. - - >>> infer_compression('http://foo/bar.tar.gz') - 'z' - >>> infer_compression('http://foo/bar.tgz') - 'z' - >>> infer_compression('file.bz') - 'j' - >>> infer_compression('file.xz') - 'J' - """ - warnings.warn( - "infer_compression is deprecated with no replacement", - DeprecationWarning, - stacklevel=2, - ) - # cheat and just assume it's the last two characters - compression_indicator = url[-2:] - mapping = dict(gz='z', bz='j', xz='J') - # Assume 'z' (gzip) if no match - return mapping.get(compression_indicator, 'z') - - -def temp_dir(remover=shutil.rmtree): - """ - Create a temporary directory context. Pass a custom remover - to override the removal behavior. - - >>> import pathlib - >>> with temp_dir() as the_dir: - ... assert os.path.isdir(the_dir) - ... _ = pathlib.Path(the_dir).joinpath('somefile').write_text('contents', encoding='utf-8') - >>> assert not os.path.exists(the_dir) - """ - temp_dir = tempfile.mkdtemp() - try: - yield temp_dir - finally: - remover(temp_dir) - - -def repo_context(url, branch=None, quiet=True, dest_ctx=temp_dir): - """ - Check out the repo indicated by url. - - If dest_ctx is supplied, it should be a context manager - to yield the target directory for the check out. - """ - exe = 'git' if 'git' in url else 'hg' - with dest_ctx() as repo_dir: - cmd = [exe, 'clone', url, repo_dir] - if branch: - cmd.extend(['--branch', branch]) - devnull = open(os.path.devnull, 'w') - stdout = devnull if quiet else None - subprocess.check_call(cmd, stdout=stdout) - yield repo_dir - - -def null(): - """ - A null context suitable to stand in for a meaningful context. - - >>> with null() as value: - ... assert value is None - - This context is most useful when dealing with two or more code - branches but only some need a context. Wrap the others in a null - context to provide symmetry across all options. - """ - warnings.warn( - "null is deprecated. Use contextlib.nullcontext", - DeprecationWarning, - stacklevel=2, - ) - return contextlib.nullcontext() - - -class ExceptionTrap: - """ - A context manager that will catch certain exceptions and provide an - indication they occurred. - - >>> with ExceptionTrap() as trap: - ... raise Exception() - >>> bool(trap) - True - - >>> with ExceptionTrap() as trap: - ... pass - >>> bool(trap) - False - - >>> with ExceptionTrap(ValueError) as trap: - ... raise ValueError("1 + 1 is not 3") - >>> bool(trap) - True - >>> trap.value - ValueError('1 + 1 is not 3') - >>> trap.tb - <traceback object at ...> - - >>> with ExceptionTrap(ValueError) as trap: - ... raise Exception() - Traceback (most recent call last): - ... - Exception - - >>> bool(trap) - False - """ - - exc_info = None, None, None - - def __init__(self, exceptions=(Exception,)): - self.exceptions = exceptions - - def __enter__(self): - return self - - @property - def type(self): - return self.exc_info[0] - - @property - def value(self): - return self.exc_info[1] - - @property - def tb(self): - return self.exc_info[2] - - def __exit__(self, *exc_info): - type = exc_info[0] - matches = type and issubclass(type, self.exceptions) - if matches: - self.exc_info = exc_info - return matches - - def __bool__(self): - return bool(self.type) - - def raises(self, func, *, _test=bool): - """ - Wrap func and replace the result with the truth - value of the trap (True if an exception occurred). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> raises = ExceptionTrap(ValueError).raises - - Now decorate a function that always fails. - - >>> @raises - ... def fail(): - ... raise ValueError('failed') - >>> fail() - True - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - with ExceptionTrap(self.exceptions) as trap: - func(*args, **kwargs) - return _test(trap) - - return wrapper - - def passes(self, func): - """ - Wrap func and replace the result with the truth - value of the trap (True if no exception). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> passes = ExceptionTrap(ValueError).passes - - Now decorate a function that always fails. - - >>> @passes - ... def fail(): - ... raise ValueError('failed') - - >>> fail() - False - """ - return self.raises(func, _test=operator.not_) - - -class suppress(contextlib.suppress, contextlib.ContextDecorator): - """ - A version of contextlib.suppress with decorator support. - - >>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ - - -class on_interrupt(contextlib.ContextDecorator): - """ - Replace a KeyboardInterrupt with SystemExit(1) - - >>> def do_interrupt(): - ... raise KeyboardInterrupt() - >>> on_interrupt('error')(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 1 - >>> on_interrupt('error', code=255)(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 255 - >>> on_interrupt('suppress')(do_interrupt)() - >>> with __import__('pytest').raises(KeyboardInterrupt): - ... on_interrupt('ignore')(do_interrupt)() - """ - - def __init__(self, action='error', /, code=1): - self.action = action - self.code = code - - def __enter__(self): - return self - - def __exit__(self, exctype, excinst, exctb): - if exctype is not KeyboardInterrupt or self.action == 'ignore': - return - elif self.action == 'error': - raise SystemExit(self.code) from excinst - return self.action == 'suppress' diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py deleted file mode 100644 index f523099c723..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py +++ /dev/null @@ -1,633 +0,0 @@ -import collections.abc -import functools -import inspect -import itertools -import operator -import time -import types -import warnings - -import pkg_resources.extern.more_itertools - - -def compose(*funcs): - """ - Compose any number of unary functions into a single unary function. - - >>> import textwrap - >>> expected = str.strip(textwrap.dedent(compose.__doc__)) - >>> strip_and_dedent = compose(str.strip, textwrap.dedent) - >>> strip_and_dedent(compose.__doc__) == expected - True - - Compose also allows the innermost function to take arbitrary arguments. - - >>> round_three = lambda x: round(x, ndigits=3) - >>> f = compose(round_three, int.__truediv__) - >>> [f(3*x, x+1) for x in range(1,10)] - [1.5, 2.0, 2.25, 2.4, 2.5, 2.571, 2.625, 2.667, 2.7] - """ - - def compose_two(f1, f2): - return lambda *args, **kwargs: f1(f2(*args, **kwargs)) - - return functools.reduce(compose_two, funcs) - - -def once(func): - """ - Decorate func so it's only ever called the first time. - - This decorator can ensure that an expensive or non-idempotent function - will not be expensive on subsequent calls and is idempotent. - - >>> add_three = once(lambda a: a+3) - >>> add_three(3) - 6 - >>> add_three(9) - 6 - >>> add_three('12') - 6 - - To reset the stored value, simply clear the property ``saved_result``. - - >>> del add_three.saved_result - >>> add_three(9) - 12 - >>> add_three(8) - 12 - - Or invoke 'reset()' on it. - - >>> add_three.reset() - >>> add_three(-3) - 0 - >>> add_three(0) - 0 - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if not hasattr(wrapper, 'saved_result'): - wrapper.saved_result = func(*args, **kwargs) - return wrapper.saved_result - - wrapper.reset = lambda: vars(wrapper).__delitem__('saved_result') - return wrapper - - -def method_cache(method, cache_wrapper=functools.lru_cache()): - """ - Wrap lru_cache to support storing the cache data in the object instances. - - Abstracts the common paradigm where the method explicitly saves an - underscore-prefixed protected property on first call and returns that - subsequently. - - >>> class MyClass: - ... calls = 0 - ... - ... @method_cache - ... def method(self, value): - ... self.calls += 1 - ... return value - - >>> a = MyClass() - >>> a.method(3) - 3 - >>> for x in range(75): - ... res = a.method(x) - >>> a.calls - 75 - - Note that the apparent behavior will be exactly like that of lru_cache - except that the cache is stored on each instance, so values in one - instance will not flush values from another, and when an instance is - deleted, so are the cached values for that instance. - - >>> b = MyClass() - >>> for x in range(35): - ... res = b.method(x) - >>> b.calls - 35 - >>> a.method(0) - 0 - >>> a.calls - 75 - - Note that if method had been decorated with ``functools.lru_cache()``, - a.calls would have been 76 (due to the cached value of 0 having been - flushed by the 'b' instance). - - Clear the cache with ``.cache_clear()`` - - >>> a.method.cache_clear() - - Same for a method that hasn't yet been called. - - >>> c = MyClass() - >>> c.method.cache_clear() - - Another cache wrapper may be supplied: - - >>> cache = functools.lru_cache(maxsize=2) - >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) - >>> a = MyClass() - >>> a.method2() - 3 - - Caution - do not subsequently wrap the method with another decorator, such - as ``@property``, which changes the semantics of the function. - - See also - http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ - for another implementation and additional justification. - """ - - def wrapper(self, *args, **kwargs): - # it's the first call, replace the method with a cached, bound method - bound_method = types.MethodType(method, self) - cached_method = cache_wrapper(bound_method) - setattr(self, method.__name__, cached_method) - return cached_method(*args, **kwargs) - - # Support cache clear even before cache has been created. - wrapper.cache_clear = lambda: None - - return _special_method_cache(method, cache_wrapper) or wrapper - - -def _special_method_cache(method, cache_wrapper): - """ - Because Python treats special methods differently, it's not - possible to use instance attributes to implement the cached - methods. - - Instead, install the wrapper method under a different name - and return a simple proxy to that wrapper. - - https://github.com/jaraco/jaraco.functools/issues/5 - """ - name = method.__name__ - special_names = '__getattr__', '__getitem__' - - if name not in special_names: - return None - - wrapper_name = '__cached' + name - - def proxy(self, /, *args, **kwargs): - if wrapper_name not in vars(self): - bound = types.MethodType(method, self) - cache = cache_wrapper(bound) - setattr(self, wrapper_name, cache) - else: - cache = getattr(self, wrapper_name) - return cache(*args, **kwargs) - - return proxy - - -def apply(transform): - """ - Decorate a function with a transform function that is - invoked on results returned from the decorated function. - - >>> @apply(reversed) - ... def get_numbers(start): - ... "doc for get_numbers" - ... return range(start, start+3) - >>> list(get_numbers(4)) - [6, 5, 4] - >>> get_numbers.__doc__ - 'doc for get_numbers' - """ - - def wrap(func): - return functools.wraps(func)(compose(transform, func)) - - return wrap - - -def result_invoke(action): - r""" - Decorate a function with an action function that is - invoked on the results returned from the decorated - function (for its side effect), then return the original - result. - - >>> @result_invoke(print) - ... def add_two(a, b): - ... return a + b - >>> x = add_two(2, 3) - 5 - >>> x - 5 - """ - - def wrap(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - action(result) - return result - - return wrapper - - return wrap - - -def invoke(f, /, *args, **kwargs): - """ - Call a function for its side effect after initialization. - - The benefit of using the decorator instead of simply invoking a function - after defining it is that it makes explicit the author's intent for the - function to be called immediately. Whereas if one simply calls the - function immediately, it's less obvious if that was intentional or - incidental. It also avoids repeating the name - the two actions, defining - the function and calling it immediately are modeled separately, but linked - by the decorator construct. - - The benefit of having a function construct (opposed to just invoking some - behavior inline) is to serve as a scope in which the behavior occurs. It - avoids polluting the global namespace with local variables, provides an - anchor on which to attach documentation (docstring), keeps the behavior - logically separated (instead of conceptually separated or not separated at - all), and provides potential to re-use the behavior for testing or other - purposes. - - This function is named as a pithy way to communicate, "call this function - primarily for its side effect", or "while defining this function, also - take it aside and call it". It exists because there's no Python construct - for "define and call" (nor should there be, as decorators serve this need - just fine). The behavior happens immediately and synchronously. - - >>> @invoke - ... def func(): print("called") - called - >>> func() - called - - Use functools.partial to pass parameters to the initial call - - >>> @functools.partial(invoke, name='bingo') - ... def func(name): print('called with', name) - called with bingo - """ - f(*args, **kwargs) - return f - - -class Throttler: - """Rate-limit a function (or other callable).""" - - def __init__(self, func, max_rate=float('Inf')): - if isinstance(func, Throttler): - func = func.func - self.func = func - self.max_rate = max_rate - self.reset() - - def reset(self): - self.last_called = 0 - - def __call__(self, *args, **kwargs): - self._wait() - return self.func(*args, **kwargs) - - def _wait(self): - """Ensure at least 1/max_rate seconds from last call.""" - elapsed = time.time() - self.last_called - must_wait = 1 / self.max_rate - elapsed - time.sleep(max(0, must_wait)) - self.last_called = time.time() - - def __get__(self, obj, owner=None): - return first_invoke(self._wait, functools.partial(self.func, obj)) - - -def first_invoke(func1, func2): - """ - Return a function that when invoked will invoke func1 without - any parameters (for its side effect) and then invoke func2 - with whatever parameters were passed, returning its result. - """ - - def wrapper(*args, **kwargs): - func1() - return func2(*args, **kwargs) - - return wrapper - - -method_caller = first_invoke( - lambda: warnings.warn( - '`jaraco.functools.method_caller` is deprecated, ' - 'use `operator.methodcaller` instead', - DeprecationWarning, - stacklevel=3, - ), - operator.methodcaller, -) - - -def retry_call(func, cleanup=lambda: None, retries=0, trap=()): - """ - Given a callable func, trap the indicated exceptions - for up to 'retries' times, invoking cleanup on the - exception. On the final attempt, allow any exceptions - to propagate. - """ - attempts = itertools.count() if retries == float('inf') else range(retries) - for _ in attempts: - try: - return func() - except trap: - cleanup() - - return func() - - -def retry(*r_args, **r_kwargs): - """ - Decorator wrapper for retry_call. Accepts arguments to retry_call - except func and then returns a decorator for the decorated function. - - Ex: - - >>> @retry(retries=3) - ... def my_func(a, b): - ... "this is my funk" - ... print(a, b) - >>> my_func.__doc__ - 'this is my funk' - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*f_args, **f_kwargs): - bound = functools.partial(func, *f_args, **f_kwargs) - return retry_call(bound, *r_args, **r_kwargs) - - return wrapper - - return decorate - - -def print_yielded(func): - """ - Convert a generator into a function that prints all yielded elements. - - >>> @print_yielded - ... def x(): - ... yield 3; yield None - >>> x() - 3 - None - """ - print_all = functools.partial(map, print) - print_results = compose(more_itertools.consume, print_all, func) - return functools.wraps(func)(print_results) - - -def pass_none(func): - """ - Wrap func so it's not called if its first param is None. - - >>> print_text = pass_none(print) - >>> print_text('text') - text - >>> print_text(None) - """ - - @functools.wraps(func) - def wrapper(param, /, *args, **kwargs): - if param is not None: - return func(param, *args, **kwargs) - return None - - return wrapper - - -def assign_params(func, namespace): - """ - Assign parameters from namespace where func solicits. - - >>> def func(x, y=3): - ... print(x, y) - >>> assigned = assign_params(func, dict(x=2, z=4)) - >>> assigned() - 2 3 - - The usual errors are raised if a function doesn't receive - its required parameters: - - >>> assigned = assign_params(func, dict(y=3, z=4)) - >>> assigned() - Traceback (most recent call last): - TypeError: func() ...argument... - - It even works on methods: - - >>> class Handler: - ... def meth(self, arg): - ... print(arg) - >>> assign_params(Handler().meth, dict(arg='crystal', foo='clear'))() - crystal - """ - sig = inspect.signature(func) - params = sig.parameters.keys() - call_ns = {k: namespace[k] for k in params if k in namespace} - return functools.partial(func, **call_ns) - - -def save_method_args(method): - """ - Wrap a method such that when it is called, the args and kwargs are - saved on the method. - - >>> class MyClass: - ... @save_method_args - ... def method(self, a, b): - ... print(a, b) - >>> my_ob = MyClass() - >>> my_ob.method(1, 2) - 1 2 - >>> my_ob._saved_method.args - (1, 2) - >>> my_ob._saved_method.kwargs - {} - >>> my_ob.method(a=3, b='foo') - 3 foo - >>> my_ob._saved_method.args - () - >>> my_ob._saved_method.kwargs == dict(a=3, b='foo') - True - - The arguments are stored on the instance, allowing for - different instance to save different args. - - >>> your_ob = MyClass() - >>> your_ob.method({str('x'): 3}, b=[4]) - {'x': 3} [4] - >>> your_ob._saved_method.args - ({'x': 3},) - >>> my_ob._saved_method.args - () - """ - args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') - - @functools.wraps(method) - def wrapper(self, /, *args, **kwargs): - attr_name = '_saved_' + method.__name__ - attr = args_and_kwargs(args, kwargs) - setattr(self, attr_name, attr) - return method(self, *args, **kwargs) - - return wrapper - - -def except_(*exceptions, replace=None, use=None): - """ - Replace the indicated exceptions, if raised, with the indicated - literal replacement or evaluated expression (if present). - - >>> safe_int = except_(ValueError)(int) - >>> safe_int('five') - >>> safe_int('5') - 5 - - Specify a literal replacement with ``replace``. - - >>> safe_int_r = except_(ValueError, replace=0)(int) - >>> safe_int_r('five') - 0 - - Provide an expression to ``use`` to pass through particular parameters. - - >>> safe_int_pt = except_(ValueError, use='args[0]')(int) - >>> safe_int_pt('five') - 'five' - - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except exceptions: - try: - return eval(use) - except TypeError: - return replace - - return wrapper - - return decorate - - -def identity(x): - """ - Return the argument. - - >>> o = object() - >>> identity(o) is o - True - """ - return x - - -def bypass_when(check, *, _op=identity): - """ - Decorate a function to return its parameter when ``check``. - - >>> bypassed = [] # False - - >>> @bypass_when(bypassed) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> bypassed[:] = [object()] # True - >>> double(2) - 2 - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(param, /): - return param if _op(check) else func(param) - - return wrapper - - return decorate - - -def bypass_unless(check): - """ - Decorate a function to return its parameter unless ``check``. - - >>> enabled = [object()] # True - - >>> @bypass_unless(enabled) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> del enabled[:] # False - >>> double(2) - 2 - """ - return bypass_when(check, _op=operator.not_) - - -def _splat_inner(args, func): - """Splat args to func.""" - return func(*args) - - -@_splat_inner.register -def _(args: collections.abc.Mapping, func): - """Splat kargs to func as kwargs.""" - return func(**args) - - -def splat(func): - """ - Wrap func to expect its parameters to be passed positionally in a tuple. - - Has a similar effect to that of ``itertools.starmap`` over - simple ``map``. - - >>> pairs = [(-1, 1), (0, 2)] - >>> pkg_resources.extern.more_itertools.consume(itertools.starmap(print, pairs)) - -1 1 - 0 2 - >>> pkg_resources.extern.more_itertools.consume(map(splat(print), pairs)) - -1 1 - 0 2 - - The approach generalizes to other iterators that don't have a "star" - equivalent, such as a "starfilter". - - >>> list(filter(splat(operator.add), pairs)) - [(0, 2)] - - Splat also accepts a mapping argument. - - >>> def is_nice(msg, code): - ... return "smile" in msg or code == 0 - >>> msgs = [ - ... dict(msg='smile!', code=20), - ... dict(msg='error :(', code=1), - ... dict(msg='unknown', code=0), - ... ] - >>> for msg in filter(splat(is_nice), msgs): - ... print(msg) - {'msg': 'smile!', 'code': 20} - {'msg': 'unknown', 'code': 0} - """ - return functools.wraps(func)(functools.partial(_splat_inner, func=func)) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi deleted file mode 100644 index c2b9ab1757e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi +++ /dev/null @@ -1,128 +0,0 @@ -from collections.abc import Callable, Hashable, Iterator -from functools import partial -from operator import methodcaller -import sys -from typing import ( - Any, - Generic, - Protocol, - TypeVar, - overload, -) - -if sys.version_info >= (3, 10): - from typing import Concatenate, ParamSpec -else: - from typing_extensions import Concatenate, ParamSpec - -_P = ParamSpec('_P') -_R = TypeVar('_R') -_T = TypeVar('_T') -_R1 = TypeVar('_R1') -_R2 = TypeVar('_R2') -_V = TypeVar('_V') -_S = TypeVar('_S') -_R_co = TypeVar('_R_co', covariant=True) - -class _OnceCallable(Protocol[_P, _R]): - saved_result: _R - reset: Callable[[], None] - def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... - -class _ProxyMethodCacheWrapper(Protocol[_R_co]): - cache_clear: Callable[[], None] - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -class _MethodCacheWrapper(Protocol[_R_co]): - def cache_clear(self) -> None: ... - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -# `compose()` overloads below will cover most use cases. - -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[_P, _R], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R1], _R], - __func3: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R2], _R], - __func3: Callable[[_R1], _R2], - __func4: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -def once(func: Callable[_P, _R]) -> _OnceCallable[_P, _R]: ... -def method_cache( - method: Callable[..., _R], - cache_wrapper: Callable[[Callable[..., _R]], _MethodCacheWrapper[_R]] = ..., -) -> _MethodCacheWrapper[_R] | _ProxyMethodCacheWrapper[_R]: ... -def apply( - transform: Callable[[_R], _T] -) -> Callable[[Callable[_P, _R]], Callable[_P, _T]]: ... -def result_invoke( - action: Callable[[_R], Any] -) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: ... -def invoke( - f: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... -def call_aside( - f: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... - -class Throttler(Generic[_R]): - last_called: float - func: Callable[..., _R] - max_rate: float - def __init__( - self, func: Callable[..., _R] | Throttler[_R], max_rate: float = ... - ) -> None: ... - def reset(self) -> None: ... - def __call__(self, *args: Any, **kwargs: Any) -> _R: ... - def __get__(self, obj: Any, owner: type[Any] | None = ...) -> Callable[..., _R]: ... - -def first_invoke( - func1: Callable[..., Any], func2: Callable[_P, _R] -) -> Callable[_P, _R]: ... - -method_caller: Callable[..., methodcaller] - -def retry_call( - func: Callable[..., _R], - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> _R: ... -def retry( - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> Callable[[Callable[..., _R]], Callable[..., _R]]: ... -def print_yielded(func: Callable[_P, Iterator[Any]]) -> Callable[_P, None]: ... -def pass_none( - func: Callable[Concatenate[_T, _P], _R] -) -> Callable[Concatenate[_T, _P], _R]: ... -def assign_params( - func: Callable[..., _R], namespace: dict[str, Any] -) -> partial[_R]: ... -def save_method_args( - method: Callable[Concatenate[_S, _P], _R] -) -> Callable[Concatenate[_S, _P], _R]: ... -def except_( - *exceptions: type[BaseException], replace: Any = ..., use: Any = ... -) -> Callable[[Callable[_P, Any]], Callable[_P, Any]]: ... -def identity(x: _T) -> _T: ... -def bypass_when( - check: _V, *, _op: Callable[[_V], Any] = ... -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... -def bypass_unless( - check: Any, -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/py.typed b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/text/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/text/__init__.py deleted file mode 100644 index c466378ceba..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/text/__init__.py +++ /dev/null @@ -1,599 +0,0 @@ -import re -import itertools -import textwrap -import functools - -try: - from importlib.resources import files # type: ignore -except ImportError: # pragma: nocover - from pkg_resources.extern.importlib_resources import files # type: ignore - -from pkg_resources.extern.jaraco.functools import compose, method_cache -from pkg_resources.extern.jaraco.context import ExceptionTrap - - -def substitution(old, new): - """ - Return a function that will perform a substitution on a string - """ - return lambda s: s.replace(old, new) - - -def multi_substitution(*substitutions): - """ - Take a sequence of pairs specifying substitutions, and create - a function that performs those substitutions. - - >>> multi_substitution(('foo', 'bar'), ('bar', 'baz'))('foo') - 'baz' - """ - substitutions = itertools.starmap(substitution, substitutions) - # compose function applies last function first, so reverse the - # substitutions to get the expected order. - substitutions = reversed(tuple(substitutions)) - return compose(*substitutions) - - -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use ``in_``: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super().lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super().lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) - - -# Python 3.8 compatibility -_unicode_trap = ExceptionTrap(UnicodeDecodeError) - - -@_unicode_trap.passes -def is_decodable(value): - r""" - Return True if the supplied value is decodable (using the default - encoding). - - >>> is_decodable(b'\xff') - False - >>> is_decodable(b'\x32') - True - """ - value.decode() - - -def is_binary(value): - r""" - Return True if the value appears to be binary (that is, it's a byte - string and isn't decodable). - - >>> is_binary(b'\xff') - True - >>> is_binary('\xff') - False - """ - return isinstance(value, bytes) and not is_decodable(value) - - -def trim(s): - r""" - Trim something like a docstring to remove the whitespace that - is common due to indentation and formatting. - - >>> trim("\n\tfoo = bar\n\t\tbar = baz\n") - 'foo = bar\n\tbar = baz' - """ - return textwrap.dedent(s).strip() - - -def wrap(s): - """ - Wrap lines of text, retaining existing newlines as - paragraph markers. - - >>> print(wrap(lorem_ipsum)) - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do - eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad - minim veniam, quis nostrud exercitation ullamco laboris nisi ut - aliquip ex ea commodo consequat. Duis aute irure dolor in - reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla - pariatur. Excepteur sint occaecat cupidatat non proident, sunt in - culpa qui officia deserunt mollit anim id est laborum. - <BLANKLINE> - Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam - varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus - magna felis sollicitudin mauris. Integer in mauris eu nibh euismod - gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis - risus a elit. Etiam tempor. Ut ullamcorper, ligula eu tempor congue, - eros est euismod turpis, id tincidunt sapien risus a quam. Maecenas - fermentum consequat mi. Donec fermentum. Pellentesque malesuada nulla - a mi. Duis sapien sem, aliquet nec, commodo eget, consequat quis, - neque. Aliquam faucibus, elit ut dictum aliquet, felis nisl adipiscing - sapien, sed malesuada diam lacus eget erat. Cras mollis scelerisque - nunc. Nullam arcu. Aliquam consequat. Curabitur augue lorem, dapibus - quis, laoreet et, pretium ac, nisi. Aenean magna nisl, mollis quis, - molestie eu, feugiat in, orci. In hac habitasse platea dictumst. - """ - paragraphs = s.splitlines() - wrapped = ('\n'.join(textwrap.wrap(para)) for para in paragraphs) - return '\n\n'.join(wrapped) - - -def unwrap(s): - r""" - Given a multi-line string, return an unwrapped version. - - >>> wrapped = wrap(lorem_ipsum) - >>> wrapped.count('\n') - 20 - >>> unwrapped = unwrap(wrapped) - >>> unwrapped.count('\n') - 1 - >>> print(unwrapped) - Lorem ipsum dolor sit amet, consectetur adipiscing ... - Curabitur pretium tincidunt lacus. Nulla gravida orci ... - - """ - paragraphs = re.split(r'\n\n+', s) - cleaned = (para.replace('\n', ' ') for para in paragraphs) - return '\n'.join(cleaned) - - - - -class Splitter(object): - """object that will split a string with the given arguments for each call - - >>> s = Splitter(',') - >>> s('hello, world, this is your, master calling') - ['hello', ' world', ' this is your', ' master calling'] - """ - - def __init__(self, *args): - self.args = args - - def __call__(self, s): - return s.split(*self.args) - - -def indent(string, prefix=' ' * 4): - """ - >>> indent('foo') - ' foo' - """ - return prefix + string - - -class WordSet(tuple): - """ - Given an identifier, return the words that identifier represents, - whether in camel case, underscore-separated, etc. - - >>> WordSet.parse("camelCase") - ('camel', 'Case') - - >>> WordSet.parse("under_sep") - ('under', 'sep') - - Acronyms should be retained - - >>> WordSet.parse("firstSNL") - ('first', 'SNL') - - >>> WordSet.parse("you_and_I") - ('you', 'and', 'I') - - >>> WordSet.parse("A simple test") - ('A', 'simple', 'test') - - Multiple caps should not interfere with the first cap of another word. - - >>> WordSet.parse("myABCClass") - ('my', 'ABC', 'Class') - - The result is a WordSet, so you can get the form you need. - - >>> WordSet.parse("myABCClass").underscore_separated() - 'my_ABC_Class' - - >>> WordSet.parse('a-command').camel_case() - 'ACommand' - - >>> WordSet.parse('someIdentifier').lowered().space_separated() - 'some identifier' - - Slices of the result should return another WordSet. - - >>> WordSet.parse('taken-out-of-context')[1:].underscore_separated() - 'out_of_context' - - >>> WordSet.from_class_name(WordSet()).lowered().space_separated() - 'word set' - - >>> example = WordSet.parse('figured it out') - >>> example.headless_camel_case() - 'figuredItOut' - >>> example.dash_separated() - 'figured-it-out' - - """ - - _pattern = re.compile('([A-Z]?[a-z]+)|([A-Z]+(?![a-z]))') - - def capitalized(self): - return WordSet(word.capitalize() for word in self) - - def lowered(self): - return WordSet(word.lower() for word in self) - - def camel_case(self): - return ''.join(self.capitalized()) - - def headless_camel_case(self): - words = iter(self) - first = next(words).lower() - new_words = itertools.chain((first,), WordSet(words).camel_case()) - return ''.join(new_words) - - def underscore_separated(self): - return '_'.join(self) - - def dash_separated(self): - return '-'.join(self) - - def space_separated(self): - return ' '.join(self) - - def trim_right(self, item): - """ - Remove the item from the end of the set. - - >>> WordSet.parse('foo bar').trim_right('foo') - ('foo', 'bar') - >>> WordSet.parse('foo bar').trim_right('bar') - ('foo',) - >>> WordSet.parse('').trim_right('bar') - () - """ - return self[:-1] if self and self[-1] == item else self - - def trim_left(self, item): - """ - Remove the item from the beginning of the set. - - >>> WordSet.parse('foo bar').trim_left('foo') - ('bar',) - >>> WordSet.parse('foo bar').trim_left('bar') - ('foo', 'bar') - >>> WordSet.parse('').trim_left('bar') - () - """ - return self[1:] if self and self[0] == item else self - - def trim(self, item): - """ - >>> WordSet.parse('foo bar').trim('foo') - ('bar',) - """ - return self.trim_left(item).trim_right(item) - - def __getitem__(self, item): - result = super(WordSet, self).__getitem__(item) - if isinstance(item, slice): - result = WordSet(result) - return result - - @classmethod - def parse(cls, identifier): - matches = cls._pattern.finditer(identifier) - return WordSet(match.group(0) for match in matches) - - @classmethod - def from_class_name(cls, subject): - return cls.parse(subject.__class__.__name__) - - -# for backward compatibility -words = WordSet.parse - - -def simple_html_strip(s): - r""" - Remove HTML from the string `s`. - - >>> str(simple_html_strip('')) - '' - - >>> print(simple_html_strip('A <bold>stormy</bold> day in paradise')) - A stormy day in paradise - - >>> print(simple_html_strip('Somebody <!-- do not --> tell the truth.')) - Somebody tell the truth. - - >>> print(simple_html_strip('What about<br/>\nmultiple lines?')) - What about - multiple lines? - """ - html_stripper = re.compile('(<!--.*?-->)|(<[^>]*>)|([^<]+)', re.DOTALL) - texts = (match.group(3) or '' for match in html_stripper.finditer(s)) - return ''.join(texts) - - -class SeparatedValues(str): - """ - A string separated by a separator. Overrides __iter__ for getting - the values. - - >>> list(SeparatedValues('a,b,c')) - ['a', 'b', 'c'] - - Whitespace is stripped and empty values are discarded. - - >>> list(SeparatedValues(' a, b , c, ')) - ['a', 'b', 'c'] - """ - - separator = ',' - - def __iter__(self): - parts = self.split(self.separator) - return filter(None, (part.strip() for part in parts)) - - -class Stripper: - r""" - Given a series of lines, find the common prefix and strip it from them. - - >>> lines = [ - ... 'abcdefg\n', - ... 'abc\n', - ... 'abcde\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix - 'abc' - >>> list(res.lines) - ['defg\n', '\n', 'de\n'] - - If no prefix is common, nothing should be stripped. - - >>> lines = [ - ... 'abcd\n', - ... '1234\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix = '' - >>> list(res.lines) - ['abcd\n', '1234\n'] - """ - - def __init__(self, prefix, lines): - self.prefix = prefix - self.lines = map(self, lines) - - @classmethod - def strip_prefix(cls, lines): - prefix_lines, lines = itertools.tee(lines) - prefix = functools.reduce(cls.common_prefix, prefix_lines) - return cls(prefix, lines) - - def __call__(self, line): - if not self.prefix: - return line - null, prefix, rest = line.partition(self.prefix) - return rest - - @staticmethod - def common_prefix(s1, s2): - """ - Return the common prefix of two lines. - """ - index = min(len(s1), len(s2)) - while s1[:index] != s2[:index]: - index -= 1 - return s1[:index] - - -def remove_prefix(text, prefix): - """ - Remove the prefix from the text if it exists. - - >>> remove_prefix('underwhelming performance', 'underwhelming ') - 'performance' - - >>> remove_prefix('something special', 'sample') - 'something special' - """ - null, prefix, rest = text.rpartition(prefix) - return rest - - -def remove_suffix(text, suffix): - """ - Remove the suffix from the text if it exists. - - >>> remove_suffix('name.git', '.git') - 'name' - - >>> remove_suffix('something special', 'sample') - 'something special' - """ - rest, suffix, null = text.partition(suffix) - return rest - - -def normalize_newlines(text): - r""" - Replace alternate newlines with the canonical newline. - - >>> normalize_newlines('Lorem Ipsum\u2029') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\r\n') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\x85') - 'Lorem Ipsum\n' - """ - newlines = ['\r\n', '\r', '\n', '\u0085', '\u2028', '\u2029'] - pattern = '|'.join(newlines) - return re.sub(pattern, '\n', text) - - -def _nonblank(str): - return str and not str.startswith('#') - - -def yield_lines(iterable): - r""" - Yield valid lines of a string or iterable. - - >>> list(yield_lines('')) - [] - >>> list(yield_lines(['foo', 'bar'])) - ['foo', 'bar'] - >>> list(yield_lines('foo\nbar')) - ['foo', 'bar'] - >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) - ['foo', 'baz #comment'] - >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) - ['foo', 'bar', 'baz', 'bing'] - """ - return itertools.chain.from_iterable(map(yield_lines, iterable)) - - -@yield_lines.register(str) -def _(text): - return filter(_nonblank, map(str.strip, text.splitlines())) - - -def drop_comment(line): - """ - Drop comments. - - >>> drop_comment('foo # bar') - 'foo' - - A hash without a space may be in a URL. - - >>> drop_comment('http://example.com/foo#bar') - 'http://example.com/foo#bar' - """ - return line.partition(' #')[0] - - -def join_continuation(lines): - r""" - Join lines continued by a trailing backslash. - - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar \\', 'baz'])) - ['foobarbaz'] - - Not sure why, but... - The character preceeding the backslash is also elided. - - >>> list(join_continuation(['goo\\', 'dly'])) - ['godly'] - - A terrible idea, but... - If no line is available to continue, suppress the lines. - - >>> list(join_continuation(['foo', 'bar\\', 'baz\\'])) - ['foo'] - """ - lines = iter(lines) - for item in lines: - while item.endswith('\\'): - try: - item = item[:-2].strip() + next(lines) - except StopIteration: - return - yield item diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py deleted file mode 100644 index aff94a9abd0..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""More routines for operating on iterables, beyond itertools""" - -from .more import * # noqa -from .recipes import * # noqa - -__version__ = '10.2.0' diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi deleted file mode 100644 index 96f6e36c7f4..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi +++ /dev/null @@ -1,2 +0,0 @@ -from .more import * -from .recipes import * diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py deleted file mode 100644 index d0957681f54..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py +++ /dev/null @@ -1,4655 +0,0 @@ -import warnings - -from collections import Counter, defaultdict, deque, abc -from collections.abc import Sequence -from functools import cached_property, partial, reduce, wraps -from heapq import heapify, heapreplace, heappop -from itertools import ( - chain, - compress, - count, - cycle, - dropwhile, - groupby, - islice, - repeat, - starmap, - takewhile, - tee, - zip_longest, - product, -) -from math import exp, factorial, floor, log, perm, comb -from queue import Empty, Queue -from random import random, randrange, uniform -from operator import itemgetter, mul, sub, gt, lt, ge, le -from sys import hexversion, maxsize -from time import monotonic - -from .recipes import ( - _marker, - _zip_equal, - UnequalIterablesError, - consume, - flatten, - pairwise, - powerset, - take, - unique_everseen, - all_equal, - batched, -) - -__all__ = [ - 'AbortThread', - 'SequenceView', - 'UnequalIterablesError', - 'adjacent', - 'all_unique', - 'always_iterable', - 'always_reversible', - 'bucket', - 'callback_iter', - 'chunked', - 'chunked_even', - 'circular_shifts', - 'collapse', - 'combination_index', - 'combination_with_replacement_index', - 'consecutive_groups', - 'constrained_batches', - 'consumer', - 'count_cycle', - 'countable', - 'difference', - 'distinct_combinations', - 'distinct_permutations', - 'distribute', - 'divide', - 'duplicates_everseen', - 'duplicates_justseen', - 'classify_unique', - 'exactly_n', - 'filter_except', - 'filter_map', - 'first', - 'gray_product', - 'groupby_transform', - 'ichunked', - 'iequals', - 'ilen', - 'interleave', - 'interleave_evenly', - 'interleave_longest', - 'intersperse', - 'is_sorted', - 'islice_extended', - 'iterate', - 'iter_suppress', - 'last', - 'locate', - 'longest_common_prefix', - 'lstrip', - 'make_decorator', - 'map_except', - 'map_if', - 'map_reduce', - 'mark_ends', - 'minmax', - 'nth_or_last', - 'nth_permutation', - 'nth_product', - 'nth_combination_with_replacement', - 'numeric_range', - 'one', - 'only', - 'outer_product', - 'padded', - 'partial_product', - 'partitions', - 'peekable', - 'permutation_index', - 'product_index', - 'raise_', - 'repeat_each', - 'repeat_last', - 'replace', - 'rlocate', - 'rstrip', - 'run_length', - 'sample', - 'seekable', - 'set_partitions', - 'side_effect', - 'sliced', - 'sort_together', - 'split_after', - 'split_at', - 'split_before', - 'split_into', - 'split_when', - 'spy', - 'stagger', - 'strip', - 'strictly_n', - 'substrings', - 'substrings_indexes', - 'takewhile_inclusive', - 'time_limited', - 'unique_in_window', - 'unique_to_each', - 'unzip', - 'value_chain', - 'windowed', - 'windowed_complete', - 'with_iter', - 'zip_broadcast', - 'zip_equal', - 'zip_offset', -] - - -def chunked(iterable, n, strict=False): - """Break *iterable* into lists of length *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) - [[1, 2, 3], [4, 5, 6]] - - By the default, the last yielded list will have fewer than *n* elements - if the length of *iterable* is not divisible by *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) - [[1, 2, 3], [4, 5, 6], [7, 8]] - - To use a fill-in value instead, see the :func:`grouper` recipe. - - If the length of *iterable* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - list is yielded. - - """ - iterator = iter(partial(take, n, iter(iterable)), []) - if strict: - if n is None: - raise ValueError('n must not be None when using strict mode.') - - def ret(): - for chunk in iterator: - if len(chunk) != n: - raise ValueError('iterable is not divisible by n.') - yield chunk - - return iter(ret()) - else: - return iterator - - -def first(iterable, default=_marker): - """Return the first item of *iterable*, or *default* if *iterable* is - empty. - - >>> first([0, 1, 2, 3]) - 0 - >>> first([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - - :func:`first` is useful when you have a generator of expensive-to-retrieve - values and want any arbitrary one. It is marginally shorter than - ``next(iter(iterable), default)``. - - """ - for item in iterable: - return item - if default is _marker: - raise ValueError( - 'first() was called on an empty iterable, and no ' - 'default value was provided.' - ) - return default - - -def last(iterable, default=_marker): - """Return the last item of *iterable*, or *default* if *iterable* is - empty. - - >>> last([0, 1, 2, 3]) - 3 - >>> last([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - try: - if isinstance(iterable, Sequence): - return iterable[-1] - # Work around https://bugs.python.org/issue38525 - elif hasattr(iterable, '__reversed__') and (hexversion != 0x030800F0): - return next(reversed(iterable)) - else: - return deque(iterable, maxlen=1)[-1] - except (IndexError, TypeError, StopIteration): - if default is _marker: - raise ValueError( - 'last() was called on an empty iterable, and no default was ' - 'provided.' - ) - return default - - -def nth_or_last(iterable, n, default=_marker): - """Return the nth or the last item of *iterable*, - or *default* if *iterable* is empty. - - >>> nth_or_last([0, 1, 2, 3], 2) - 2 - >>> nth_or_last([0, 1], 2) - 1 - >>> nth_or_last([], 0, 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - return last(islice(iterable, n + 1), default=default) - - -class peekable: - """Wrap an iterator to allow lookahead and prepending elements. - - Call :meth:`peek` on the result to get the value that will be returned - by :func:`next`. This won't advance the iterator: - - >>> p = peekable(['a', 'b']) - >>> p.peek() - 'a' - >>> next(p) - 'a' - - Pass :meth:`peek` a default value to return that instead of raising - ``StopIteration`` when the iterator is exhausted. - - >>> p = peekable([]) - >>> p.peek('hi') - 'hi' - - peekables also offer a :meth:`prepend` method, which "inserts" items - at the head of the iterable: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> p.peek() - 11 - >>> list(p) - [11, 12, 1, 2, 3] - - peekables can be indexed. Index 0 is the item that will be returned by - :func:`next`, index 1 is the item after that, and so on: - The values up to the given index will be cached. - - >>> p = peekable(['a', 'b', 'c', 'd']) - >>> p[0] - 'a' - >>> p[1] - 'b' - >>> next(p) - 'a' - - Negative indexes are supported, but be aware that they will cache the - remaining items in the source iterator, which may require significant - storage. - - To check whether a peekable is exhausted, check its truth value: - - >>> p = peekable(['a', 'b']) - >>> if p: # peekable has items - ... list(p) - ['a', 'b'] - >>> if not p: # peekable is exhausted - ... list(p) - [] - - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self._cache = deque() - - def __iter__(self): - return self - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - """Return the item that will be next returned from ``next()``. - - Return ``default`` if there are no items left. If ``default`` is not - provided, raise ``StopIteration``. - - """ - if not self._cache: - try: - self._cache.append(next(self._it)) - except StopIteration: - if default is _marker: - raise - return default - return self._cache[0] - - def prepend(self, *items): - """Stack up items to be the next ones returned from ``next()`` or - ``self.peek()``. The items will be returned in - first in, first out order:: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> list(p) - [11, 12, 1, 2, 3] - - It is possible, by prepending items, to "resurrect" a peekable that - previously raised ``StopIteration``. - - >>> p = peekable([]) - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - >>> p.prepend(1) - >>> next(p) - 1 - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - - """ - self._cache.extendleft(reversed(items)) - - def __next__(self): - if self._cache: - return self._cache.popleft() - - return next(self._it) - - def _get_slice(self, index): - # Normalize the slice's arguments - step = 1 if (index.step is None) else index.step - if step > 0: - start = 0 if (index.start is None) else index.start - stop = maxsize if (index.stop is None) else index.stop - elif step < 0: - start = -1 if (index.start is None) else index.start - stop = (-maxsize - 1) if (index.stop is None) else index.stop - else: - raise ValueError('slice step cannot be zero') - - # If either the start or stop index is negative, we'll need to cache - # the rest of the iterable in order to slice from the right side. - if (start < 0) or (stop < 0): - self._cache.extend(self._it) - # Otherwise we'll need to find the rightmost index and cache to that - # point. - else: - n = min(max(start, stop) + 1, maxsize) - cache_len = len(self._cache) - if n >= cache_len: - self._cache.extend(islice(self._it, n - cache_len)) - - return list(self._cache)[index] - - def __getitem__(self, index): - if isinstance(index, slice): - return self._get_slice(index) - - cache_len = len(self._cache) - if index < 0: - self._cache.extend(self._it) - elif index >= cache_len: - self._cache.extend(islice(self._it, index + 1 - cache_len)) - - return self._cache[index] - - -def consumer(func): - """Decorator that automatically advances a PEP-342-style "reverse iterator" - to its first yield point so you don't have to call ``next()`` on it - manually. - - >>> @consumer - ... def tally(): - ... i = 0 - ... while True: - ... print('Thing number %s is %s.' % (i, (yield))) - ... i += 1 - ... - >>> t = tally() - >>> t.send('red') - Thing number 0 is red. - >>> t.send('fish') - Thing number 1 is fish. - - Without the decorator, you would have to call ``next(t)`` before - ``t.send()`` could be used. - - """ - - @wraps(func) - def wrapper(*args, **kwargs): - gen = func(*args, **kwargs) - next(gen) - return gen - - return wrapper - - -def ilen(iterable): - """Return the number of items in *iterable*. - - >>> ilen(x for x in range(1000000) if x % 3 == 0) - 333334 - - This consumes the iterable, so handle with care. - - """ - # This approach was selected because benchmarks showed it's likely the - # fastest of the known implementations at the time of writing. - # See GitHub tracker: #236, #230. - counter = count() - deque(zip(iterable, counter), maxlen=0) - return next(counter) - - -def iterate(func, start): - """Return ``start``, ``func(start)``, ``func(func(start))``, ... - - >>> from itertools import islice - >>> list(islice(iterate(lambda x: 2*x, 1), 10)) - [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - - """ - while True: - yield start - try: - start = func(start) - except StopIteration: - break - - -def with_iter(context_manager): - """Wrap an iterable in a ``with`` statement, so it closes once exhausted. - - For example, this will close the file when the iterator is exhausted:: - - upper_lines = (line.upper() for line in with_iter(open('foo'))) - - Any context manager which returns an iterable is a candidate for - ``with_iter``. - - """ - with context_manager as iterable: - yield from iterable - - -def one(iterable, too_short=None, too_long=None): - """Return the first item from *iterable*, which is expected to contain only - that item. Raise an exception if *iterable* is empty or has more than one - item. - - :func:`one` is useful for ensuring that an iterable contains only one item. - For example, it can be used to retrieve the result of a database query - that is expected to return a single row. - - If *iterable* is empty, ``ValueError`` will be raised. You may specify a - different exception with the *too_short* keyword: - - >>> it = [] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (expected 1)' - >>> too_short = IndexError('too few items') - >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - IndexError: too few items - - Similarly, if *iterable* contains more than one item, ``ValueError`` will - be raised. You may specify a different exception with the *too_long* - keyword: - - >>> it = ['too', 'many'] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 'too', - 'many', and perhaps more. - >>> too_long = RuntimeError - >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - Note that :func:`one` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check iterable - contents less destructively. - - """ - it = iter(iterable) - - try: - first_value = next(it) - except StopIteration as e: - raise ( - too_short or ValueError('too few items in iterable (expected 1)') - ) from e - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def raise_(exception, *args): - raise exception(*args) - - -def strictly_n(iterable, n, too_short=None, too_long=None): - """Validate that *iterable* has exactly *n* items and return them if - it does. If it has fewer than *n* items, call function *too_short* - with those items. If it has more than *n* items, call function - *too_long* with the first ``n + 1`` items. - - >>> iterable = ['a', 'b', 'c', 'd'] - >>> n = 4 - >>> list(strictly_n(iterable, n)) - ['a', 'b', 'c', 'd'] - - Note that the returned iterable must be consumed in order for the check to - be made. - - By default, *too_short* and *too_long* are functions that raise - ``ValueError``. - - >>> list(strictly_n('ab', 3)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too few items in iterable (got 2) - - >>> list(strictly_n('abc', 2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (got at least 3) - - You can instead supply functions that do something else. - *too_short* will be called with the number of items in *iterable*. - *too_long* will be called with `n + 1`. - - >>> def too_short(item_count): - ... raise RuntimeError - >>> it = strictly_n('abcd', 6, too_short=too_short) - >>> list(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - >>> def too_long(item_count): - ... print('The boss is going to hear about this') - >>> it = strictly_n('abcdef', 4, too_long=too_long) - >>> list(it) - The boss is going to hear about this - ['a', 'b', 'c', 'd'] - - """ - if too_short is None: - too_short = lambda item_count: raise_( - ValueError, - 'Too few items in iterable (got {})'.format(item_count), - ) - - if too_long is None: - too_long = lambda item_count: raise_( - ValueError, - 'Too many items in iterable (got at least {})'.format(item_count), - ) - - it = iter(iterable) - for i in range(n): - try: - item = next(it) - except StopIteration: - too_short(i) - return - else: - yield item - - try: - next(it) - except StopIteration: - pass - else: - too_long(n + 1) - - -def distinct_permutations(iterable, r=None): - """Yield successive distinct permutations of the elements in *iterable*. - - >>> sorted(distinct_permutations([1, 0, 1])) - [(0, 1, 1), (1, 0, 1), (1, 1, 0)] - - Equivalent to ``set(permutations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - Duplicate permutations arise when there are duplicated elements in the - input iterable. The number of items returned is - `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of - items input, and each `x_i` is the count of a distinct item in the input - sequence. - - If *r* is given, only the *r*-length permutations are yielded. - - >>> sorted(distinct_permutations([1, 0, 1], r=2)) - [(0, 1), (1, 0), (1, 1)] - >>> sorted(distinct_permutations(range(3), r=2)) - [(0, 1), (0, 2), (1, 0), (1, 2), (2, 0), (2, 1)] - - """ - - # Algorithm: https://w.wiki/Qai - def _full(A): - while True: - # Yield the permutation we have - yield tuple(A) - - # Find the largest index i such that A[i] < A[i + 1] - for i in range(size - 2, -1, -1): - if A[i] < A[i + 1]: - break - # If no such index exists, this permutation is the last one - else: - return - - # Find the largest index j greater than j such that A[i] < A[j] - for j in range(size - 1, i, -1): - if A[i] < A[j]: - break - - # Swap the value of A[i] with that of A[j], then reverse the - # sequence from A[i + 1] to form the new permutation - A[i], A[j] = A[j], A[i] - A[i + 1 :] = A[: i - size : -1] # A[i + 1:][::-1] - - # Algorithm: modified from the above - def _partial(A, r): - # Split A into the first r items and the last r items - head, tail = A[:r], A[r:] - right_head_indexes = range(r - 1, -1, -1) - left_tail_indexes = range(len(tail)) - - while True: - # Yield the permutation we have - yield tuple(head) - - # Starting from the right, find the first index of the head with - # value smaller than the maximum value of the tail - call it i. - pivot = tail[-1] - for i in right_head_indexes: - if head[i] < pivot: - break - pivot = head[i] - else: - return - - # Starting from the left, find the first value of the tail - # with a value greater than head[i] and swap. - for j in left_tail_indexes: - if tail[j] > head[i]: - head[i], tail[j] = tail[j], head[i] - break - # If we didn't find one, start from the right and find the first - # index of the head with a value greater than head[i] and swap. - else: - for j in right_head_indexes: - if head[j] > head[i]: - head[i], head[j] = head[j], head[i] - break - - # Reverse head[i + 1:] and swap it with tail[:r - (i + 1)] - tail += head[: i - r : -1] # head[i + 1:][::-1] - i += 1 - head[i:], tail[:] = tail[: r - i], tail[r - i :] - - items = sorted(iterable) - - size = len(items) - if r is None: - r = size - - if 0 < r <= size: - return _full(items) if (r == size) else _partial(items, r) - - return iter(() if r else ((),)) - - -def intersperse(e, iterable, n=1): - """Intersperse filler element *e* among the items in *iterable*, leaving - *n* items between each filler element. - - >>> list(intersperse('!', [1, 2, 3, 4, 5])) - [1, '!', 2, '!', 3, '!', 4, '!', 5] - - >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) - [1, 2, None, 3, 4, None, 5] - - """ - if n == 0: - raise ValueError('n must be > 0') - elif n == 1: - # interleave(repeat(e), iterable) -> e, x_0, e, x_1, e, x_2... - # islice(..., 1, None) -> x_0, e, x_1, e, x_2... - return islice(interleave(repeat(e), iterable), 1, None) - else: - # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... - # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... - # flatten(...) -> x_0, x_1, e, x_2, x_3... - filler = repeat([e]) - chunks = chunked(iterable, n) - return flatten(islice(interleave(filler, chunks), 1, None)) - - -def unique_to_each(*iterables): - """Return the elements from each of the input iterables that aren't in the - other input iterables. - - For example, suppose you have a set of packages, each with a set of - dependencies:: - - {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} - - If you remove one package, which dependencies can also be removed? - - If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not - associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for - ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: - - >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) - [['A'], ['C'], ['D']] - - If there are duplicates in one input iterable that aren't in the others - they will be duplicated in the output. Input order is preserved:: - - >>> unique_to_each("mississippi", "missouri") - [['p', 'p'], ['o', 'u', 'r']] - - It is assumed that the elements of each iterable are hashable. - - """ - pool = [list(it) for it in iterables] - counts = Counter(chain.from_iterable(map(set, pool))) - uniques = {element for element in counts if counts[element] == 1} - return [list(filter(uniques.__contains__, it)) for it in pool] - - -def windowed(seq, n, fillvalue=None, step=1): - """Return a sliding window of width *n* over the given iterable. - - >>> all_windows = windowed([1, 2, 3, 4, 5], 3) - >>> list(all_windows) - [(1, 2, 3), (2, 3, 4), (3, 4, 5)] - - When the window is larger than the iterable, *fillvalue* is used in place - of missing values: - - >>> list(windowed([1, 2, 3], 4)) - [(1, 2, 3, None)] - - Each window will advance in increments of *step*: - - >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) - [(1, 2, 3), (3, 4, 5), (5, 6, '!')] - - To slide into the iterable's items, use :func:`chain` to add filler items - to the left: - - >>> iterable = [1, 2, 3, 4] - >>> n = 3 - >>> padding = [None] * (n - 1) - >>> list(windowed(chain(padding, iterable), 3)) - [(None, None, 1), (None, 1, 2), (1, 2, 3), (2, 3, 4)] - """ - if n < 0: - raise ValueError('n must be >= 0') - if n == 0: - yield tuple() - return - if step < 1: - raise ValueError('step must be >= 1') - - window = deque(maxlen=n) - i = n - for _ in map(window.append, seq): - i -= 1 - if not i: - i = step - yield tuple(window) - - size = len(window) - if size == 0: - return - elif size < n: - yield tuple(chain(window, repeat(fillvalue, n - size))) - elif 0 < i < min(step, n): - window += (fillvalue,) * i - yield tuple(window) - - -def substrings(iterable): - """Yield all of the substrings of *iterable*. - - >>> [''.join(s) for s in substrings('more')] - ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] - - Note that non-string iterables can also be subdivided. - - >>> list(substrings([0, 1, 2])) - [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] - - """ - # The length-1 substrings - seq = [] - for item in iter(iterable): - seq.append(item) - yield (item,) - seq = tuple(seq) - item_count = len(seq) - - # And the rest - for n in range(2, item_count + 1): - for i in range(item_count - n + 1): - yield seq[i : i + n] - - -def substrings_indexes(seq, reverse=False): - """Yield all substrings and their positions in *seq* - - The items yielded will be a tuple of the form ``(substr, i, j)``, where - ``substr == seq[i:j]``. - - This function only works for iterables that support slicing, such as - ``str`` objects. - - >>> for item in substrings_indexes('more'): - ... print(item) - ('m', 0, 1) - ('o', 1, 2) - ('r', 2, 3) - ('e', 3, 4) - ('mo', 0, 2) - ('or', 1, 3) - ('re', 2, 4) - ('mor', 0, 3) - ('ore', 1, 4) - ('more', 0, 4) - - Set *reverse* to ``True`` to yield the same items in the opposite order. - - - """ - r = range(1, len(seq) + 1) - if reverse: - r = reversed(r) - return ( - (seq[i : i + L], i, i + L) for L in r for i in range(len(seq) - L + 1) - ) - - -class bucket: - """Wrap *iterable* and return an object that buckets the iterable into - child iterables based on a *key* function. - - >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] - >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character - >>> sorted(list(s)) # Get the keys - ['a', 'b', 'c'] - >>> a_iterable = s['a'] - >>> next(a_iterable) - 'a1' - >>> next(a_iterable) - 'a2' - >>> list(s['b']) - ['b1', 'b2', 'b3'] - - The original iterable will be advanced and its items will be cached until - they are used by the child iterables. This may require significant storage. - - By default, attempting to select a bucket to which no items belong will - exhaust the iterable and cache all values. - If you specify a *validator* function, selected buckets will instead be - checked against it. - - >>> from itertools import count - >>> it = count(1, 2) # Infinite sequence of odd numbers - >>> key = lambda x: x % 10 # Bucket by last digit - >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only - >>> s = bucket(it, key=key, validator=validator) - >>> 2 in s - False - >>> list(s[2]) - [] - - """ - - def __init__(self, iterable, key, validator=None): - self._it = iter(iterable) - self._key = key - self._cache = defaultdict(deque) - self._validator = validator or (lambda x: True) - - def __contains__(self, value): - if not self._validator(value): - return False - - try: - item = next(self[value]) - except StopIteration: - return False - else: - self._cache[value].appendleft(item) - - return True - - def _get_values(self, value): - """ - Helper to yield items from the parent iterator that match *value*. - Items that don't match are stored in the local cache as they - are encountered. - """ - while True: - # If we've cached some items that match the target value, emit - # the first one and evict it from the cache. - if self._cache[value]: - yield self._cache[value].popleft() - # Otherwise we need to advance the parent iterator to search for - # a matching item, caching the rest. - else: - while True: - try: - item = next(self._it) - except StopIteration: - return - item_value = self._key(item) - if item_value == value: - yield item - break - elif self._validator(item_value): - self._cache[item_value].append(item) - - def __iter__(self): - for item in self._it: - item_value = self._key(item) - if self._validator(item_value): - self._cache[item_value].append(item) - - yield from self._cache.keys() - - def __getitem__(self, value): - if not self._validator(value): - return iter(()) - - return self._get_values(value) - - -def spy(iterable, n=1): - """Return a 2-tuple with a list containing the first *n* elements of - *iterable*, and an iterator with the same items as *iterable*. - This allows you to "look ahead" at the items in the iterable without - advancing it. - - There is one item in the list by default: - - >>> iterable = 'abcdefg' - >>> head, iterable = spy(iterable) - >>> head - ['a'] - >>> list(iterable) - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - - You may use unpacking to retrieve items instead of lists: - - >>> (head,), iterable = spy('abcdefg') - >>> head - 'a' - >>> (first, second), iterable = spy('abcdefg', 2) - >>> first - 'a' - >>> second - 'b' - - The number of items requested can be larger than the number of items in - the iterable: - - >>> iterable = [1, 2, 3, 4, 5] - >>> head, iterable = spy(iterable, 10) - >>> head - [1, 2, 3, 4, 5] - >>> list(iterable) - [1, 2, 3, 4, 5] - - """ - it = iter(iterable) - head = take(n, it) - - return head.copy(), chain(head, it) - - -def interleave(*iterables): - """Return a new iterable yielding from each iterable in turn, - until the shortest is exhausted. - - >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7] - - For a version that doesn't terminate after the shortest iterable is - exhausted, see :func:`interleave_longest`. - - """ - return chain.from_iterable(zip(*iterables)) - - -def interleave_longest(*iterables): - """Return a new iterable yielding from each iterable in turn, - skipping any that are exhausted. - - >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7, 3, 8] - - This function produces the same output as :func:`roundrobin`, but may - perform better for some inputs (in particular when the number of iterables - is large). - - """ - i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) - return (x for x in i if x is not _marker) - - -def interleave_evenly(iterables, lengths=None): - """ - Interleave multiple iterables so that their elements are evenly distributed - throughout the output sequence. - - >>> iterables = [1, 2, 3, 4, 5], ['a', 'b'] - >>> list(interleave_evenly(iterables)) - [1, 2, 'a', 3, 4, 'b', 5] - - >>> iterables = [[1, 2, 3], [4, 5], [6, 7, 8]] - >>> list(interleave_evenly(iterables)) - [1, 6, 4, 2, 7, 3, 8, 5] - - This function requires iterables of known length. Iterables without - ``__len__()`` can be used by manually specifying lengths with *lengths*: - - >>> from itertools import combinations, repeat - >>> iterables = [combinations(range(4), 2), ['a', 'b', 'c']] - >>> lengths = [4 * (4 - 1) // 2, 3] - >>> list(interleave_evenly(iterables, lengths=lengths)) - [(0, 1), (0, 2), 'a', (0, 3), (1, 2), 'b', (1, 3), (2, 3), 'c'] - - Based on Bresenham's algorithm. - """ - if lengths is None: - try: - lengths = [len(it) for it in iterables] - except TypeError: - raise ValueError( - 'Iterable lengths could not be determined automatically. ' - 'Specify them with the lengths keyword.' - ) - elif len(iterables) != len(lengths): - raise ValueError('Mismatching number of iterables and lengths.') - - dims = len(lengths) - - # sort iterables by length, descending - lengths_permute = sorted( - range(dims), key=lambda i: lengths[i], reverse=True - ) - lengths_desc = [lengths[i] for i in lengths_permute] - iters_desc = [iter(iterables[i]) for i in lengths_permute] - - # the longest iterable is the primary one (Bresenham: the longest - # distance along an axis) - delta_primary, deltas_secondary = lengths_desc[0], lengths_desc[1:] - iter_primary, iters_secondary = iters_desc[0], iters_desc[1:] - errors = [delta_primary // dims] * len(deltas_secondary) - - to_yield = sum(lengths) - while to_yield: - yield next(iter_primary) - to_yield -= 1 - # update errors for each secondary iterable - errors = [e - delta for e, delta in zip(errors, deltas_secondary)] - - # those iterables for which the error is negative are yielded - # ("diagonal step" in Bresenham) - for i, e in enumerate(errors): - if e < 0: - yield next(iters_secondary[i]) - to_yield -= 1 - errors[i] += delta_primary - - -def collapse(iterable, base_type=None, levels=None): - """Flatten an iterable with multiple levels of nesting (e.g., a list of - lists of tuples) into non-iterable types. - - >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] - >>> list(collapse(iterable)) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and - will not be collapsed. - - To avoid collapsing other types, specify *base_type*: - - >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] - >>> list(collapse(iterable, base_type=tuple)) - ['ab', ('cd', 'ef'), 'gh', 'ij'] - - Specify *levels* to stop flattening after a certain level: - - >>> iterable = [('a', ['b']), ('c', ['d'])] - >>> list(collapse(iterable)) # Fully flattened - ['a', 'b', 'c', 'd'] - >>> list(collapse(iterable, levels=1)) # Only one level flattened - ['a', ['b'], 'c', ['d']] - - """ - - def walk(node, level): - if ( - ((levels is not None) and (level > levels)) - or isinstance(node, (str, bytes)) - or ((base_type is not None) and isinstance(node, base_type)) - ): - yield node - return - - try: - tree = iter(node) - except TypeError: - yield node - return - else: - for child in tree: - yield from walk(child, level + 1) - - yield from walk(iterable, 0) - - -def side_effect(func, iterable, chunk_size=None, before=None, after=None): - """Invoke *func* on each item in *iterable* (or on each *chunk_size* group - of items) before yielding the item. - - `func` must be a function that takes a single argument. Its return value - will be discarded. - - *before* and *after* are optional functions that take no arguments. They - will be executed before iteration starts and after it ends, respectively. - - `side_effect` can be used for logging, updating progress bars, or anything - that is not functionally "pure." - - Emitting a status message: - - >>> from more_itertools import consume - >>> func = lambda item: print('Received {}'.format(item)) - >>> consume(side_effect(func, range(2))) - Received 0 - Received 1 - - Operating on chunks of items: - - >>> pair_sums = [] - >>> func = lambda chunk: pair_sums.append(sum(chunk)) - >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) - [0, 1, 2, 3, 4, 5] - >>> list(pair_sums) - [1, 5, 9] - - Writing to a file-like object: - - >>> from io import StringIO - >>> from more_itertools import consume - >>> f = StringIO() - >>> func = lambda x: print(x, file=f) - >>> before = lambda: print(u'HEADER', file=f) - >>> after = f.close - >>> it = [u'a', u'b', u'c'] - >>> consume(side_effect(func, it, before=before, after=after)) - >>> f.closed - True - - """ - try: - if before is not None: - before() - - if chunk_size is None: - for item in iterable: - func(item) - yield item - else: - for chunk in chunked(iterable, chunk_size): - func(chunk) - yield from chunk - finally: - if after is not None: - after() - - -def sliced(seq, n, strict=False): - """Yield slices of length *n* from the sequence *seq*. - - >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) - [(1, 2, 3), (4, 5, 6)] - - By the default, the last yielded slice will have fewer than *n* elements - if the length of *seq* is not divisible by *n*: - - >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) - [(1, 2, 3), (4, 5, 6), (7, 8)] - - If the length of *seq* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - slice is yielded. - - This function will only work for iterables that support slicing. - For non-sliceable iterables, see :func:`chunked`. - - """ - iterator = takewhile(len, (seq[i : i + n] for i in count(0, n))) - if strict: - - def ret(): - for _slice in iterator: - if len(_slice) != n: - raise ValueError("seq is not divisible by n.") - yield _slice - - return iter(ret()) - else: - return iterator - - -def split_at(iterable, pred, maxsplit=-1, keep_separator=False): - """Yield lists of items from *iterable*, where each list is delimited by - an item where callable *pred* returns ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b')) - [['a'], ['c', 'd', 'c'], ['a']] - - >>> list(split_at(range(10), lambda n: n % 2 == 1)) - [[0], [2], [4], [6], [8], []] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_at(range(10), lambda n: n % 2 == 1, maxsplit=2)) - [[0], [2], [4, 5, 6, 7, 8, 9]] - - By default, the delimiting items are not included in the output. - To include them, set *keep_separator* to ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b', keep_separator=True)) - [['a'], ['b'], ['c', 'd', 'c'], ['b'], ['a']] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item): - yield buf - if keep_separator: - yield [item] - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - else: - buf.append(item) - yield buf - - -def split_before(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends just before - an item for which callable *pred* returns ``True``: - - >>> list(split_before('OneTwo', lambda s: s.isupper())) - [['O', 'n', 'e'], ['T', 'w', 'o']] - - >>> list(split_before(range(10), lambda n: n % 3 == 0)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_before(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]] - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield [item] + list(it) - return - buf = [] - maxsplit -= 1 - buf.append(item) - if buf: - yield buf - - -def split_after(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends with an - item where callable *pred* returns ``True``: - - >>> list(split_after('one1two2', lambda s: s.isdigit())) - [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] - - >>> list(split_after(range(10), lambda n: n % 3 == 0)) - [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_after(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0], [1, 2, 3], [4, 5, 6, 7, 8, 9]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - buf.append(item) - if pred(item) and buf: - yield buf - if maxsplit == 1: - buf = list(it) - if buf: - yield buf - return - buf = [] - maxsplit -= 1 - if buf: - yield buf - - -def split_when(iterable, pred, maxsplit=-1): - """Split *iterable* into pieces based on the output of *pred*. - *pred* should be a function that takes successive pairs of items and - returns ``True`` if the iterable should be split in between them. - - For example, to find runs of increasing numbers, split the iterable when - element ``i`` is larger than element ``i + 1``: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], lambda x, y: x > y)) - [[1, 2, 3, 3], [2, 5], [2, 4], [2]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], - ... lambda x, y: x > y, maxsplit=2)) - [[1, 2, 3, 3], [2, 5], [2, 4, 2]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - it = iter(iterable) - try: - cur_item = next(it) - except StopIteration: - return - - buf = [cur_item] - for next_item in it: - if pred(cur_item, next_item): - yield buf - if maxsplit == 1: - yield [next_item] + list(it) - return - buf = [] - maxsplit -= 1 - - buf.append(next_item) - cur_item = next_item - - yield buf - - -def split_into(iterable, sizes): - """Yield a list of sequential items from *iterable* of length 'n' for each - integer 'n' in *sizes*. - - >>> list(split_into([1,2,3,4,5,6], [1,2,3])) - [[1], [2, 3], [4, 5, 6]] - - If the sum of *sizes* is smaller than the length of *iterable*, then the - remaining items of *iterable* will not be returned. - - >>> list(split_into([1,2,3,4,5,6], [2,3])) - [[1, 2], [3, 4, 5]] - - If the sum of *sizes* is larger than the length of *iterable*, fewer items - will be returned in the iteration that overruns *iterable* and further - lists will be empty: - - >>> list(split_into([1,2,3,4], [1,2,3,4])) - [[1], [2, 3], [4], []] - - When a ``None`` object is encountered in *sizes*, the returned list will - contain items up to the end of *iterable* the same way that itertools.slice - does: - - >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) - [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] - - :func:`split_into` can be useful for grouping a series of items where the - sizes of the groups are not uniform. An example would be where in a row - from a table, multiple columns represent elements of the same feature - (e.g. a point represented by x,y,z) but, the format is not the same for - all columns. - """ - # convert the iterable argument into an iterator so its contents can - # be consumed by islice in case it is a generator - it = iter(iterable) - - for size in sizes: - if size is None: - yield list(it) - return - else: - yield list(islice(it, size)) - - -def padded(iterable, fillvalue=None, n=None, next_multiple=False): - """Yield the elements from *iterable*, followed by *fillvalue*, such that - at least *n* items are emitted. - - >>> list(padded([1, 2, 3], '?', 5)) - [1, 2, 3, '?', '?'] - - If *next_multiple* is ``True``, *fillvalue* will be emitted until the - number of items emitted is a multiple of *n*:: - - >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) - [1, 2, 3, 4, None, None] - - If *n* is ``None``, *fillvalue* will be emitted indefinitely. - - """ - it = iter(iterable) - if n is None: - yield from chain(it, repeat(fillvalue)) - elif n < 1: - raise ValueError('n must be at least 1') - else: - item_count = 0 - for item in it: - yield item - item_count += 1 - - remaining = (n - item_count) % n if next_multiple else n - item_count - for _ in range(remaining): - yield fillvalue - - -def repeat_each(iterable, n=2): - """Repeat each element in *iterable* *n* times. - - >>> list(repeat_each('ABC', 3)) - ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C', 'C'] - """ - return chain.from_iterable(map(repeat, iterable, repeat(n))) - - -def repeat_last(iterable, default=None): - """After the *iterable* is exhausted, keep yielding its last element. - - >>> list(islice(repeat_last(range(3)), 5)) - [0, 1, 2, 2, 2] - - If the iterable is empty, yield *default* forever:: - - >>> list(islice(repeat_last(range(0), 42), 5)) - [42, 42, 42, 42, 42] - - """ - item = _marker - for item in iterable: - yield item - final = default if item is _marker else item - yield from repeat(final) - - -def distribute(n, iterable): - """Distribute the items from *iterable* among *n* smaller iterables. - - >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 3, 5] - >>> list(group_2) - [2, 4, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 4, 7], [2, 5], [3, 6]] - - If the length of *iterable* is smaller than *n*, then the last returned - iterables will be empty: - - >>> children = distribute(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function uses :func:`itertools.tee` and may require significant - storage. If you need the order items in the smaller iterables to match the - original iterable, see :func:`divide`. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - children = tee(iterable, n) - return [islice(it, index, None, n) for index, it in enumerate(children)] - - -def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): - """Yield tuples whose elements are offset from *iterable*. - The amount by which the `i`-th item in each tuple is offset is given by - the `i`-th item in *offsets*. - - >>> list(stagger([0, 1, 2, 3])) - [(None, 0, 1), (0, 1, 2), (1, 2, 3)] - >>> list(stagger(range(8), offsets=(0, 2, 4))) - [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] - - By default, the sequence will end when the final element of a tuple is the - last item in the iterable. To continue until the first element of a tuple - is the last item in the iterable, set *longest* to ``True``:: - - >>> list(stagger([0, 1, 2, 3], longest=True)) - [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - children = tee(iterable, len(offsets)) - - return zip_offset( - *children, offsets=offsets, longest=longest, fillvalue=fillvalue - ) - - -def zip_equal(*iterables): - """``zip`` the input *iterables* together, but raise - ``UnequalIterablesError`` if they aren't all the same length. - - >>> it_1 = range(3) - >>> it_2 = iter('abc') - >>> list(zip_equal(it_1, it_2)) - [(0, 'a'), (1, 'b'), (2, 'c')] - - >>> it_1 = range(3) - >>> it_2 = iter('abcd') - >>> list(zip_equal(it_1, it_2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - more_itertools.more.UnequalIterablesError: Iterables have different - lengths - - """ - if hexversion >= 0x30A00A6: - warnings.warn( - ( - 'zip_equal will be removed in a future version of ' - 'more-itertools. Use the builtin zip function with ' - 'strict=True instead.' - ), - DeprecationWarning, - ) - - return _zip_equal(*iterables) - - -def zip_offset(*iterables, offsets, longest=False, fillvalue=None): - """``zip`` the input *iterables* together, but offset the `i`-th iterable - by the `i`-th item in *offsets*. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] - - This can be used as a lightweight alternative to SciPy or pandas to analyze - data sets in which some series have a lead or lag relationship. - - By default, the sequence will end when the shortest iterable is exhausted. - To continue until the longest iterable is exhausted, set *longest* to - ``True``. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - if len(iterables) != len(offsets): - raise ValueError("Number of iterables and offsets didn't match") - - staggered = [] - for it, n in zip(iterables, offsets): - if n < 0: - staggered.append(chain(repeat(fillvalue, -n), it)) - elif n > 0: - staggered.append(islice(it, n, None)) - else: - staggered.append(it) - - if longest: - return zip_longest(*staggered, fillvalue=fillvalue) - - return zip(*staggered) - - -def sort_together(iterables, key_list=(0,), key=None, reverse=False): - """Return the input iterables sorted together, with *key_list* as the - priority for sorting. All iterables are trimmed to the length of the - shortest one. - - This can be used like the sorting function in a spreadsheet. If each - iterable represents a column of data, the key list determines which - columns are used for sorting. - - By default, all iterables are sorted using the ``0``-th iterable:: - - >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] - >>> sort_together(iterables) - [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] - - Set a different key list to sort according to another iterable. - Specifying multiple keys dictates how ties are broken:: - - >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] - >>> sort_together(iterables, key_list=(1, 2)) - [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] - - To sort by a function of the elements of the iterable, pass a *key* - function. Its arguments are the elements of the iterables corresponding to - the key list:: - - >>> names = ('a', 'b', 'c') - >>> lengths = (1, 2, 3) - >>> widths = (5, 2, 1) - >>> def area(length, width): - ... return length * width - >>> sort_together([names, lengths, widths], key_list=(1, 2), key=area) - [('c', 'b', 'a'), (3, 2, 1), (1, 2, 5)] - - Set *reverse* to ``True`` to sort in descending order. - - >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) - [(3, 2, 1), ('a', 'b', 'c')] - - """ - if key is None: - # if there is no key function, the key argument to sorted is an - # itemgetter - key_argument = itemgetter(*key_list) - else: - # if there is a key function, call it with the items at the offsets - # specified by the key function as arguments - key_list = list(key_list) - if len(key_list) == 1: - # if key_list contains a single item, pass the item at that offset - # as the only argument to the key function - key_offset = key_list[0] - key_argument = lambda zipped_items: key(zipped_items[key_offset]) - else: - # if key_list contains multiple items, use itemgetter to return a - # tuple of items, which we pass as *args to the key function - get_key_items = itemgetter(*key_list) - key_argument = lambda zipped_items: key( - *get_key_items(zipped_items) - ) - - return list( - zip(*sorted(zip(*iterables), key=key_argument, reverse=reverse)) - ) - - -def unzip(iterable): - """The inverse of :func:`zip`, this function disaggregates the elements - of the zipped *iterable*. - - The ``i``-th iterable contains the ``i``-th element from each element - of the zipped iterable. The first element is used to determine the - length of the remaining elements. - - >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> letters, numbers = unzip(iterable) - >>> list(letters) - ['a', 'b', 'c', 'd'] - >>> list(numbers) - [1, 2, 3, 4] - - This is similar to using ``zip(*iterable)``, but it avoids reading - *iterable* into memory. Note, however, that this function uses - :func:`itertools.tee` and thus may require significant storage. - - """ - head, iterable = spy(iter(iterable)) - if not head: - # empty iterable, e.g. zip([], [], []) - return () - # spy returns a one-length iterable as head - head = head[0] - iterables = tee(iterable, len(head)) - - def itemgetter(i): - def getter(obj): - try: - return obj[i] - except IndexError: - # basically if we have an iterable like - # iter([(1, 2, 3), (4, 5), (6,)]) - # the second unzipped iterable would fail at the third tuple - # since it would try to access tup[1] - # same with the third unzipped iterable and the second tuple - # to support these "improperly zipped" iterables, - # we create a custom itemgetter - # which just stops the unzipped iterables - # at first length mismatch - raise StopIteration - - return getter - - return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) - - -def divide(n, iterable): - """Divide the elements from *iterable* into *n* parts, maintaining - order. - - >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 2, 3] - >>> list(group_2) - [4, 5, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 2, 3], [4, 5], [6, 7]] - - If the length of the iterable is smaller than n, then the last returned - iterables will be empty: - - >>> children = divide(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function will exhaust the iterable before returning and may require - significant storage. If order is not important, see :func:`distribute`, - which does not first pull the iterable into memory. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - try: - iterable[:0] - except TypeError: - seq = tuple(iterable) - else: - seq = iterable - - q, r = divmod(len(seq), n) - - ret = [] - stop = 0 - for i in range(1, n + 1): - start = stop - stop += q + 1 if i <= r else q - ret.append(iter(seq[start:stop])) - - return ret - - -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) - - -def adjacent(predicate, iterable, distance=1): - """Return an iterable over `(bool, item)` tuples where the `item` is - drawn from *iterable* and the `bool` indicates whether - that item satisfies the *predicate* or is adjacent to an item that does. - - For example, to find whether items are adjacent to a ``3``:: - - >>> list(adjacent(lambda x: x == 3, range(6))) - [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] - - Set *distance* to change what counts as adjacent. For example, to find - whether items are two places away from a ``3``: - - >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) - [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] - - This is useful for contextualizing the results of a search function. - For example, a code comparison tool might want to identify lines that - have changed, but also surrounding lines to give the viewer of the diff - context. - - The predicate function will only be called once for each item in the - iterable. - - See also :func:`groupby_transform`, which can be used with this function - to group ranges of items with the same `bool` value. - - """ - # Allow distance=0 mainly for testing that it reproduces results with map() - if distance < 0: - raise ValueError('distance must be at least 0') - - i1, i2 = tee(iterable) - padding = [False] * distance - selected = chain(padding, map(predicate, i1), padding) - adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) - return zip(adjacent_to_selected, i2) - - -def groupby_transform(iterable, keyfunc=None, valuefunc=None, reducefunc=None): - """An extension of :func:`itertools.groupby` that can apply transformations - to the grouped data. - - * *keyfunc* is a function computing a key value for each item in *iterable* - * *valuefunc* is a function that transforms the individual items from - *iterable* after grouping - * *reducefunc* is a function that transforms each group of items - - >>> iterable = 'aAAbBBcCC' - >>> keyfunc = lambda k: k.upper() - >>> valuefunc = lambda v: v.lower() - >>> reducefunc = lambda g: ''.join(g) - >>> list(groupby_transform(iterable, keyfunc, valuefunc, reducefunc)) - [('A', 'aaa'), ('B', 'bbb'), ('C', 'ccc')] - - Each optional argument defaults to an identity function if not specified. - - :func:`groupby_transform` is useful when grouping elements of an iterable - using a separate iterable as the key. To do this, :func:`zip` the iterables - and pass a *keyfunc* that extracts the first element and a *valuefunc* - that extracts the second element:: - - >>> from operator import itemgetter - >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] - >>> values = 'abcdefghi' - >>> iterable = zip(keys, values) - >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) - >>> [(k, ''.join(g)) for k, g in grouper] - [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] - - Note that the order of items in the iterable is significant. - Only adjacent items are grouped together, so if you don't want any - duplicate groups, you should sort the iterable by the key function. - - """ - ret = groupby(iterable, keyfunc) - if valuefunc: - ret = ((k, map(valuefunc, g)) for k, g in ret) - if reducefunc: - ret = ((k, reducefunc(g)) for k, g in ret) - - return ret - - -class numeric_range(abc.Sequence, abc.Hashable): - """An extension of the built-in ``range()`` function whose arguments can - be any orderable numeric type. - - With only *stop* specified, *start* defaults to ``0`` and *step* - defaults to ``1``. The output items will match the type of *stop*: - - >>> list(numeric_range(3.5)) - [0.0, 1.0, 2.0, 3.0] - - With only *start* and *stop* specified, *step* defaults to ``1``. The - output items will match the type of *start*: - - >>> from decimal import Decimal - >>> start = Decimal('2.1') - >>> stop = Decimal('5.1') - >>> list(numeric_range(start, stop)) - [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] - - With *start*, *stop*, and *step* specified the output items will match - the type of ``start + step``: - - >>> from fractions import Fraction - >>> start = Fraction(1, 2) # Start at 1/2 - >>> stop = Fraction(5, 2) # End at 5/2 - >>> step = Fraction(1, 2) # Count by 1/2 - >>> list(numeric_range(start, stop, step)) - [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] - - If *step* is zero, ``ValueError`` is raised. Negative steps are supported: - - >>> list(numeric_range(3, -1, -1.0)) - [3.0, 2.0, 1.0, 0.0] - - Be aware of the limitations of floating point numbers; the representation - of the yielded numbers may be surprising. - - ``datetime.datetime`` objects can be used for *start* and *stop*, if *step* - is a ``datetime.timedelta`` object: - - >>> import datetime - >>> start = datetime.datetime(2019, 1, 1) - >>> stop = datetime.datetime(2019, 1, 3) - >>> step = datetime.timedelta(days=1) - >>> items = iter(numeric_range(start, stop, step)) - >>> next(items) - datetime.datetime(2019, 1, 1, 0, 0) - >>> next(items) - datetime.datetime(2019, 1, 2, 0, 0) - - """ - - _EMPTY_HASH = hash(range(0, 0)) - - def __init__(self, *args): - argc = len(args) - if argc == 1: - (self._stop,) = args - self._start = type(self._stop)(0) - self._step = type(self._stop - self._start)(1) - elif argc == 2: - self._start, self._stop = args - self._step = type(self._stop - self._start)(1) - elif argc == 3: - self._start, self._stop, self._step = args - elif argc == 0: - raise TypeError( - 'numeric_range expected at least ' - '1 argument, got {}'.format(argc) - ) - else: - raise TypeError( - 'numeric_range expected at most ' - '3 arguments, got {}'.format(argc) - ) - - self._zero = type(self._step)(0) - if self._step == self._zero: - raise ValueError('numeric_range() arg 3 must not be zero') - self._growing = self._step > self._zero - - def __bool__(self): - if self._growing: - return self._start < self._stop - else: - return self._start > self._stop - - def __contains__(self, elem): - if self._growing: - if self._start <= elem < self._stop: - return (elem - self._start) % self._step == self._zero - else: - if self._start >= elem > self._stop: - return (self._start - elem) % (-self._step) == self._zero - - return False - - def __eq__(self, other): - if isinstance(other, numeric_range): - empty_self = not bool(self) - empty_other = not bool(other) - if empty_self or empty_other: - return empty_self and empty_other # True if both empty - else: - return ( - self._start == other._start - and self._step == other._step - and self._get_by_index(-1) == other._get_by_index(-1) - ) - else: - return False - - def __getitem__(self, key): - if isinstance(key, int): - return self._get_by_index(key) - elif isinstance(key, slice): - step = self._step if key.step is None else key.step * self._step - - if key.start is None or key.start <= -self._len: - start = self._start - elif key.start >= self._len: - start = self._stop - else: # -self._len < key.start < self._len - start = self._get_by_index(key.start) - - if key.stop is None or key.stop >= self._len: - stop = self._stop - elif key.stop <= -self._len: - stop = self._start - else: # -self._len < key.stop < self._len - stop = self._get_by_index(key.stop) - - return numeric_range(start, stop, step) - else: - raise TypeError( - 'numeric range indices must be ' - 'integers or slices, not {}'.format(type(key).__name__) - ) - - def __hash__(self): - if self: - return hash((self._start, self._get_by_index(-1), self._step)) - else: - return self._EMPTY_HASH - - def __iter__(self): - values = (self._start + (n * self._step) for n in count()) - if self._growing: - return takewhile(partial(gt, self._stop), values) - else: - return takewhile(partial(lt, self._stop), values) - - def __len__(self): - return self._len - - @cached_property - def _len(self): - if self._growing: - start = self._start - stop = self._stop - step = self._step - else: - start = self._stop - stop = self._start - step = -self._step - distance = stop - start - if distance <= self._zero: - return 0 - else: # distance > 0 and step > 0: regular euclidean division - q, r = divmod(distance, step) - return int(q) + int(r != self._zero) - - def __reduce__(self): - return numeric_range, (self._start, self._stop, self._step) - - def __repr__(self): - if self._step == 1: - return "numeric_range({}, {})".format( - repr(self._start), repr(self._stop) - ) - else: - return "numeric_range({}, {}, {})".format( - repr(self._start), repr(self._stop), repr(self._step) - ) - - def __reversed__(self): - return iter( - numeric_range( - self._get_by_index(-1), self._start - self._step, -self._step - ) - ) - - def count(self, value): - return int(value in self) - - def index(self, value): - if self._growing: - if self._start <= value < self._stop: - q, r = divmod(value - self._start, self._step) - if r == self._zero: - return int(q) - else: - if self._start >= value > self._stop: - q, r = divmod(self._start - value, -self._step) - if r == self._zero: - return int(q) - - raise ValueError("{} is not in numeric range".format(value)) - - def _get_by_index(self, i): - if i < 0: - i += self._len - if i < 0 or i >= self._len: - raise IndexError("numeric range object index out of range") - return self._start + i * self._step - - -def count_cycle(iterable, n=None): - """Cycle through the items from *iterable* up to *n* times, yielding - the number of completed cycles along with each item. If *n* is omitted the - process repeats indefinitely. - - >>> list(count_cycle('AB', 3)) - [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] - - """ - iterable = tuple(iterable) - if not iterable: - return iter(()) - counter = count() if n is None else range(n) - return ((i, item) for i in counter for item in iterable) - - -def mark_ends(iterable): - """Yield 3-tuples of the form ``(is_first, is_last, item)``. - - >>> list(mark_ends('ABC')) - [(True, False, 'A'), (False, False, 'B'), (False, True, 'C')] - - Use this when looping over an iterable to take special action on its first - and/or last items: - - >>> iterable = ['Header', 100, 200, 'Footer'] - >>> total = 0 - >>> for is_first, is_last, item in mark_ends(iterable): - ... if is_first: - ... continue # Skip the header - ... if is_last: - ... continue # Skip the footer - ... total += item - >>> print(total) - 300 - """ - it = iter(iterable) - - try: - b = next(it) - except StopIteration: - return - - try: - for i in count(): - a = b - b = next(it) - yield i == 0, False, a - - except StopIteration: - yield i == 0, True, a - - -def locate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(locate([0, 1, 1, 0, 1, 0, 0])) - [1, 2, 4] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item. - - >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) - [1, 3] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(locate(iterable, pred=pred, window_size=3)) - [1, 5, 9] - - Use with :func:`seekable` to find indexes and then retrieve the associated - items: - - >>> from itertools import count - >>> from more_itertools import seekable - >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) - >>> it = seekable(source) - >>> pred = lambda x: x > 100 - >>> indexes = locate(it, pred=pred) - >>> i = next(indexes) - >>> it.seek(i) - >>> next(it) - 106 - - """ - if window_size is None: - return compress(count(), map(pred, iterable)) - - if window_size < 1: - raise ValueError('window size must be at least 1') - - it = windowed(iterable, window_size, fillvalue=_marker) - return compress(count(), starmap(pred, it)) - - -def longest_common_prefix(iterables): - """Yield elements of the longest common prefix amongst given *iterables*. - - >>> ''.join(longest_common_prefix(['abcd', 'abc', 'abf'])) - 'ab' - - """ - return (c[0] for c in takewhile(all_equal, zip(*iterables))) - - -def lstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the beginning - for which *pred* returns ``True``. - - For example, to remove a set of items from the start of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(lstrip(iterable, pred)) - [1, 2, None, 3, False, None] - - This function is analogous to to :func:`str.lstrip`, and is essentially - an wrapper for :func:`itertools.dropwhile`. - - """ - return dropwhile(pred, iterable) - - -def rstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the end - for which *pred* returns ``True``. - - For example, to remove a set of items from the end of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(rstrip(iterable, pred)) - [None, False, None, 1, 2, None, 3] - - This function is analogous to :func:`str.rstrip`. - - """ - cache = [] - cache_append = cache.append - cache_clear = cache.clear - for x in iterable: - if pred(x): - cache_append(x) - else: - yield from cache - cache_clear() - yield x - - -def strip(iterable, pred): - """Yield the items from *iterable*, but strip any from the - beginning and end for which *pred* returns ``True``. - - For example, to remove a set of items from both ends of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(strip(iterable, pred)) - [1, 2, None, 3] - - This function is analogous to :func:`str.strip`. - - """ - return rstrip(lstrip(iterable, pred), pred) - - -class islice_extended: - """An extension of :func:`itertools.islice` that supports negative values - for *stop*, *start*, and *step*. - - >>> iterable = iter('abcdefgh') - >>> list(islice_extended(iterable, -4, -1)) - ['e', 'f', 'g'] - - Slices with negative values require some caching of *iterable*, but this - function takes care to minimize the amount of memory required. - - For example, you can use a negative step with an infinite iterator: - - >>> from itertools import count - >>> list(islice_extended(count(), 110, 99, -2)) - [110, 108, 106, 104, 102, 100] - - You can also use slice notation directly: - - >>> iterable = map(str, count()) - >>> it = islice_extended(iterable)[10:20:2] - >>> list(it) - ['10', '12', '14', '16', '18'] - - """ - - def __init__(self, iterable, *args): - it = iter(iterable) - if args: - self._iterable = _islice_helper(it, slice(*args)) - else: - self._iterable = it - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterable) - - def __getitem__(self, key): - if isinstance(key, slice): - return islice_extended(_islice_helper(self._iterable, key)) - - raise TypeError('islice_extended.__getitem__ argument must be a slice') - - -def _islice_helper(it, s): - start = s.start - stop = s.stop - if s.step == 0: - raise ValueError('step argument must be a non-zero integer or None.') - step = s.step or 1 - - if step > 0: - start = 0 if (start is None) else start - - if start < 0: - # Consume all but the last -start items - cache = deque(enumerate(it, 1), maxlen=-start) - len_iter = cache[-1][0] if cache else 0 - - # Adjust start to be positive - i = max(len_iter + start, 0) - - # Adjust stop to be positive - if stop is None: - j = len_iter - elif stop >= 0: - j = min(stop, len_iter) - else: - j = max(len_iter + stop, 0) - - # Slice the cache - n = j - i - if n <= 0: - return - - for index, item in islice(cache, 0, n, step): - yield item - elif (stop is not None) and (stop < 0): - # Advance to the start position - next(islice(it, start, start), None) - - # When stop is negative, we have to carry -stop items while - # iterating - cache = deque(islice(it, -stop), maxlen=-stop) - - for index, item in enumerate(it): - cached_item = cache.popleft() - if index % step == 0: - yield cached_item - cache.append(item) - else: - # When both start and stop are positive we have the normal case - yield from islice(it, start, stop, step) - else: - start = -1 if (start is None) else start - - if (stop is not None) and (stop < 0): - # Consume all but the last items - n = -stop - 1 - cache = deque(enumerate(it, 1), maxlen=n) - len_iter = cache[-1][0] if cache else 0 - - # If start and stop are both negative they are comparable and - # we can just slice. Otherwise we can adjust start to be negative - # and then slice. - if start < 0: - i, j = start, stop - else: - i, j = min(start - len_iter, -1), None - - for index, item in list(cache)[i:j:step]: - yield item - else: - # Advance to the stop position - if stop is not None: - m = stop + 1 - next(islice(it, m, m), None) - - # stop is positive, so if start is negative they are not comparable - # and we need the rest of the items. - if start < 0: - i = start - n = None - # stop is None and start is positive, so we just need items up to - # the start index. - elif stop is None: - i = None - n = start + 1 - # Both stop and start are positive, so they are comparable. - else: - i = None - n = start - stop - if n <= 0: - return - - cache = list(islice(it, n)) - - yield from cache[i::step] - - -def always_reversible(iterable): - """An extension of :func:`reversed` that supports all iterables, not - just those which implement the ``Reversible`` or ``Sequence`` protocols. - - >>> print(*always_reversible(x for x in range(3))) - 2 1 0 - - If the iterable is already reversible, this function returns the - result of :func:`reversed()`. If the iterable is not reversible, - this function will cache the remaining items in the iterable and - yield them in reverse order, which may require significant storage. - """ - try: - return reversed(iterable) - except TypeError: - return reversed(list(iterable)) - - -def consecutive_groups(iterable, ordering=lambda x: x): - """Yield groups of consecutive items using :func:`itertools.groupby`. - The *ordering* function determines whether two items are adjacent by - returning their position. - - By default, the ordering function is the identity function. This is - suitable for finding runs of numbers: - - >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] - >>> for group in consecutive_groups(iterable): - ... print(list(group)) - [1] - [10, 11, 12] - [20] - [30, 31, 32, 33] - [40] - - For finding runs of adjacent letters, try using the :meth:`index` method - of a string of letters: - - >>> from string import ascii_lowercase - >>> iterable = 'abcdfgilmnop' - >>> ordering = ascii_lowercase.index - >>> for group in consecutive_groups(iterable, ordering): - ... print(list(group)) - ['a', 'b', 'c', 'd'] - ['f', 'g'] - ['i'] - ['l', 'm', 'n', 'o', 'p'] - - Each group of consecutive items is an iterator that shares it source with - *iterable*. When an an output group is advanced, the previous group is - no longer available unless its elements are copied (e.g., into a ``list``). - - >>> iterable = [1, 2, 11, 12, 21, 22] - >>> saved_groups = [] - >>> for group in consecutive_groups(iterable): - ... saved_groups.append(list(group)) # Copy group elements - >>> saved_groups - [[1, 2], [11, 12], [21, 22]] - - """ - for k, g in groupby( - enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) - ): - yield map(itemgetter(1), g) - - -def difference(iterable, func=sub, *, initial=None): - """This function is the inverse of :func:`itertools.accumulate`. By default - it will compute the first difference of *iterable* using - :func:`operator.sub`: - - >>> from itertools import accumulate - >>> iterable = accumulate([0, 1, 2, 3, 4]) # produces 0, 1, 3, 6, 10 - >>> list(difference(iterable)) - [0, 1, 2, 3, 4] - - *func* defaults to :func:`operator.sub`, but other functions can be - specified. They will be applied as follows:: - - A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... - - For example, to do progressive division: - - >>> iterable = [1, 2, 6, 24, 120] - >>> func = lambda x, y: x // y - >>> list(difference(iterable, func)) - [1, 2, 3, 4, 5] - - If the *initial* keyword is set, the first element will be skipped when - computing successive differences. - - >>> it = [10, 11, 13, 16] # from accumulate([1, 2, 3], initial=10) - >>> list(difference(it, initial=10)) - [1, 2, 3] - - """ - a, b = tee(iterable) - try: - first = [next(b)] - except StopIteration: - return iter([]) - - if initial is not None: - first = [] - - return chain(first, map(func, b, a)) - - -class SequenceView(Sequence): - """Return a read-only view of the sequence object *target*. - - :class:`SequenceView` objects are analogous to Python's built-in - "dictionary view" types. They provide a dynamic view of a sequence's items, - meaning that when the sequence updates, so does the view. - - >>> seq = ['0', '1', '2'] - >>> view = SequenceView(seq) - >>> view - SequenceView(['0', '1', '2']) - >>> seq.append('3') - >>> view - SequenceView(['0', '1', '2', '3']) - - Sequence views support indexing, slicing, and length queries. They act - like the underlying sequence, except they don't allow assignment: - - >>> view[1] - '1' - >>> view[1:-1] - ['1', '2'] - >>> len(view) - 4 - - Sequence views are useful as an alternative to copying, as they don't - require (much) extra storage. - - """ - - def __init__(self, target): - if not isinstance(target, Sequence): - raise TypeError - self._target = target - - def __getitem__(self, index): - return self._target[index] - - def __len__(self): - return len(self._target) - - def __repr__(self): - return '{}({})'.format(self.__class__.__name__, repr(self._target)) - - -class seekable: - """Wrap an iterator to allow for seeking backward and forward. This - progressively caches the items in the source iterable so they can be - re-visited. - - Call :meth:`seek` with an index to seek to that position in the source - iterable. - - To "reset" an iterator, seek to ``0``: - - >>> from itertools import count - >>> it = seekable((str(n) for n in count())) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> it.seek(0) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> next(it) - '3' - - You can also seek forward: - - >>> it = seekable((str(n) for n in range(20))) - >>> it.seek(10) - >>> next(it) - '10' - >>> it.relative_seek(-2) # Seeking relative to the current position - >>> next(it) - '9' - >>> it.seek(20) # Seeking past the end of the source isn't a problem - >>> list(it) - [] - >>> it.seek(0) # Resetting works even after hitting the end - >>> next(it), next(it), next(it) - ('0', '1', '2') - - Call :meth:`peek` to look ahead one item without advancing the iterator: - - >>> it = seekable('1234') - >>> it.peek() - '1' - >>> list(it) - ['1', '2', '3', '4'] - >>> it.peek(default='empty') - 'empty' - - Before the iterator is at its end, calling :func:`bool` on it will return - ``True``. After it will return ``False``: - - >>> it = seekable('5678') - >>> bool(it) - True - >>> list(it) - ['5', '6', '7', '8'] - >>> bool(it) - False - - You may view the contents of the cache with the :meth:`elements` method. - That returns a :class:`SequenceView`, a view that updates automatically: - - >>> it = seekable((str(n) for n in range(10))) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> elements = it.elements() - >>> elements - SequenceView(['0', '1', '2']) - >>> next(it) - '3' - >>> elements - SequenceView(['0', '1', '2', '3']) - - By default, the cache grows as the source iterable progresses, so beware of - wrapping very large or infinite iterables. Supply *maxlen* to limit the - size of the cache (this of course limits how far back you can seek). - - >>> from itertools import count - >>> it = seekable((str(n) for n in count()), maxlen=2) - >>> next(it), next(it), next(it), next(it) - ('0', '1', '2', '3') - >>> list(it.elements()) - ['2', '3'] - >>> it.seek(0) - >>> next(it), next(it), next(it), next(it) - ('2', '3', '4', '5') - >>> next(it) - '6' - - """ - - def __init__(self, iterable, maxlen=None): - self._source = iter(iterable) - if maxlen is None: - self._cache = [] - else: - self._cache = deque([], maxlen) - self._index = None - - def __iter__(self): - return self - - def __next__(self): - if self._index is not None: - try: - item = self._cache[self._index] - except IndexError: - self._index = None - else: - self._index += 1 - return item - - item = next(self._source) - self._cache.append(item) - return item - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - try: - peeked = next(self) - except StopIteration: - if default is _marker: - raise - return default - if self._index is None: - self._index = len(self._cache) - self._index -= 1 - return peeked - - def elements(self): - return SequenceView(self._cache) - - def seek(self, index): - self._index = index - remainder = index - len(self._cache) - if remainder > 0: - consume(self, remainder) - - def relative_seek(self, count): - index = len(self._cache) - self.seek(max(index + count, 0)) - - -class run_length: - """ - :func:`run_length.encode` compresses an iterable with run-length encoding. - It yields groups of repeated items with the count of how many times they - were repeated: - - >>> uncompressed = 'abbcccdddd' - >>> list(run_length.encode(uncompressed)) - [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - - :func:`run_length.decode` decompresses an iterable that was previously - compressed with run-length encoding. It yields the items of the - decompressed iterable: - - >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> list(run_length.decode(compressed)) - ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] - - """ - - @staticmethod - def encode(iterable): - return ((k, ilen(g)) for k, g in groupby(iterable)) - - @staticmethod - def decode(iterable): - return chain.from_iterable(repeat(k, n) for k, n in iterable) - - -def exactly_n(iterable, n, predicate=bool): - """Return ``True`` if exactly ``n`` items in the iterable are ``True`` - according to the *predicate* function. - - >>> exactly_n([True, True, False], 2) - True - >>> exactly_n([True, True, False], 1) - False - >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) - True - - The iterable will be advanced until ``n + 1`` truthy items are encountered, - so avoid calling it on infinite iterables. - - """ - return len(take(n + 1, filter(predicate, iterable))) == n - - -def circular_shifts(iterable): - """Return a list of circular shifts of *iterable*. - - >>> circular_shifts(range(4)) - [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] - """ - lst = list(iterable) - return take(len(lst), windowed(cycle(lst), len(lst))) - - -def make_decorator(wrapping_func, result_index=0): - """Return a decorator version of *wrapping_func*, which is a function that - modifies an iterable. *result_index* is the position in that function's - signature where the iterable goes. - - This lets you use itertools on the "production end," i.e. at function - definition. This can augment what the function returns without changing the - function's code. - - For example, to produce a decorator version of :func:`chunked`: - - >>> from more_itertools import chunked - >>> chunker = make_decorator(chunked, result_index=0) - >>> @chunker(3) - ... def iter_range(n): - ... return iter(range(n)) - ... - >>> list(iter_range(9)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8]] - - To only allow truthy items to be returned: - - >>> truth_serum = make_decorator(filter, result_index=1) - >>> @truth_serum(bool) - ... def boolean_test(): - ... return [0, 1, '', ' ', False, True] - ... - >>> list(boolean_test()) - [1, ' ', True] - - The :func:`peekable` and :func:`seekable` wrappers make for practical - decorators: - - >>> from more_itertools import peekable - >>> peekable_function = make_decorator(peekable) - >>> @peekable_function() - ... def str_range(*args): - ... return (str(x) for x in range(*args)) - ... - >>> it = str_range(1, 20, 2) - >>> next(it), next(it), next(it) - ('1', '3', '5') - >>> it.peek() - '7' - >>> next(it) - '7' - - """ - - # See https://sites.google.com/site/bbayles/index/decorator_factory for - # notes on how this works. - def decorator(*wrapping_args, **wrapping_kwargs): - def outer_wrapper(f): - def inner_wrapper(*args, **kwargs): - result = f(*args, **kwargs) - wrapping_args_ = list(wrapping_args) - wrapping_args_.insert(result_index, result) - return wrapping_func(*wrapping_args_, **wrapping_kwargs) - - return inner_wrapper - - return outer_wrapper - - return decorator - - -def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): - """Return a dictionary that maps the items in *iterable* to categories - defined by *keyfunc*, transforms them with *valuefunc*, and - then summarizes them by category with *reducefunc*. - - *valuefunc* defaults to the identity function if it is unspecified. - If *reducefunc* is unspecified, no summarization takes place: - - >>> keyfunc = lambda x: x.upper() - >>> result = map_reduce('abbccc', keyfunc) - >>> sorted(result.items()) - [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] - - Specifying *valuefunc* transforms the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> result = map_reduce('abbccc', keyfunc, valuefunc) - >>> sorted(result.items()) - [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] - - Specifying *reducefunc* summarizes the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> reducefunc = sum - >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) - >>> sorted(result.items()) - [('A', 1), ('B', 2), ('C', 3)] - - You may want to filter the input iterable before applying the map/reduce - procedure: - - >>> all_items = range(30) - >>> items = [x for x in all_items if 10 <= x <= 20] # Filter - >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 - >>> categories = map_reduce(items, keyfunc=keyfunc) - >>> sorted(categories.items()) - [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] - >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) - >>> sorted(summaries.items()) - [(0, 90), (1, 75)] - - Note that all items in the iterable are gathered into a list before the - summarization step, which may require significant storage. - - The returned object is a :obj:`collections.defaultdict` with the - ``default_factory`` set to ``None``, such that it behaves like a normal - dictionary. - - """ - valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc - - ret = defaultdict(list) - for item in iterable: - key = keyfunc(item) - value = valuefunc(item) - ret[key].append(value) - - if reducefunc is not None: - for key, value_list in ret.items(): - ret[key] = reducefunc(value_list) - - ret.default_factory = None - return ret - - -def rlocate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``, starting from the right and moving left. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 - [4, 2, 1] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item: - - >>> iterable = iter('abcb') - >>> pred = lambda x: x == 'b' - >>> list(rlocate(iterable, pred)) - [3, 1] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(rlocate(iterable, pred=pred, window_size=3)) - [9, 5, 1] - - Beware, this function won't return anything for infinite iterables. - If *iterable* is reversible, ``rlocate`` will reverse it and search from - the right. Otherwise, it will search from the left and return the results - in reverse order. - - See :func:`locate` to for other example applications. - - """ - if window_size is None: - try: - len_iter = len(iterable) - return (len_iter - i - 1 for i in locate(reversed(iterable), pred)) - except TypeError: - pass - - return reversed(list(locate(iterable, pred, window_size))) - - -def replace(iterable, pred, substitutes, count=None, window_size=1): - """Yield the items from *iterable*, replacing the items for which *pred* - returns ``True`` with the items from the iterable *substitutes*. - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] - >>> pred = lambda x: x == 0 - >>> substitutes = (2, 3) - >>> list(replace(iterable, pred, substitutes)) - [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] - - If *count* is given, the number of replacements will be limited: - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] - >>> pred = lambda x: x == 0 - >>> substitutes = [None] - >>> list(replace(iterable, pred, substitutes, count=2)) - [1, 1, None, 1, 1, None, 1, 1, 0] - - Use *window_size* to control the number of items passed as arguments to - *pred*. This allows for locating and replacing subsequences. - - >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] - >>> window_size = 3 - >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred - >>> substitutes = [3, 4] # Splice in these items - >>> list(replace(iterable, pred, substitutes, window_size=window_size)) - [3, 4, 5, 3, 4, 5] - - """ - if window_size < 1: - raise ValueError('window_size must be at least 1') - - # Save the substitutes iterable, since it's used more than once - substitutes = tuple(substitutes) - - # Add padding such that the number of windows matches the length of the - # iterable - it = chain(iterable, [_marker] * (window_size - 1)) - windows = windowed(it, window_size) - - n = 0 - for w in windows: - # If the current window matches our predicate (and we haven't hit - # our maximum number of replacements), splice in the substitutes - # and then consume the following windows that overlap with this one. - # For example, if the iterable is (0, 1, 2, 3, 4...) - # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... - # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) - if pred(*w): - if (count is None) or (n < count): - n += 1 - yield from substitutes - consume(windows, window_size - 1) - continue - - # If there was no match (or we've reached the replacement limit), - # yield the first item from the window. - if w and (w[0] is not _marker): - yield w[0] - - -def partitions(iterable): - """Yield all possible order-preserving partitions of *iterable*. - - >>> iterable = 'abc' - >>> for part in partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['a', 'b', 'c'] - - This is unrelated to :func:`partition`. - - """ - sequence = list(iterable) - n = len(sequence) - for i in powerset(range(1, n)): - yield [sequence[i:j] for i, j in zip((0,) + i, i + (n,))] - - -def set_partitions(iterable, k=None): - """ - Yield the set partitions of *iterable* into *k* parts. Set partitions are - not order-preserving. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable, 2): - ... print([''.join(p) for p in part]) - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - - - If *k* is not given, every set partition is generated. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - ['a', 'b', 'c'] - - """ - L = list(iterable) - n = len(L) - if k is not None: - if k < 1: - raise ValueError( - "Can't partition in a negative or zero number of groups" - ) - elif k > n: - return - - def set_partitions_helper(L, k): - n = len(L) - if k == 1: - yield [L] - elif n == k: - yield [[s] for s in L] - else: - e, *M = L - for p in set_partitions_helper(M, k - 1): - yield [[e], *p] - for p in set_partitions_helper(M, k): - for i in range(len(p)): - yield p[:i] + [[e] + p[i]] + p[i + 1 :] - - if k is None: - for k in range(1, n + 1): - yield from set_partitions_helper(L, k) - else: - yield from set_partitions_helper(L, k) - - -class time_limited: - """ - Yield items from *iterable* until *limit_seconds* have passed. - If the time limit expires before all items have been yielded, the - ``timed_out`` parameter will be set to ``True``. - - >>> from time import sleep - >>> def generator(): - ... yield 1 - ... yield 2 - ... sleep(0.2) - ... yield 3 - >>> iterable = time_limited(0.1, generator()) - >>> list(iterable) - [1, 2] - >>> iterable.timed_out - True - - Note that the time is checked before each item is yielded, and iteration - stops if the time elapsed is greater than *limit_seconds*. If your time - limit is 1 second, but it takes 2 seconds to generate the first item from - the iterable, the function will run for 2 seconds and not yield anything. - As a special case, when *limit_seconds* is zero, the iterator never - returns anything. - - """ - - def __init__(self, limit_seconds, iterable): - if limit_seconds < 0: - raise ValueError('limit_seconds must be positive') - self.limit_seconds = limit_seconds - self._iterable = iter(iterable) - self._start_time = monotonic() - self.timed_out = False - - def __iter__(self): - return self - - def __next__(self): - if self.limit_seconds == 0: - self.timed_out = True - raise StopIteration - item = next(self._iterable) - if monotonic() - self._start_time > self.limit_seconds: - self.timed_out = True - raise StopIteration - - return item - - -def only(iterable, default=None, too_long=None): - """If *iterable* has only one item, return it. - If it has zero items, return *default*. - If it has more than one item, raise the exception given by *too_long*, - which is ``ValueError`` by default. - - >>> only([], default='missing') - 'missing' - >>> only([1]) - 1 - >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 1, 2, - and perhaps more.' - >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - TypeError - - Note that :func:`only` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check - iterable contents less destructively. - """ - it = iter(iterable) - first_value = next(it, default) - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -class _IChunk: - def __init__(self, iterable, n): - self._it = islice(iterable, n) - self._cache = deque() - - def fill_cache(self): - self._cache.extend(self._it) - - def __iter__(self): - return self - - def __next__(self): - try: - return next(self._it) - except StopIteration: - if self._cache: - return self._cache.popleft() - else: - raise - - -def ichunked(iterable, n): - """Break *iterable* into sub-iterables with *n* elements each. - :func:`ichunked` is like :func:`chunked`, but it yields iterables - instead of lists. - - If the sub-iterables are read in order, the elements of *iterable* - won't be stored in memory. - If they are read out of order, :func:`itertools.tee` is used to cache - elements as necessary. - - >>> from itertools import count - >>> all_chunks = ichunked(count(), 4) - >>> c_1, c_2, c_3 = next(all_chunks), next(all_chunks), next(all_chunks) - >>> list(c_2) # c_1's elements have been cached; c_3's haven't been - [4, 5, 6, 7] - >>> list(c_1) - [0, 1, 2, 3] - >>> list(c_3) - [8, 9, 10, 11] - - """ - source = peekable(iter(iterable)) - ichunk_marker = object() - while True: - # Check to see whether we're at the end of the source iterable - item = source.peek(ichunk_marker) - if item is ichunk_marker: - return - - chunk = _IChunk(source, n) - yield chunk - - # Advance the source iterable and fill previous chunk's cache - chunk.fill_cache() - - -def iequals(*iterables): - """Return ``True`` if all given *iterables* are equal to each other, - which means that they contain the same elements in the same order. - - The function is useful for comparing iterables of different data types - or iterables that do not support equality checks. - - >>> iequals("abc", ['a', 'b', 'c'], ('a', 'b', 'c'), iter("abc")) - True - - >>> iequals("abc", "acb") - False - - Not to be confused with :func:`all_equal`, which checks whether all - elements of iterable are equal to each other. - - """ - return all(map(all_equal, zip_longest(*iterables, fillvalue=object()))) - - -def distinct_combinations(iterable, r): - """Yield the distinct combinations of *r* items taken from *iterable*. - - >>> list(distinct_combinations([0, 0, 1], 2)) - [(0, 0), (0, 1)] - - Equivalent to ``set(combinations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - """ - if r < 0: - raise ValueError('r must be non-negative') - elif r == 0: - yield () - return - pool = tuple(iterable) - generators = [unique_everseen(enumerate(pool), key=itemgetter(1))] - current_combo = [None] * r - level = 0 - while generators: - try: - cur_idx, p = next(generators[-1]) - except StopIteration: - generators.pop() - level -= 1 - continue - current_combo[level] = p - if level + 1 == r: - yield tuple(current_combo) - else: - generators.append( - unique_everseen( - enumerate(pool[cur_idx + 1 :], cur_idx + 1), - key=itemgetter(1), - ) - ) - level += 1 - - -def filter_except(validator, iterable, *exceptions): - """Yield the items from *iterable* for which the *validator* function does - not raise one of the specified *exceptions*. - - *validator* is called for each item in *iterable*. - It should be a function that accepts one argument and raises an exception - if that item is not valid. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(filter_except(int, iterable, ValueError, TypeError)) - ['1', '2', '4'] - - If an exception other than one given by *exceptions* is raised by - *validator*, it is raised like normal. - """ - for item in iterable: - try: - validator(item) - except exceptions: - pass - else: - yield item - - -def map_except(function, iterable, *exceptions): - """Transform each item from *iterable* with *function* and yield the - result, unless *function* raises one of the specified *exceptions*. - - *function* is called to transform each item in *iterable*. - It should accept one argument. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(map_except(int, iterable, ValueError, TypeError)) - [1, 2, 4] - - If an exception other than one given by *exceptions* is raised by - *function*, it is raised like normal. - """ - for item in iterable: - try: - yield function(item) - except exceptions: - pass - - -def map_if(iterable, pred, func, func_else=lambda x: x): - """Evaluate each item from *iterable* using *pred*. If the result is - equivalent to ``True``, transform the item with *func* and yield it. - Otherwise, transform the item with *func_else* and yield it. - - *pred*, *func*, and *func_else* should each be functions that accept - one argument. By default, *func_else* is the identity function. - - >>> from math import sqrt - >>> iterable = list(range(-5, 5)) - >>> iterable - [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4] - >>> list(map_if(iterable, lambda x: x > 3, lambda x: 'toobig')) - [-5, -4, -3, -2, -1, 0, 1, 2, 3, 'toobig'] - >>> list(map_if(iterable, lambda x: x >= 0, - ... lambda x: f'{sqrt(x):.2f}', lambda x: None)) - [None, None, None, None, None, '0.00', '1.00', '1.41', '1.73', '2.00'] - """ - for item in iterable: - yield func(item) if pred(item) else func_else(item) - - -def _sample_unweighted(iterable, k): - # Implementation of "Algorithm L" from the 1994 paper by Kim-Hung Li: - # "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". - - # Fill up the reservoir (collection of samples) with the first `k` samples - reservoir = take(k, iterable) - - # Generate random number that's the largest in a sample of k U(0,1) numbers - # Largest order statistic: https://en.wikipedia.org/wiki/Order_statistic - W = exp(log(random()) / k) - - # The number of elements to skip before changing the reservoir is a random - # number with a geometric distribution. Sample it using random() and logs. - next_index = k + floor(log(random()) / log(1 - W)) - - for index, element in enumerate(iterable, k): - if index == next_index: - reservoir[randrange(k)] = element - # The new W is the largest in a sample of k U(0, `old_W`) numbers - W *= exp(log(random()) / k) - next_index += floor(log(random()) / log(1 - W)) + 1 - - return reservoir - - -def _sample_weighted(iterable, k, weights): - # Implementation of "A-ExpJ" from the 2006 paper by Efraimidis et al. : - # "Weighted random sampling with a reservoir". - - # Log-transform for numerical stability for weights that are small/large - weight_keys = (log(random()) / weight for weight in weights) - - # Fill up the reservoir (collection of samples) with the first `k` - # weight-keys and elements, then heapify the list. - reservoir = take(k, zip(weight_keys, iterable)) - heapify(reservoir) - - # The number of jumps before changing the reservoir is a random variable - # with an exponential distribution. Sample it using random() and logs. - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - - for weight, element in zip(weights, iterable): - if weight >= weights_to_skip: - # The notation here is consistent with the paper, but we store - # the weight-keys in log-space for better numerical stability. - smallest_weight_key, _ = reservoir[0] - t_w = exp(weight * smallest_weight_key) - r_2 = uniform(t_w, 1) # generate U(t_w, 1) - weight_key = log(r_2) / weight - heapreplace(reservoir, (weight_key, element)) - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - else: - weights_to_skip -= weight - - # Equivalent to [element for weight_key, element in sorted(reservoir)] - return [heappop(reservoir)[1] for _ in range(k)] - - -def sample(iterable, k, weights=None): - """Return a *k*-length list of elements chosen (without replacement) - from the *iterable*. Like :func:`random.sample`, but works on iterables - of unknown length. - - >>> iterable = range(100) - >>> sample(iterable, 5) # doctest: +SKIP - [81, 60, 96, 16, 4] - - An iterable with *weights* may also be given: - - >>> iterable = range(100) - >>> weights = (i * i + 1 for i in range(100)) - >>> sampled = sample(iterable, 5, weights=weights) # doctest: +SKIP - [79, 67, 74, 66, 78] - - The algorithm can also be used to generate weighted random permutations. - The relative weight of each item determines the probability that it - appears late in the permutation. - - >>> data = "abcdefgh" - >>> weights = range(1, len(data) + 1) - >>> sample(data, k=len(data), weights=weights) # doctest: +SKIP - ['c', 'a', 'b', 'e', 'g', 'd', 'h', 'f'] - """ - if k == 0: - return [] - - iterable = iter(iterable) - if weights is None: - return _sample_unweighted(iterable, k) - else: - weights = iter(weights) - return _sample_weighted(iterable, k, weights) - - -def is_sorted(iterable, key=None, reverse=False, strict=False): - """Returns ``True`` if the items of iterable are in sorted order, and - ``False`` otherwise. *key* and *reverse* have the same meaning that they do - in the built-in :func:`sorted` function. - - >>> is_sorted(['1', '2', '3', '4', '5'], key=int) - True - >>> is_sorted([5, 4, 3, 1, 2], reverse=True) - False - - If *strict*, tests for strict sorting, that is, returns ``False`` if equal - elements are found: - - >>> is_sorted([1, 2, 2]) - True - >>> is_sorted([1, 2, 2], strict=True) - False - - The function returns ``False`` after encountering the first out-of-order - item. If there are no out-of-order items, the iterable is exhausted. - """ - - compare = (le if reverse else ge) if strict else (lt if reverse else gt) - it = iterable if key is None else map(key, iterable) - return not any(starmap(compare, pairwise(it))) - - -class AbortThread(BaseException): - pass - - -class callback_iter: - """Convert a function that uses callbacks to an iterator. - - Let *func* be a function that takes a `callback` keyword argument. - For example: - - >>> def func(callback=None): - ... for i, c in [(1, 'a'), (2, 'b'), (3, 'c')]: - ... if callback: - ... callback(i, c) - ... return 4 - - - Use ``with callback_iter(func)`` to get an iterator over the parameters - that are delivered to the callback. - - >>> with callback_iter(func) as it: - ... for args, kwargs in it: - ... print(args) - (1, 'a') - (2, 'b') - (3, 'c') - - The function will be called in a background thread. The ``done`` property - indicates whether it has completed execution. - - >>> it.done - True - - If it completes successfully, its return value will be available - in the ``result`` property. - - >>> it.result - 4 - - Notes: - - * If the function uses some keyword argument besides ``callback``, supply - *callback_kwd*. - * If it finished executing, but raised an exception, accessing the - ``result`` property will raise the same exception. - * If it hasn't finished executing, accessing the ``result`` - property from within the ``with`` block will raise ``RuntimeError``. - * If it hasn't finished executing, accessing the ``result`` property from - outside the ``with`` block will raise a - ``more_itertools.AbortThread`` exception. - * Provide *wait_seconds* to adjust how frequently the it is polled for - output. - - """ - - def __init__(self, func, callback_kwd='callback', wait_seconds=0.1): - self._func = func - self._callback_kwd = callback_kwd - self._aborted = False - self._future = None - self._wait_seconds = wait_seconds - # Lazily import concurrent.future - self._executor = __import__( - ).futures.__import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1) - self._iterator = self._reader() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._aborted = True - self._executor.shutdown() - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterator) - - @property - def done(self): - if self._future is None: - return False - return self._future.done() - - @property - def result(self): - if not self.done: - raise RuntimeError('Function has not yet completed') - - return self._future.result() - - def _reader(self): - q = Queue() - - def callback(*args, **kwargs): - if self._aborted: - raise AbortThread('canceled by user') - - q.put((args, kwargs)) - - self._future = self._executor.submit( - self._func, **{self._callback_kwd: callback} - ) - - while True: - try: - item = q.get(timeout=self._wait_seconds) - except Empty: - pass - else: - q.task_done() - yield item - - if self._future.done(): - break - - remaining = [] - while True: - try: - item = q.get_nowait() - except Empty: - break - else: - q.task_done() - remaining.append(item) - q.join() - yield from remaining - - -def windowed_complete(iterable, n): - """ - Yield ``(beginning, middle, end)`` tuples, where: - - * Each ``middle`` has *n* items from *iterable* - * Each ``beginning`` has the items before the ones in ``middle`` - * Each ``end`` has the items after the ones in ``middle`` - - >>> iterable = range(7) - >>> n = 3 - >>> for beginning, middle, end in windowed_complete(iterable, n): - ... print(beginning, middle, end) - () (0, 1, 2) (3, 4, 5, 6) - (0,) (1, 2, 3) (4, 5, 6) - (0, 1) (2, 3, 4) (5, 6) - (0, 1, 2) (3, 4, 5) (6,) - (0, 1, 2, 3) (4, 5, 6) () - - Note that *n* must be at least 0 and most equal to the length of - *iterable*. - - This function will exhaust the iterable and may require significant - storage. - """ - if n < 0: - raise ValueError('n must be >= 0') - - seq = tuple(iterable) - size = len(seq) - - if n > size: - raise ValueError('n must be <= len(seq)') - - for i in range(size - n + 1): - beginning = seq[:i] - middle = seq[i : i + n] - end = seq[i + n :] - yield beginning, middle, end - - -def all_unique(iterable, key=None): - """ - Returns ``True`` if all the elements of *iterable* are unique (no two - elements are equal). - - >>> all_unique('ABCB') - False - - If a *key* function is specified, it will be used to make comparisons. - - >>> all_unique('ABCb') - True - >>> all_unique('ABCb', str.lower) - False - - The function returns as soon as the first non-unique element is - encountered. Iterables with a mix of hashable and unhashable items can - be used, but the function will be slower for unhashable items. - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - for element in map(key, iterable) if key else iterable: - try: - if element in seenset: - return False - seenset_add(element) - except TypeError: - if element in seenlist: - return False - seenlist_add(element) - return True - - -def nth_product(index, *args): - """Equivalent to ``list(product(*args))[index]``. - - The products of *args* can be ordered lexicographically. - :func:`nth_product` computes the product at sort position *index* without - computing the previous products. - - >>> nth_product(8, range(2), range(2), range(2), range(2)) - (1, 0, 0, 0) - - ``IndexError`` will be raised if the given *index* is invalid. - """ - pools = list(map(tuple, reversed(args))) - ns = list(map(len, pools)) - - c = reduce(mul, ns) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - result = [] - for pool, n in zip(pools, ns): - result.append(pool[index % n]) - index //= n - - return tuple(reversed(result)) - - -def nth_permutation(iterable, r, index): - """Equivalent to ``list(permutations(iterable, r))[index]``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`nth_permutation` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences. - - >>> nth_permutation('ghijk', 2, 5) - ('h', 'i') - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = list(iterable) - n = len(pool) - - if r is None or r == n: - r, c = n, factorial(n) - elif not 0 <= r < n: - raise ValueError - else: - c = perm(n, r) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - if c == 0: - return tuple() - - result = [0] * r - q = index * factorial(n) // c if r < n else index - for d in range(1, n + 1): - q, i = divmod(q, d) - if 0 <= n - d < r: - result[n - d] = i - if q == 0: - break - - return tuple(map(pool.pop, result)) - - -def nth_combination_with_replacement(iterable, r, index): - """Equivalent to - ``list(combinations_with_replacement(iterable, r))[index]``. - - - The subsequences with repetition of *iterable* that are of length *r* can - be ordered lexicographically. :func:`nth_combination_with_replacement` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences with replacement. - - >>> nth_combination_with_replacement(range(5), 3, 5) - (0, 1, 1) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = comb(n + r - 1, r) - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - i = 0 - while r: - r -= 1 - while n >= 0: - num_combs = comb(n + r - 1, r) - if index < num_combs: - break - n -= 1 - i += 1 - index -= num_combs - result.append(pool[i]) - - return tuple(result) - - -def value_chain(*args): - """Yield all arguments passed to the function in the same order in which - they were passed. If an argument itself is iterable then iterate over its - values. - - >>> list(value_chain(1, 2, 3, [4, 5, 6])) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and are emitted - as-is: - - >>> list(value_chain('12', '34', ['56', '78'])) - ['12', '34', '56', '78'] - - - Multiple levels of nesting are not flattened. - - """ - for value in args: - if isinstance(value, (str, bytes)): - yield value - continue - try: - yield from value - except TypeError: - yield value - - -def product_index(element, *args): - """Equivalent to ``list(product(*args)).index(element)`` - - The products of *args* can be ordered lexicographically. - :func:`product_index` computes the first index of *element* without - computing the previous products. - - >>> product_index([8, 2], range(10), range(5)) - 42 - - ``ValueError`` will be raised if the given *element* isn't in the product - of *args*. - """ - index = 0 - - for x, pool in zip_longest(element, args, fillvalue=_marker): - if x is _marker or pool is _marker: - raise ValueError('element is not a product of args') - - pool = tuple(pool) - index = index * len(pool) + pool.index(x) - - return index - - -def combination_index(element, iterable): - """Equivalent to ``list(combinations(iterable, r)).index(element)`` - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`combination_index` computes the index of the - first *element*, without computing the previous combinations. - - >>> combination_index('adf', 'abcdefg') - 10 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations of *iterable*. - """ - element = enumerate(element) - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = enumerate(iterable) - for n, x in pool: - if x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - else: - raise ValueError('element is not a combination of iterable') - - n, _ = last(pool, default=(n, None)) - - # Python versions below 3.8 don't have math.comb - index = 1 - for i, j in enumerate(reversed(indexes), start=1): - j = n - j - if i <= j: - index += comb(j, i) - - return comb(n + 1, k + 1) - index - - -def combination_with_replacement_index(element, iterable): - """Equivalent to - ``list(combinations_with_replacement(iterable, r)).index(element)`` - - The subsequences with repetition of *iterable* that are of length *r* can - be ordered lexicographically. :func:`combination_with_replacement_index` - computes the index of the first *element*, without computing the previous - combinations with replacement. - - >>> combination_with_replacement_index('adf', 'abcdefg') - 20 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations with replacement of *iterable*. - """ - element = tuple(element) - l = len(element) - element = enumerate(element) - - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = tuple(iterable) - for n, x in enumerate(pool): - while x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - if y is None: - break - else: - raise ValueError( - 'element is not a combination with replacement of iterable' - ) - - n = len(pool) - occupations = [0] * n - for p in indexes: - occupations[p] += 1 - - index = 0 - cumulative_sum = 0 - for k in range(1, n): - cumulative_sum += occupations[k - 1] - j = l + n - 1 - k - cumulative_sum - i = n - k - if i <= j: - index += comb(j, i) - - return index - - -def permutation_index(element, iterable): - """Equivalent to ``list(permutations(iterable, r)).index(element)``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`permutation_index` - computes the index of the first *element* directly, without computing - the previous permutations. - - >>> permutation_index([1, 3, 2], range(5)) - 19 - - ``ValueError`` will be raised if the given *element* isn't one of the - permutations of *iterable*. - """ - index = 0 - pool = list(iterable) - for i, x in zip(range(len(pool), -1, -1), element): - r = pool.index(x) - index = index * i + r - del pool[r] - - return index - - -class countable: - """Wrap *iterable* and keep a count of how many items have been consumed. - - The ``items_seen`` attribute starts at ``0`` and increments as the iterable - is consumed: - - >>> iterable = map(str, range(10)) - >>> it = countable(iterable) - >>> it.items_seen - 0 - >>> next(it), next(it) - ('0', '1') - >>> list(it) - ['2', '3', '4', '5', '6', '7', '8', '9'] - >>> it.items_seen - 10 - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self.items_seen = 0 - - def __iter__(self): - return self - - def __next__(self): - item = next(self._it) - self.items_seen += 1 - - return item - - -def chunked_even(iterable, n): - """Break *iterable* into lists of approximately length *n*. - Items are distributed such the lengths of the lists differ by at most - 1 item. - - >>> iterable = [1, 2, 3, 4, 5, 6, 7] - >>> n = 3 - >>> list(chunked_even(iterable, n)) # List lengths: 3, 2, 2 - [[1, 2, 3], [4, 5], [6, 7]] - >>> list(chunked(iterable, n)) # List lengths: 3, 3, 1 - [[1, 2, 3], [4, 5, 6], [7]] - - """ - - len_method = getattr(iterable, '__len__', None) - - if len_method is None: - return _chunked_even_online(iterable, n) - else: - return _chunked_even_finite(iterable, len_method(), n) - - -def _chunked_even_online(iterable, n): - buffer = [] - maxbuf = n + (n - 2) * (n - 1) - for x in iterable: - buffer.append(x) - if len(buffer) == maxbuf: - yield buffer[:n] - buffer = buffer[n:] - yield from _chunked_even_finite(buffer, len(buffer), n) - - -def _chunked_even_finite(iterable, N, n): - if N < 1: - return - - # Lists are either size `full_size <= n` or `partial_size = full_size - 1` - q, r = divmod(N, n) - num_lists = q + (1 if r > 0 else 0) - q, r = divmod(N, num_lists) - full_size = q + (1 if r > 0 else 0) - partial_size = full_size - 1 - num_full = N - partial_size * num_lists - num_partial = num_lists - num_full - - # Yield num_full lists of full_size - partial_start_idx = num_full * full_size - if full_size > 0: - for i in range(0, partial_start_idx, full_size): - yield list(islice(iterable, i, i + full_size)) - - # Yield num_partial lists of partial_size - if partial_size > 0: - for i in range( - partial_start_idx, - partial_start_idx + (num_partial * partial_size), - partial_size, - ): - yield list(islice(iterable, i, i + partial_size)) - - -def zip_broadcast(*objects, scalar_types=(str, bytes), strict=False): - """A version of :func:`zip` that "broadcasts" any scalar - (i.e., non-iterable) items into output tuples. - - >>> iterable_1 = [1, 2, 3] - >>> iterable_2 = ['a', 'b', 'c'] - >>> scalar = '_' - >>> list(zip_broadcast(iterable_1, iterable_2, scalar)) - [(1, 'a', '_'), (2, 'b', '_'), (3, 'c', '_')] - - The *scalar_types* keyword argument determines what types are considered - scalar. It is set to ``(str, bytes)`` by default. Set it to ``None`` to - treat strings and byte strings as iterable: - - >>> list(zip_broadcast('abc', 0, 'xyz', scalar_types=None)) - [('a', 0, 'x'), ('b', 0, 'y'), ('c', 0, 'z')] - - If the *strict* keyword argument is ``True``, then - ``UnequalIterablesError`` will be raised if any of the iterables have - different lengths. - """ - - def is_scalar(obj): - if scalar_types and isinstance(obj, scalar_types): - return True - try: - iter(obj) - except TypeError: - return True - else: - return False - - size = len(objects) - if not size: - return - - new_item = [None] * size - iterables, iterable_positions = [], [] - for i, obj in enumerate(objects): - if is_scalar(obj): - new_item[i] = obj - else: - iterables.append(iter(obj)) - iterable_positions.append(i) - - if not iterables: - yield tuple(objects) - return - - zipper = _zip_equal if strict else zip - for item in zipper(*iterables): - for i, new_item[i] in zip(iterable_positions, item): - pass - yield tuple(new_item) - - -def unique_in_window(iterable, n, key=None): - """Yield the items from *iterable* that haven't been seen recently. - *n* is the size of the lookback window. - - >>> iterable = [0, 1, 0, 2, 3, 0] - >>> n = 3 - >>> list(unique_in_window(iterable, n)) - [0, 1, 2, 3, 0] - - The *key* function, if provided, will be used to determine uniqueness: - - >>> list(unique_in_window('abAcda', 3, key=lambda x: x.lower())) - ['a', 'b', 'c', 'd', 'a'] - - The items in *iterable* must be hashable. - - """ - if n <= 0: - raise ValueError('n must be greater than 0') - - window = deque(maxlen=n) - counts = defaultdict(int) - use_key = key is not None - - for item in iterable: - if len(window) == n: - to_discard = window[0] - if counts[to_discard] == 1: - del counts[to_discard] - else: - counts[to_discard] -= 1 - - k = key(item) if use_key else item - if k not in counts: - yield item - counts[k] += 1 - window.append(k) - - -def duplicates_everseen(iterable, key=None): - """Yield duplicate elements after their first appearance. - - >>> list(duplicates_everseen('mississippi')) - ['s', 'i', 's', 's', 'i', 'p', 'i'] - >>> list(duplicates_everseen('AaaBbbCccAaa', str.lower)) - ['a', 'a', 'b', 'b', 'c', 'c', 'A', 'a', 'a'] - - This function is analogous to :func:`unique_everseen` and is subject to - the same performance considerations. - - """ - seen_set = set() - seen_list = [] - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seen_set: - seen_set.add(k) - else: - yield element - except TypeError: - if k not in seen_list: - seen_list.append(k) - else: - yield element - - -def duplicates_justseen(iterable, key=None): - """Yields serially-duplicate elements after their first appearance. - - >>> list(duplicates_justseen('mississippi')) - ['s', 's', 'p'] - >>> list(duplicates_justseen('AaaBbbCccAaa', str.lower)) - ['a', 'a', 'b', 'b', 'c', 'c', 'a', 'a'] - - This function is analogous to :func:`unique_justseen`. - - """ - return flatten(g for _, g in groupby(iterable, key) for _ in g) - - -def classify_unique(iterable, key=None): - """Classify each element in terms of its uniqueness. - - For each element in the input iterable, return a 3-tuple consisting of: - - 1. The element itself - 2. ``False`` if the element is equal to the one preceding it in the input, - ``True`` otherwise (i.e. the equivalent of :func:`unique_justseen`) - 3. ``False`` if this element has been seen anywhere in the input before, - ``True`` otherwise (i.e. the equivalent of :func:`unique_everseen`) - - >>> list(classify_unique('otto')) # doctest: +NORMALIZE_WHITESPACE - [('o', True, True), - ('t', True, True), - ('t', False, False), - ('o', True, False)] - - This function is analogous to :func:`unique_everseen` and is subject to - the same performance considerations. - - """ - seen_set = set() - seen_list = [] - use_key = key is not None - previous = None - - for i, element in enumerate(iterable): - k = key(element) if use_key else element - is_unique_justseen = not i or previous != k - previous = k - is_unique_everseen = False - try: - if k not in seen_set: - seen_set.add(k) - is_unique_everseen = True - except TypeError: - if k not in seen_list: - seen_list.append(k) - is_unique_everseen = True - yield element, is_unique_justseen, is_unique_everseen - - -def minmax(iterable_or_value, *others, key=None, default=_marker): - """Returns both the smallest and largest items in an iterable - or the largest of two or more arguments. - - >>> minmax([3, 1, 5]) - (1, 5) - - >>> minmax(4, 2, 6) - (2, 6) - - If a *key* function is provided, it will be used to transform the input - items for comparison. - - >>> minmax([5, 30], key=str) # '30' sorts before '5' - (30, 5) - - If a *default* value is provided, it will be returned if there are no - input items. - - >>> minmax([], default=(0, 0)) - (0, 0) - - Otherwise ``ValueError`` is raised. - - This function is based on the - `recipe <http://code.activestate.com/recipes/577916/>`__ by - Raymond Hettinger and takes care to minimize the number of comparisons - performed. - """ - iterable = (iterable_or_value, *others) if others else iterable_or_value - - it = iter(iterable) - - try: - lo = hi = next(it) - except StopIteration as e: - if default is _marker: - raise ValueError( - '`minmax()` argument is an empty iterable. ' - 'Provide a `default` value to suppress this error.' - ) from e - return default - - # Different branches depending on the presence of key. This saves a lot - # of unimportant copies which would slow the "key=None" branch - # significantly down. - if key is None: - for x, y in zip_longest(it, it, fillvalue=lo): - if y < x: - x, y = y, x - if x < lo: - lo = x - if hi < y: - hi = y - - else: - lo_key = hi_key = key(lo) - - for x, y in zip_longest(it, it, fillvalue=lo): - x_key, y_key = key(x), key(y) - - if y_key < x_key: - x, y, x_key, y_key = y, x, y_key, x_key - if x_key < lo_key: - lo, lo_key = x, x_key - if hi_key < y_key: - hi, hi_key = y, y_key - - return lo, hi - - -def constrained_batches( - iterable, max_size, max_count=None, get_len=len, strict=True -): - """Yield batches of items from *iterable* with a combined size limited by - *max_size*. - - >>> iterable = [b'12345', b'123', b'12345678', b'1', b'1', b'12', b'1'] - >>> list(constrained_batches(iterable, 10)) - [(b'12345', b'123'), (b'12345678', b'1', b'1'), (b'12', b'1')] - - If a *max_count* is supplied, the number of items per batch is also - limited: - - >>> iterable = [b'12345', b'123', b'12345678', b'1', b'1', b'12', b'1'] - >>> list(constrained_batches(iterable, 10, max_count = 2)) - [(b'12345', b'123'), (b'12345678', b'1'), (b'1', b'12'), (b'1',)] - - If a *get_len* function is supplied, use that instead of :func:`len` to - determine item size. - - If *strict* is ``True``, raise ``ValueError`` if any single item is bigger - than *max_size*. Otherwise, allow single items to exceed *max_size*. - """ - if max_size <= 0: - raise ValueError('maximum size must be greater than zero') - - batch = [] - batch_size = 0 - batch_count = 0 - for item in iterable: - item_len = get_len(item) - if strict and item_len > max_size: - raise ValueError('item size exceeds maximum size') - - reached_count = batch_count == max_count - reached_size = item_len + batch_size > max_size - if batch_count and (reached_size or reached_count): - yield tuple(batch) - batch.clear() - batch_size = 0 - batch_count = 0 - - batch.append(item) - batch_size += item_len - batch_count += 1 - - if batch: - yield tuple(batch) - - -def gray_product(*iterables): - """Like :func:`itertools.product`, but return tuples in an order such - that only one element in the generated tuple changes from one iteration - to the next. - - >>> list(gray_product('AB','CD')) - [('A', 'C'), ('B', 'C'), ('B', 'D'), ('A', 'D')] - - This function consumes all of the input iterables before producing output. - If any of the input iterables have fewer than two items, ``ValueError`` - is raised. - - For information on the algorithm, see - `this section <https://www-cs-faculty.stanford.edu/~knuth/fasc2a.ps.gz>`__ - of Donald Knuth's *The Art of Computer Programming*. - """ - all_iterables = tuple(tuple(x) for x in iterables) - iterable_count = len(all_iterables) - for iterable in all_iterables: - if len(iterable) < 2: - raise ValueError("each iterable must have two or more items") - - # This is based on "Algorithm H" from section 7.2.1.1, page 20. - # a holds the indexes of the source iterables for the n-tuple to be yielded - # f is the array of "focus pointers" - # o is the array of "directions" - a = [0] * iterable_count - f = list(range(iterable_count + 1)) - o = [1] * iterable_count - while True: - yield tuple(all_iterables[i][a[i]] for i in range(iterable_count)) - j = f[0] - f[0] = 0 - if j == iterable_count: - break - a[j] = a[j] + o[j] - if a[j] == 0 or a[j] == len(all_iterables[j]) - 1: - o[j] = -o[j] - f[j] = f[j + 1] - f[j + 1] = j + 1 - - -def partial_product(*iterables): - """Yields tuples containing one item from each iterator, with subsequent - tuples changing a single item at a time by advancing each iterator until it - is exhausted. This sequence guarantees every value in each iterable is - output at least once without generating all possible combinations. - - This may be useful, for example, when testing an expensive function. - - >>> list(partial_product('AB', 'C', 'DEF')) - [('A', 'C', 'D'), ('B', 'C', 'D'), ('B', 'C', 'E'), ('B', 'C', 'F')] - """ - - iterators = list(map(iter, iterables)) - - try: - prod = [next(it) for it in iterators] - except StopIteration: - return - yield tuple(prod) - - for i, it in enumerate(iterators): - for prod[i] in it: - yield tuple(prod) - - -def takewhile_inclusive(predicate, iterable): - """A variant of :func:`takewhile` that yields one additional element. - - >>> list(takewhile_inclusive(lambda x: x < 5, [1, 4, 6, 4, 1])) - [1, 4, 6] - - :func:`takewhile` would return ``[1, 4]``. - """ - for x in iterable: - yield x - if not predicate(x): - break - - -def outer_product(func, xs, ys, *args, **kwargs): - """A generalized outer product that applies a binary function to all - pairs of items. Returns a 2D matrix with ``len(xs)`` rows and ``len(ys)`` - columns. - Also accepts ``*args`` and ``**kwargs`` that are passed to ``func``. - - Multiplication table: - - >>> list(outer_product(mul, range(1, 4), range(1, 6))) - [(1, 2, 3, 4, 5), (2, 4, 6, 8, 10), (3, 6, 9, 12, 15)] - - Cross tabulation: - - >>> xs = ['A', 'B', 'A', 'A', 'B', 'B', 'A', 'A', 'B', 'B'] - >>> ys = ['X', 'X', 'X', 'Y', 'Z', 'Z', 'Y', 'Y', 'Z', 'Z'] - >>> rows = list(zip(xs, ys)) - >>> count_rows = lambda x, y: rows.count((x, y)) - >>> list(outer_product(count_rows, sorted(set(xs)), sorted(set(ys)))) - [(2, 3, 0), (1, 0, 4)] - - Usage with ``*args`` and ``**kwargs``: - - >>> animals = ['cat', 'wolf', 'mouse'] - >>> list(outer_product(min, animals, animals, key=len)) - [('cat', 'cat', 'cat'), ('cat', 'wolf', 'wolf'), ('cat', 'wolf', 'mouse')] - """ - ys = tuple(ys) - return batched( - starmap(lambda x, y: func(x, y, *args, **kwargs), product(xs, ys)), - n=len(ys), - ) - - -def iter_suppress(iterable, *exceptions): - """Yield each of the items from *iterable*. If the iteration raises one of - the specified *exceptions*, that exception will be suppressed and iteration - will stop. - - >>> from itertools import chain - >>> def breaks_at_five(x): - ... while True: - ... if x >= 5: - ... raise RuntimeError - ... yield x - ... x += 1 - >>> it_1 = iter_suppress(breaks_at_five(1), RuntimeError) - >>> it_2 = iter_suppress(breaks_at_five(2), RuntimeError) - >>> list(chain(it_1, it_2)) - [1, 2, 3, 4, 2, 3, 4] - """ - try: - yield from iterable - except exceptions: - return - - -def filter_map(func, iterable): - """Apply *func* to every element of *iterable*, yielding only those which - are not ``None``. - - >>> elems = ['1', 'a', '2', 'b', '3'] - >>> list(filter_map(lambda s: int(s) if s.isnumeric() else None, elems)) - [1, 2, 3] - """ - for x in iterable: - y = func(x) - if y is not None: - yield y diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi deleted file mode 100644 index 9a5fc911a3e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi +++ /dev/null @@ -1,695 +0,0 @@ -"""Stubs for more_itertools.more""" -from __future__ import annotations - -from types import TracebackType -from typing import ( - Any, - Callable, - Container, - ContextManager, - Generic, - Hashable, - Iterable, - Iterator, - overload, - Reversible, - Sequence, - Sized, - Type, - TypeVar, - type_check_only, -) -from typing_extensions import Protocol - -# Type and type variable definitions -_T = TypeVar('_T') -_T1 = TypeVar('_T1') -_T2 = TypeVar('_T2') -_U = TypeVar('_U') -_V = TypeVar('_V') -_W = TypeVar('_W') -_T_co = TypeVar('_T_co', covariant=True) -_GenFn = TypeVar('_GenFn', bound=Callable[..., Iterator[Any]]) -_Raisable = BaseException | Type[BaseException] - -@type_check_only -class _SizedIterable(Protocol[_T_co], Sized, Iterable[_T_co]): ... - -@type_check_only -class _SizedReversible(Protocol[_T_co], Sized, Reversible[_T_co]): ... - -@type_check_only -class _SupportsSlicing(Protocol[_T_co]): - def __getitem__(self, __k: slice) -> _T_co: ... - -def chunked( - iterable: Iterable[_T], n: int | None, strict: bool = ... -) -> Iterator[list[_T]]: ... -@overload -def first(iterable: Iterable[_T]) -> _T: ... -@overload -def first(iterable: Iterable[_T], default: _U) -> _T | _U: ... -@overload -def last(iterable: Iterable[_T]) -> _T: ... -@overload -def last(iterable: Iterable[_T], default: _U) -> _T | _U: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int) -> _T: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int, default: _U) -> _T | _U: ... - -class peekable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> peekable[_T]: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> _T | _U: ... - def prepend(self, *items: _T) -> None: ... - def __next__(self) -> _T: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> list[_T]: ... - -def consumer(func: _GenFn) -> _GenFn: ... -def ilen(iterable: Iterable[_T]) -> int: ... -def iterate(func: Callable[[_T], _T], start: _T) -> Iterator[_T]: ... -def with_iter( - context_manager: ContextManager[Iterable[_T]], -) -> Iterator[_T]: ... -def one( - iterable: Iterable[_T], - too_short: _Raisable | None = ..., - too_long: _Raisable | None = ..., -) -> _T: ... -def raise_(exception: _Raisable, *args: Any) -> None: ... -def strictly_n( - iterable: Iterable[_T], - n: int, - too_short: _GenFn | None = ..., - too_long: _GenFn | None = ..., -) -> list[_T]: ... -def distinct_permutations( - iterable: Iterable[_T], r: int | None = ... -) -> Iterator[tuple[_T, ...]]: ... -def intersperse( - e: _U, iterable: Iterable[_T], n: int = ... -) -> Iterator[_T | _U]: ... -def unique_to_each(*iterables: Iterable[_T]) -> list[list[_T]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, *, step: int = ... -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, fillvalue: _U, step: int = ... -) -> Iterator[tuple[_T | _U, ...]]: ... -def substrings(iterable: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def substrings_indexes( - seq: Sequence[_T], reverse: bool = ... -) -> Iterator[tuple[Sequence[_T], int, int]]: ... - -class bucket(Generic[_T, _U], Container[_U]): - def __init__( - self, - iterable: Iterable[_T], - key: Callable[[_T], _U], - validator: Callable[[_U], object] | None = ..., - ) -> None: ... - def __contains__(self, value: object) -> bool: ... - def __iter__(self) -> Iterator[_U]: ... - def __getitem__(self, value: object) -> Iterator[_T]: ... - -def spy( - iterable: Iterable[_T], n: int = ... -) -> tuple[list[_T], Iterator[_T]]: ... -def interleave(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_longest(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_evenly( - iterables: list[Iterable[_T]], lengths: list[int] | None = ... -) -> Iterator[_T]: ... -def collapse( - iterable: Iterable[Any], - base_type: type | None = ..., - levels: int | None = ..., -) -> Iterator[Any]: ... -@overload -def side_effect( - func: Callable[[_T], object], - iterable: Iterable[_T], - chunk_size: None = ..., - before: Callable[[], object] | None = ..., - after: Callable[[], object] | None = ..., -) -> Iterator[_T]: ... -@overload -def side_effect( - func: Callable[[list[_T]], object], - iterable: Iterable[_T], - chunk_size: int, - before: Callable[[], object] | None = ..., - after: Callable[[], object] | None = ..., -) -> Iterator[_T]: ... -def sliced( - seq: _SupportsSlicing[_T], n: int, strict: bool = ... -) -> Iterator[_T]: ... -def split_at( - iterable: Iterable[_T], - pred: Callable[[_T], object], - maxsplit: int = ..., - keep_separator: bool = ..., -) -> Iterator[list[_T]]: ... -def split_before( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[list[_T]]: ... -def split_after( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[list[_T]]: ... -def split_when( - iterable: Iterable[_T], - pred: Callable[[_T, _T], object], - maxsplit: int = ..., -) -> Iterator[list[_T]]: ... -def split_into( - iterable: Iterable[_T], sizes: Iterable[int | None] -) -> Iterator[list[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - *, - n: int | None = ..., - next_multiple: bool = ..., -) -> Iterator[_T | None]: ... -@overload -def padded( - iterable: Iterable[_T], - fillvalue: _U, - n: int | None = ..., - next_multiple: bool = ..., -) -> Iterator[_T | _U]: ... -@overload -def repeat_last(iterable: Iterable[_T]) -> Iterator[_T]: ... -@overload -def repeat_last(iterable: Iterable[_T], default: _U) -> Iterator[_T | _U]: ... -def distribute(n: int, iterable: Iterable[_T]) -> list[Iterator[_T]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., - fillvalue: _U = ..., -) -> Iterator[tuple[_T | _U, ...]]: ... - -class UnequalIterablesError(ValueError): - def __init__(self, details: tuple[int, int, int] | None = ...) -> None: ... - -@overload -def zip_equal(__iter1: Iterable[_T1]) -> Iterator[tuple[_T1]]: ... -@overload -def zip_equal( - __iter1: Iterable[_T1], __iter2: Iterable[_T2] -) -> Iterator[tuple[_T1, _T2]]: ... -@overload -def zip_equal( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], -) -> Iterator[tuple[_T, ...]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T1 | None]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - __iter2: Iterable[_T2], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T1 | None, _T2 | None]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T1 | _U]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - __iter2: Iterable[_T2], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T1 | _U, _T2 | _U]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T | _U, ...]]: ... -def sort_together( - iterables: Iterable[Iterable[_T]], - key_list: Iterable[int] = ..., - key: Callable[..., Any] | None = ..., - reverse: bool = ..., -) -> list[tuple[_T, ...]]: ... -def unzip(iterable: Iterable[Sequence[_T]]) -> tuple[Iterator[_T], ...]: ... -def divide(n: int, iterable: Iterable[_T]) -> list[Iterator[_T]]: ... -def always_iterable( - obj: object, - base_type: type | tuple[type | tuple[Any, ...], ...] | None = ..., -) -> Iterator[Any]: ... -def adjacent( - predicate: Callable[[_T], bool], - iterable: Iterable[_T], - distance: int = ..., -) -> Iterator[tuple[bool, _T]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None = None, - valuefunc: None = None, - reducefunc: None = None, -) -> Iterator[tuple[_T, Iterator[_T]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None, - reducefunc: None, -) -> Iterator[tuple[_U, Iterator[_T]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: Callable[[_T], _V], - reducefunc: None, -) -> Iterable[tuple[_T, Iterable[_V]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None, -) -> Iterable[tuple[_U, Iterator[_V]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: None, - reducefunc: Callable[[Iterator[_T]], _W], -) -> Iterable[tuple[_T, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None, - reducefunc: Callable[[Iterator[_T]], _W], -) -> Iterable[tuple[_U, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[Iterable[_V]], _W], -) -> Iterable[tuple[_T, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[Iterable[_V]], _W], -) -> Iterable[tuple[_U, _W]]: ... - -class numeric_range(Generic[_T, _U], Sequence[_T], Hashable, Reversible[_T]): - @overload - def __init__(self, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T, __step: _U) -> None: ... - def __bool__(self) -> bool: ... - def __contains__(self, elem: object) -> bool: ... - def __eq__(self, other: object) -> bool: ... - @overload - def __getitem__(self, key: int) -> _T: ... - @overload - def __getitem__(self, key: slice) -> numeric_range[_T, _U]: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[_T]: ... - def __len__(self) -> int: ... - def __reduce__( - self, - ) -> tuple[Type[numeric_range[_T, _U]], tuple[_T, _T, _U]]: ... - def __repr__(self) -> str: ... - def __reversed__(self) -> Iterator[_T]: ... - def count(self, value: _T) -> int: ... - def index(self, value: _T) -> int: ... # type: ignore - -def count_cycle( - iterable: Iterable[_T], n: int | None = ... -) -> Iterable[tuple[int, _T]]: ... -def mark_ends( - iterable: Iterable[_T], -) -> Iterable[tuple[bool, bool, _T]]: ... -def locate( - iterable: Iterable[_T], - pred: Callable[..., Any] = ..., - window_size: int | None = ..., -) -> Iterator[int]: ... -def lstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def rstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def strip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... - -class islice_extended(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T], *args: int | None) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - def __getitem__(self, index: slice) -> islice_extended[_T]: ... - -def always_reversible(iterable: Iterable[_T]) -> Iterator[_T]: ... -def consecutive_groups( - iterable: Iterable[_T], ordering: Callable[[_T], int] = ... -) -> Iterator[Iterator[_T]]: ... -@overload -def difference( - iterable: Iterable[_T], - func: Callable[[_T, _T], _U] = ..., - *, - initial: None = ..., -) -> Iterator[_T | _U]: ... -@overload -def difference( - iterable: Iterable[_T], func: Callable[[_T, _T], _U] = ..., *, initial: _U -) -> Iterator[_U]: ... - -class SequenceView(Generic[_T], Sequence[_T]): - def __init__(self, target: Sequence[_T]) -> None: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> Sequence[_T]: ... - def __len__(self) -> int: ... - -class seekable(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], maxlen: int | None = ... - ) -> None: ... - def __iter__(self) -> seekable[_T]: ... - def __next__(self) -> _T: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> _T | _U: ... - def elements(self) -> SequenceView[_T]: ... - def seek(self, index: int) -> None: ... - def relative_seek(self, count: int) -> None: ... - -class run_length: - @staticmethod - def encode(iterable: Iterable[_T]) -> Iterator[tuple[_T, int]]: ... - @staticmethod - def decode(iterable: Iterable[tuple[_T, int]]) -> Iterator[_T]: ... - -def exactly_n( - iterable: Iterable[_T], n: int, predicate: Callable[[_T], object] = ... -) -> bool: ... -def circular_shifts(iterable: Iterable[_T]) -> list[tuple[_T, ...]]: ... -def make_decorator( - wrapping_func: Callable[..., _U], result_index: int = ... -) -> Callable[..., Callable[[Callable[..., Any]], Callable[..., _U]]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: None = ..., -) -> dict[_U, list[_T]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None = ..., -) -> dict[_U, list[_V]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: Callable[[list[_T]], _W] = ..., -) -> dict[_U, _W]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[list[_V]], _W], -) -> dict[_U, _W]: ... -def rlocate( - iterable: Iterable[_T], - pred: Callable[..., object] = ..., - window_size: int | None = ..., -) -> Iterator[int]: ... -def replace( - iterable: Iterable[_T], - pred: Callable[..., object], - substitutes: Iterable[_U], - count: int | None = ..., - window_size: int = ..., -) -> Iterator[_T | _U]: ... -def partitions(iterable: Iterable[_T]) -> Iterator[list[list[_T]]]: ... -def set_partitions( - iterable: Iterable[_T], k: int | None = ... -) -> Iterator[list[list[_T]]]: ... - -class time_limited(Generic[_T], Iterator[_T]): - def __init__( - self, limit_seconds: float, iterable: Iterable[_T] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - -@overload -def only( - iterable: Iterable[_T], *, too_long: _Raisable | None = ... -) -> _T | None: ... -@overload -def only( - iterable: Iterable[_T], default: _U, too_long: _Raisable | None = ... -) -> _T | _U: ... -def ichunked(iterable: Iterable[_T], n: int) -> Iterator[Iterator[_T]]: ... -def distinct_combinations( - iterable: Iterable[_T], r: int -) -> Iterator[tuple[_T, ...]]: ... -def filter_except( - validator: Callable[[Any], object], - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_T]: ... -def map_except( - function: Callable[[Any], _U], - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_U]: ... -def map_if( - iterable: Iterable[Any], - pred: Callable[[Any], bool], - func: Callable[[Any], Any], - func_else: Callable[[Any], Any] | None = ..., -) -> Iterator[Any]: ... -def sample( - iterable: Iterable[_T], - k: int, - weights: Iterable[float] | None = ..., -) -> list[_T]: ... -def is_sorted( - iterable: Iterable[_T], - key: Callable[[_T], _U] | None = ..., - reverse: bool = False, - strict: bool = False, -) -> bool: ... - -class AbortThread(BaseException): - pass - -class callback_iter(Generic[_T], Iterator[_T]): - def __init__( - self, - func: Callable[..., Any], - callback_kwd: str = ..., - wait_seconds: float = ..., - ) -> None: ... - def __enter__(self) -> callback_iter[_T]: ... - def __exit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool | None: ... - def __iter__(self) -> callback_iter[_T]: ... - def __next__(self) -> _T: ... - def _reader(self) -> Iterator[_T]: ... - @property - def done(self) -> bool: ... - @property - def result(self) -> Any: ... - -def windowed_complete( - iterable: Iterable[_T], n: int -) -> Iterator[tuple[_T, ...]]: ... -def all_unique( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> bool: ... -def nth_product(index: int, *args: Iterable[_T]) -> tuple[_T, ...]: ... -def nth_combination_with_replacement( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def nth_permutation( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def value_chain(*args: _T | Iterable[_T]) -> Iterable[_T]: ... -def product_index(element: Iterable[_T], *args: Iterable[_T]) -> int: ... -def combination_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def combination_with_replacement_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def permutation_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def repeat_each(iterable: Iterable[_T], n: int = ...) -> Iterator[_T]: ... - -class countable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> countable[_T]: ... - def __next__(self) -> _T: ... - -def chunked_even(iterable: Iterable[_T], n: int) -> Iterator[list[_T]]: ... -def zip_broadcast( - *objects: _T | Iterable[_T], - scalar_types: type | tuple[type | tuple[Any, ...], ...] | None = ..., - strict: bool = ..., -) -> Iterable[tuple[_T, ...]]: ... -def unique_in_window( - iterable: Iterable[_T], n: int, key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def duplicates_everseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def duplicates_justseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def classify_unique( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[tuple[_T, bool, bool]]: ... - -class _SupportsLessThan(Protocol): - def __lt__(self, __other: Any) -> bool: ... - -_SupportsLessThanT = TypeVar("_SupportsLessThanT", bound=_SupportsLessThan) - -@overload -def minmax( - iterable_or_value: Iterable[_SupportsLessThanT], *, key: None = None -) -> tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: Iterable[_T], *, key: Callable[[_T], _SupportsLessThan] -) -> tuple[_T, _T]: ... -@overload -def minmax( - iterable_or_value: Iterable[_SupportsLessThanT], - *, - key: None = None, - default: _U, -) -> _U | tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: Iterable[_T], - *, - key: Callable[[_T], _SupportsLessThan], - default: _U, -) -> _U | tuple[_T, _T]: ... -@overload -def minmax( - iterable_or_value: _SupportsLessThanT, - __other: _SupportsLessThanT, - *others: _SupportsLessThanT, -) -> tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: _T, - __other: _T, - *others: _T, - key: Callable[[_T], _SupportsLessThan], -) -> tuple[_T, _T]: ... -def longest_common_prefix( - iterables: Iterable[Iterable[_T]], -) -> Iterator[_T]: ... -def iequals(*iterables: Iterable[Any]) -> bool: ... -def constrained_batches( - iterable: Iterable[_T], - max_size: int, - max_count: int | None = ..., - get_len: Callable[[_T], object] = ..., - strict: bool = ..., -) -> Iterator[tuple[_T]]: ... -def gray_product(*iterables: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def partial_product(*iterables: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def takewhile_inclusive( - predicate: Callable[[_T], bool], iterable: Iterable[_T] -) -> Iterator[_T]: ... -def outer_product( - func: Callable[[_T, _U], _V], - xs: Iterable[_T], - ys: Iterable[_U], - *args: Any, - **kwargs: Any, -) -> Iterator[tuple[_V, ...]]: ... -def iter_suppress( - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_T]: ... -def filter_map( - func: Callable[[_T], _V | None], - iterable: Iterable[_T], -) -> Iterator[_V]: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/py.typed b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py deleted file mode 100644 index 145e3cb5bd6..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py +++ /dev/null @@ -1,1012 +0,0 @@ -"""Imported from the recipes section of the itertools documentation. - -All functions taken from the recipes section of the itertools library docs -[1]_. -Some backward-compatible usability improvements have been made. - -.. [1] http://docs.python.org/library/itertools.html#recipes - -""" -import math -import operator - -from collections import deque -from collections.abc import Sized -from functools import partial, reduce -from itertools import ( - chain, - combinations, - compress, - count, - cycle, - groupby, - islice, - product, - repeat, - starmap, - tee, - zip_longest, -) -from random import randrange, sample, choice -from sys import hexversion - -__all__ = [ - 'all_equal', - 'batched', - 'before_and_after', - 'consume', - 'convolve', - 'dotproduct', - 'first_true', - 'factor', - 'flatten', - 'grouper', - 'iter_except', - 'iter_index', - 'matmul', - 'ncycles', - 'nth', - 'nth_combination', - 'padnone', - 'pad_none', - 'pairwise', - 'partition', - 'polynomial_eval', - 'polynomial_from_roots', - 'polynomial_derivative', - 'powerset', - 'prepend', - 'quantify', - 'reshape', - 'random_combination_with_replacement', - 'random_combination', - 'random_permutation', - 'random_product', - 'repeatfunc', - 'roundrobin', - 'sieve', - 'sliding_window', - 'subslices', - 'sum_of_squares', - 'tabulate', - 'tail', - 'take', - 'totient', - 'transpose', - 'triplewise', - 'unique_everseen', - 'unique_justseen', -] - -_marker = object() - - -# zip with strict is available for Python 3.10+ -try: - zip(strict=True) -except TypeError: - _zip_strict = zip -else: - _zip_strict = partial(zip, strict=True) - -# math.sumprod is available for Python 3.12+ -_sumprod = getattr(math, 'sumprod', lambda x, y: dotproduct(x, y)) - - -def take(n, iterable): - """Return first *n* items of the iterable as a list. - - >>> take(3, range(10)) - [0, 1, 2] - - If there are fewer than *n* items in the iterable, all of them are - returned. - - >>> take(10, range(3)) - [0, 1, 2] - - """ - return list(islice(iterable, n)) - - -def tabulate(function, start=0): - """Return an iterator over the results of ``func(start)``, - ``func(start + 1)``, ``func(start + 2)``... - - *func* should be a function that accepts one integer argument. - - If *start* is not specified it defaults to 0. It will be incremented each - time the iterator is advanced. - - >>> square = lambda x: x ** 2 - >>> iterator = tabulate(square, -3) - >>> take(4, iterator) - [9, 4, 1, 0] - - """ - return map(function, count(start)) - - -def tail(n, iterable): - """Return an iterator over the last *n* items of *iterable*. - - >>> t = tail(3, 'ABCDEFG') - >>> list(t) - ['E', 'F', 'G'] - - """ - # If the given iterable has a length, then we can use islice to get its - # final elements. Note that if the iterable is not actually Iterable, - # either islice or deque will throw a TypeError. This is why we don't - # check if it is Iterable. - if isinstance(iterable, Sized): - yield from islice(iterable, max(0, len(iterable) - n), None) - else: - yield from iter(deque(iterable, maxlen=n)) - - -def consume(iterator, n=None): - """Advance *iterable* by *n* steps. If *n* is ``None``, consume it - entirely. - - Efficiently exhausts an iterator without returning values. Defaults to - consuming the whole iterator, but an optional second argument may be - provided to limit consumption. - - >>> i = (x for x in range(10)) - >>> next(i) - 0 - >>> consume(i, 3) - >>> next(i) - 4 - >>> consume(i) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - If the iterator has fewer items remaining than the provided limit, the - whole iterator will be consumed. - - >>> i = (x for x in range(3)) - >>> consume(i, 5) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - """ - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - -def nth(iterable, n, default=None): - """Returns the nth item or a default value. - - >>> l = range(10) - >>> nth(l, 3) - 3 - >>> nth(l, 20, "zebra") - 'zebra' - - """ - return next(islice(iterable, n, None), default) - - -def all_equal(iterable): - """ - Returns ``True`` if all the elements are equal to each other. - - >>> all_equal('aaaa') - True - >>> all_equal('aaab') - False - - """ - g = groupby(iterable) - return next(g, True) and not next(g, False) - - -def quantify(iterable, pred=bool): - """Return the how many times the predicate is true. - - >>> quantify([True, False, True]) - 2 - - """ - return sum(map(pred, iterable)) - - -def pad_none(iterable): - """Returns the sequence of elements and then returns ``None`` indefinitely. - - >>> take(5, pad_none(range(3))) - [0, 1, 2, None, None] - - Useful for emulating the behavior of the built-in :func:`map` function. - - See also :func:`padded`. - - """ - return chain(iterable, repeat(None)) - - -padnone = pad_none - - -def ncycles(iterable, n): - """Returns the sequence elements *n* times - - >>> list(ncycles(["a", "b"], 3)) - ['a', 'b', 'a', 'b', 'a', 'b'] - - """ - return chain.from_iterable(repeat(tuple(iterable), n)) - - -def dotproduct(vec1, vec2): - """Returns the dot product of the two iterables. - - >>> dotproduct([10, 10], [20, 20]) - 400 - - """ - return sum(map(operator.mul, vec1, vec2)) - - -def flatten(listOfLists): - """Return an iterator flattening one level of nesting in a list of lists. - - >>> list(flatten([[0, 1], [2, 3]])) - [0, 1, 2, 3] - - See also :func:`collapse`, which can flatten multiple levels of nesting. - - """ - return chain.from_iterable(listOfLists) - - -def repeatfunc(func, times=None, *args): - """Call *func* with *args* repeatedly, returning an iterable over the - results. - - If *times* is specified, the iterable will terminate after that many - repetitions: - - >>> from operator import add - >>> times = 4 - >>> args = 3, 5 - >>> list(repeatfunc(add, times, *args)) - [8, 8, 8, 8] - - If *times* is ``None`` the iterable will not terminate: - - >>> from random import randrange - >>> times = None - >>> args = 1, 11 - >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP - [2, 4, 8, 1, 8, 4] - - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - -def _pairwise(iterable): - """Returns an iterator of paired items, overlapping, from the original - - >>> take(4, pairwise(count())) - [(0, 1), (1, 2), (2, 3), (3, 4)] - - On Python 3.10 and above, this is an alias for :func:`itertools.pairwise`. - - """ - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - -try: - from itertools import pairwise as itertools_pairwise -except ImportError: - pairwise = _pairwise -else: - - def pairwise(iterable): - return itertools_pairwise(iterable) - - pairwise.__doc__ = _pairwise.__doc__ - - -class UnequalIterablesError(ValueError): - def __init__(self, details=None): - msg = 'Iterables have different lengths' - if details is not None: - msg += (': index 0 has length {}; index {} has length {}').format( - *details - ) - - super().__init__(msg) - - -def _zip_equal_generator(iterables): - for combo in zip_longest(*iterables, fillvalue=_marker): - for val in combo: - if val is _marker: - raise UnequalIterablesError() - yield combo - - -def _zip_equal(*iterables): - # Check whether the iterables are all the same size. - try: - first_size = len(iterables[0]) - for i, it in enumerate(iterables[1:], 1): - size = len(it) - if size != first_size: - raise UnequalIterablesError(details=(first_size, i, size)) - # All sizes are equal, we can use the built-in zip. - return zip(*iterables) - # If any one of the iterables didn't have a length, start reading - # them until one runs out. - except TypeError: - return _zip_equal_generator(iterables) - - -def grouper(iterable, n, incomplete='fill', fillvalue=None): - """Group elements from *iterable* into fixed-length groups of length *n*. - - >>> list(grouper('ABCDEF', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F')] - - The keyword arguments *incomplete* and *fillvalue* control what happens for - iterables whose length is not a multiple of *n*. - - When *incomplete* is `'fill'`, the last group will contain instances of - *fillvalue*. - - >>> list(grouper('ABCDEFG', 3, incomplete='fill', fillvalue='x')) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] - - When *incomplete* is `'ignore'`, the last group will not be emitted. - - >>> list(grouper('ABCDEFG', 3, incomplete='ignore', fillvalue='x')) - [('A', 'B', 'C'), ('D', 'E', 'F')] - - When *incomplete* is `'strict'`, a subclass of `ValueError` will be raised. - - >>> it = grouper('ABCDEFG', 3, incomplete='strict') - >>> list(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - UnequalIterablesError - - """ - args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': - return _zip_equal(*args) - if incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') - - -def roundrobin(*iterables): - """Yields an item from each iterable, alternating between them. - - >>> list(roundrobin('ABC', 'D', 'EF')) - ['A', 'D', 'E', 'B', 'F', 'C'] - - This function produces the same output as :func:`interleave_longest`, but - may perform better for some inputs (in particular when the number of - iterables is small). - - """ - # Recipe credited to George Sakkis - pending = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while pending: - try: - for next in nexts: - yield next() - except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) - - -def partition(pred, iterable): - """ - Returns a 2-tuple of iterables derived from the input iterable. - The first yields the items that have ``pred(item) == False``. - The second yields the items that have ``pred(item) == True``. - - >>> is_odd = lambda x: x % 2 != 0 - >>> iterable = range(10) - >>> even_items, odd_items = partition(is_odd, iterable) - >>> list(even_items), list(odd_items) - ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) - - If *pred* is None, :func:`bool` is used. - - >>> iterable = [0, 1, False, True, '', ' '] - >>> false_items, true_items = partition(None, iterable) - >>> list(false_items), list(true_items) - ([0, False, ''], [1, True, ' ']) - - """ - if pred is None: - pred = bool - - t1, t2, p = tee(iterable, 3) - p1, p2 = tee(map(pred, p)) - return (compress(t1, map(operator.not_, p1)), compress(t2, p2)) - - -def powerset(iterable): - """Yields all possible subsets of the iterable. - - >>> list(powerset([1, 2, 3])) - [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - - :func:`powerset` will operate on iterables that aren't :class:`set` - instances, so repeated elements in the input will produce repeated elements - in the output. Use :func:`unique_everseen` on the input to avoid generating - duplicates: - - >>> seq = [1, 1, 0] - >>> list(powerset(seq)) - [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] - >>> from more_itertools import unique_everseen - >>> list(powerset(unique_everseen(seq))) - [(), (1,), (0,), (1, 0)] - - """ - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) - - -def unique_everseen(iterable, key=None): - """ - Yield unique elements, preserving order. - - >>> list(unique_everseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'D'] - - Sequences with a mix of hashable and unhashable items can be used. - The function will be slower (i.e., `O(n^2)`) for unhashable items. - - Remember that ``list`` objects are unhashable - you can use the *key* - parameter to transform the list to a tuple (which is hashable) to - avoid a slowdown. - - >>> iterable = ([1, 2], [2, 3], [1, 2]) - >>> list(unique_everseen(iterable)) # Slow - [[1, 2], [2, 3]] - >>> list(unique_everseen(iterable, key=tuple)) # Faster - [[1, 2], [2, 3]] - - Similarly, you may want to convert unhashable ``set`` objects with - ``key=frozenset``. For ``dict`` objects, - ``key=lambda x: frozenset(x.items())`` can be used. - - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seenset: - seenset_add(k) - yield element - except TypeError: - if k not in seenlist: - seenlist_add(k) - yield element - - -def unique_justseen(iterable, key=None): - """Yields elements in order, ignoring serial duplicates - - >>> list(unique_justseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'A', 'D'] - - """ - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - -def iter_except(func, exception, first=None): - """Yields results from a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel - to end the loop. - - >>> l = [0, 1, 2] - >>> list(iter_except(l.pop, IndexError)) - [2, 1, 0] - - Multiple exceptions can be specified as a stopping condition: - - >>> l = [1, 2, 3, '...', 4, 5, 6] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [7, 6, 5] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [4, 3, 2] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [] - - """ - try: - if first is not None: - yield first() - while 1: - yield func() - except exception: - pass - - -def first_true(iterable, default=None, pred=None): - """ - Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item for which - ``pred(item) == True`` . - - >>> first_true(range(10)) - 1 - >>> first_true(range(10), pred=lambda x: x > 5) - 6 - >>> first_true(range(10), default='missing', pred=lambda x: x > 9) - 'missing' - - """ - return next(filter(pred, iterable), default) - - -def random_product(*args, repeat=1): - """Draw an item at random from each of the input iterables. - - >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP - ('c', 3, 'Z') - - If *repeat* is provided as a keyword argument, that many items will be - drawn from each iterable. - - >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP - ('a', 2, 'd', 3) - - This equivalent to taking a random selection from - ``itertools.product(*args, **kwarg)``. - - """ - pools = [tuple(pool) for pool in args] * repeat - return tuple(choice(pool) for pool in pools) - - -def random_permutation(iterable, r=None): - """Return a random *r* length permutation of the elements in *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length of - *iterable*. - - >>> random_permutation(range(5)) # doctest:+SKIP - (3, 4, 0, 1, 2) - - This equivalent to taking a random selection from - ``itertools.permutations(iterable, r)``. - - """ - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(sample(pool, r)) - - -def random_combination(iterable, r): - """Return a random *r* length subsequence of the elements in *iterable*. - - >>> random_combination(range(5), 3) # doctest:+SKIP - (2, 3, 4) - - This equivalent to taking a random selection from - ``itertools.combinations(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(sample(range(n), r)) - return tuple(pool[i] for i in indices) - - -def random_combination_with_replacement(iterable, r): - """Return a random *r* length subsequence of elements in *iterable*, - allowing individual elements to be repeated. - - >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP - (0, 0, 1, 2, 2) - - This equivalent to taking a random selection from - ``itertools.combinations_with_replacement(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(randrange(n) for i in range(r)) - return tuple(pool[i] for i in indices) - - -def nth_combination(iterable, r, index): - """Equivalent to ``list(combinations(iterable, r))[index]``. - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`nth_combination` computes the subsequence at - sort position *index* directly, without computing the previous - subsequences. - - >>> nth_combination(range(5), 3, 5) - (0, 3, 4) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = 1 - k = min(r, n - r) - for i in range(1, k + 1): - c = c * (n - k + i) // i - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - while r: - c, n, r = c * r // n, n - 1, r - 1 - while index >= c: - index -= c - c, n = c * (n - r) // n, n - 1 - result.append(pool[-1 - n]) - - return tuple(result) - - -def prepend(value, iterator): - """Yield *value*, followed by the elements in *iterator*. - - >>> value = '0' - >>> iterator = ['1', '2', '3'] - >>> list(prepend(value, iterator)) - ['0', '1', '2', '3'] - - To prepend multiple values, see :func:`itertools.chain` - or :func:`value_chain`. - - """ - return chain([value], iterator) - - -def convolve(signal, kernel): - """Convolve the iterable *signal* with the iterable *kernel*. - - >>> signal = (1, 2, 3, 4, 5) - >>> kernel = [3, 2, 1] - >>> list(convolve(signal, kernel)) - [3, 8, 14, 20, 26, 14, 5] - - Note: the input arguments are not interchangeable, as the *kernel* - is immediately consumed and stored. - - """ - # This implementation intentionally doesn't match the one in the itertools - # documentation. - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n - 1)): - window.append(x) - yield _sumprod(kernel, window) - - -def before_and_after(predicate, it): - """A variant of :func:`takewhile` that allows complete access to the - remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the first iterator must be fully consumed before the second - iterator can generate valid results. - """ - it = iter(it) - transition = [] - - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - - # Note: this is different from itertools recipes to allow nesting - # before_and_after remainders into before_and_after again. See tests - # for an example. - remainder_iterator = chain(transition, it) - - return true_iterator(), remainder_iterator - - -def triplewise(iterable): - """Return overlapping triplets from *iterable*. - - >>> list(triplewise('ABCDE')) - [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E')] - - """ - for (a, _), (b, c) in pairwise(pairwise(iterable)): - yield a, b, c - - -def sliding_window(iterable, n): - """Return a sliding window of width *n* over *iterable*. - - >>> list(sliding_window(range(6), 4)) - [(0, 1, 2, 3), (1, 2, 3, 4), (2, 3, 4, 5)] - - If *iterable* has fewer than *n* items, then nothing is yielded: - - >>> list(sliding_window(range(3), 4)) - [] - - For a variant with more features, see :func:`windowed`. - """ - it = iter(iterable) - window = deque(islice(it, n - 1), maxlen=n) - for x in it: - window.append(x) - yield tuple(window) - - -def subslices(iterable): - """Return all contiguous non-empty subslices of *iterable*. - - >>> list(subslices('ABC')) - [['A'], ['A', 'B'], ['A', 'B', 'C'], ['B'], ['B', 'C'], ['C']] - - This is similar to :func:`substrings`, but emits items in a different - order. - """ - seq = list(iterable) - slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) - - -def polynomial_from_roots(roots): - """Compute a polynomial's coefficients from its roots. - - >>> roots = [5, -4, 3] # (x - 5) * (x + 4) * (x - 3) - >>> polynomial_from_roots(roots) # x^3 - 4 * x^2 - 17 * x + 60 - [1, -4, -17, 60] - """ - factors = zip(repeat(1), map(operator.neg, roots)) - return list(reduce(convolve, factors, [1])) - - -def iter_index(iterable, value, start=0, stop=None): - """Yield the index of each place in *iterable* that *value* occurs, - beginning with index *start* and ending before index *stop*. - - See :func:`locate` for a more general means of finding the indexes - associated with particular values. - - >>> list(iter_index('AABCADEAF', 'A')) - [0, 1, 4, 7] - >>> list(iter_index('AABCADEAF', 'A', 1)) # start index is inclusive - [1, 4, 7] - >>> list(iter_index('AABCADEAF', 'A', 1, 7)) # stop index is not inclusive - [1, 4] - """ - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Slow path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i - else: - # Fast path for sequences - stop = len(iterable) if stop is None else stop - i = start - 1 - try: - while True: - yield (i := seq_index(value, i + 1, stop)) - except ValueError: - pass - - -def sieve(n): - """Yield the primes less than n. - - >>> list(sieve(30)) - [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] - """ - if n > 2: - yield 2 - start = 3 - data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p * p) - data[p * p : n : p + p] = bytes(len(range(p * p, n, p + p))) - start = p * p - yield from iter_index(data, 1, start) - - -def _batched(iterable, n, *, strict=False): - """Batch data into tuples of length *n*. If the number of items in - *iterable* is not divisible by *n*: - * The last batch will be shorter if *strict* is ``False``. - * :exc:`ValueError` will be raised if *strict* is ``True``. - - >>> list(batched('ABCDEFG', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] - - On Python 3.13 and above, this is an alias for :func:`itertools.batched`. - """ - if n < 1: - raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): - if strict and len(batch) != n: - raise ValueError('batched(): incomplete batch') - yield batch - - -if hexversion >= 0x30D00A2: - from itertools import batched as itertools_batched - - def batched(iterable, n, *, strict=False): - return itertools_batched(iterable, n, strict=strict) - -else: - batched = _batched - - batched.__doc__ = _batched.__doc__ - - -def transpose(it): - """Swap the rows and columns of the input matrix. - - >>> list(transpose([(1, 2, 3), (11, 22, 33)])) - [(1, 11), (2, 22), (3, 33)] - - The caller should ensure that the dimensions of the input are compatible. - If the input is empty, no output will be produced. - """ - return _zip_strict(*it) - - -def reshape(matrix, cols): - """Reshape the 2-D input *matrix* to have a column count given by *cols*. - - >>> matrix = [(0, 1), (2, 3), (4, 5)] - >>> cols = 3 - >>> list(reshape(matrix, cols)) - [(0, 1, 2), (3, 4, 5)] - """ - return batched(chain.from_iterable(matrix), cols) - - -def matmul(m1, m2): - """Multiply two matrices. - - >>> list(matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)])) - [(49, 80), (41, 60)] - - The caller should ensure that the dimensions of the input matrices are - compatible with each other. - """ - n = len(m2[0]) - return batched(starmap(_sumprod, product(m1, transpose(m2))), n) - - -def factor(n): - """Yield the prime factors of n. - - >>> list(factor(360)) - [2, 2, 2, 3, 3, 5] - """ - for prime in sieve(math.isqrt(n) + 1): - while not n % prime: - yield prime - n //= prime - if n == 1: - return - if n > 1: - yield n - - -def polynomial_eval(coefficients, x): - """Evaluate a polynomial at a specific value. - - Example: evaluating x^3 - 4 * x^2 - 17 * x + 60 at x = 2.5: - - >>> coefficients = [1, -4, -17, 60] - >>> x = 2.5 - >>> polynomial_eval(coefficients, x) - 8.125 - """ - n = len(coefficients) - if n == 0: - return x * 0 # coerce zero to the type of x - powers = map(pow, repeat(x), reversed(range(n))) - return _sumprod(coefficients, powers) - - -def sum_of_squares(it): - """Return the sum of the squares of the input values. - - >>> sum_of_squares([10, 20, 30]) - 1400 - """ - return _sumprod(*tee(it)) - - -def polynomial_derivative(coefficients): - """Compute the first derivative of a polynomial. - - Example: evaluating the derivative of x^3 - 4 * x^2 - 17 * x + 60 - - >>> coefficients = [1, -4, -17, 60] - >>> derivative_coefficients = polynomial_derivative(coefficients) - >>> derivative_coefficients - [3, -8, -17] - """ - n = len(coefficients) - powers = reversed(range(1, n)) - return list(map(operator.mul, coefficients, powers)) - - -def totient(n): - """Return the count of natural numbers up to *n* that are coprime with *n*. - - >>> totient(9) - 6 - >>> totient(12) - 4 - """ - for p in unique_justseen(factor(n)): - n = n // p * (p - 1) - - return n diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi deleted file mode 100644 index ed4c19db49b..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi +++ /dev/null @@ -1,128 +0,0 @@ -"""Stubs for more_itertools.recipes""" -from __future__ import annotations - -from typing import ( - Any, - Callable, - Iterable, - Iterator, - overload, - Sequence, - Type, - TypeVar, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_T1 = TypeVar('_T1') -_T2 = TypeVar('_T2') -_U = TypeVar('_U') - -def take(n: int, iterable: Iterable[_T]) -> list[_T]: ... -def tabulate( - function: Callable[[int], _T], start: int = ... -) -> Iterator[_T]: ... -def tail(n: int, iterable: Iterable[_T]) -> Iterator[_T]: ... -def consume(iterator: Iterable[_T], n: int | None = ...) -> None: ... -@overload -def nth(iterable: Iterable[_T], n: int) -> _T | None: ... -@overload -def nth(iterable: Iterable[_T], n: int, default: _U) -> _T | _U: ... -def all_equal(iterable: Iterable[_T]) -> bool: ... -def quantify( - iterable: Iterable[_T], pred: Callable[[_T], bool] = ... -) -> int: ... -def pad_none(iterable: Iterable[_T]) -> Iterator[_T | None]: ... -def padnone(iterable: Iterable[_T]) -> Iterator[_T | None]: ... -def ncycles(iterable: Iterable[_T], n: int) -> Iterator[_T]: ... -def dotproduct(vec1: Iterable[_T1], vec2: Iterable[_T2]) -> Any: ... -def flatten(listOfLists: Iterable[Iterable[_T]]) -> Iterator[_T]: ... -def repeatfunc( - func: Callable[..., _U], times: int | None = ..., *args: Any -) -> Iterator[_U]: ... -def pairwise(iterable: Iterable[_T]) -> Iterator[tuple[_T, _T]]: ... -def grouper( - iterable: Iterable[_T], - n: int, - incomplete: str = ..., - fillvalue: _U = ..., -) -> Iterator[tuple[_T | _U, ...]]: ... -def roundrobin(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def partition( - pred: Callable[[_T], object] | None, iterable: Iterable[_T] -) -> tuple[Iterator[_T], Iterator[_T]]: ... -def powerset(iterable: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def unique_everseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def unique_justseen( - iterable: Iterable[_T], key: Callable[[_T], object] | None = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException] | tuple[Type[BaseException], ...], - first: None = ..., -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException] | tuple[Type[BaseException], ...], - first: Callable[[], _U], -) -> Iterator[_T | _U]: ... -@overload -def first_true( - iterable: Iterable[_T], *, pred: Callable[[_T], object] | None = ... -) -> _T | None: ... -@overload -def first_true( - iterable: Iterable[_T], - default: _U, - pred: Callable[[_T], object] | None = ..., -) -> _T | _U: ... -def random_product( - *args: Iterable[_T], repeat: int = ... -) -> tuple[_T, ...]: ... -def random_permutation( - iterable: Iterable[_T], r: int | None = ... -) -> tuple[_T, ...]: ... -def random_combination(iterable: Iterable[_T], r: int) -> tuple[_T, ...]: ... -def random_combination_with_replacement( - iterable: Iterable[_T], r: int -) -> tuple[_T, ...]: ... -def nth_combination( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def prepend(value: _T, iterator: Iterable[_U]) -> Iterator[_T | _U]: ... -def convolve(signal: Iterable[_T], kernel: Iterable[_T]) -> Iterator[_T]: ... -def before_and_after( - predicate: Callable[[_T], bool], it: Iterable[_T] -) -> tuple[Iterator[_T], Iterator[_T]]: ... -def triplewise(iterable: Iterable[_T]) -> Iterator[tuple[_T, _T, _T]]: ... -def sliding_window( - iterable: Iterable[_T], n: int -) -> Iterator[tuple[_T, ...]]: ... -def subslices(iterable: Iterable[_T]) -> Iterator[list[_T]]: ... -def polynomial_from_roots(roots: Sequence[_T]) -> list[_T]: ... -def iter_index( - iterable: Iterable[_T], - value: Any, - start: int | None = ..., - stop: int | None = ..., -) -> Iterator[int]: ... -def sieve(n: int) -> Iterator[int]: ... -def batched( - iterable: Iterable[_T], n: int, *, strict: bool = False -) -> Iterator[tuple[_T]]: ... -def transpose( - it: Iterable[Iterable[_T]], -) -> Iterator[tuple[_T, ...]]: ... -def reshape( - matrix: Iterable[Iterable[_T]], cols: int -) -> Iterator[tuple[_T, ...]]: ... -def matmul(m1: Sequence[_T], m2: Sequence[_T]) -> Iterator[tuple[_T]]: ... -def factor(n: int) -> Iterator[int]: ... -def polynomial_eval(coefficients: Sequence[_T], x: _U) -> _U: ... -def sum_of_squares(it: Iterable[_T]) -> _T: ... -def polynomial_derivative(coefficients: Sequence[_T]) -> list[_T]: ... -def totient(n: int) -> int: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py deleted file mode 100644 index e7c0aa12ca9..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -__title__ = "packaging" -__summary__ = "Core utilities for Python packages" -__uri__ = "https://github.com/pypa/packaging" - -__version__ = "24.0" - -__author__ = "Donald Stufft and individual contributors" -__email__ = "[email protected]" - -__license__ = "BSD-2-Clause or Apache-2.0" -__copyright__ = "2014 %s" % __author__ diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_elffile.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_elffile.py deleted file mode 100644 index 6fb19b30bb5..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_elffile.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -ELF file parser. - -This provides a class ``ELFFile`` that parses an ELF executable in a similar -interface to ``ZipFile``. Only the read interface is implemented. - -Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca -ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html -""" - -import enum -import os -import struct -from typing import IO, Optional, Tuple - - -class ELFInvalid(ValueError): - pass - - -class EIClass(enum.IntEnum): - C32 = 1 - C64 = 2 - - -class EIData(enum.IntEnum): - Lsb = 1 - Msb = 2 - - -class EMachine(enum.IntEnum): - I386 = 3 - S390 = 22 - Arm = 40 - X8664 = 62 - AArc64 = 183 - - -class ELFFile: - """ - Representation of an ELF executable. - """ - - def __init__(self, f: IO[bytes]) -> None: - self._f = f - - try: - ident = self._read("16B") - except struct.error: - raise ELFInvalid("unable to parse identification") - magic = bytes(ident[:4]) - if magic != b"\x7fELF": - raise ELFInvalid(f"invalid magic: {magic!r}") - - self.capacity = ident[4] # Format for program header (bitness). - self.encoding = ident[5] # Data structure encoding (endianness). - - try: - # e_fmt: Format for program header. - # p_fmt: Format for section header. - # p_idx: Indexes to find p_type, p_offset, and p_filesz. - e_fmt, self._p_fmt, self._p_idx = { - (1, 1): ("<HHIIIIIHHH", "<IIIIIIII", (0, 1, 4)), # 32-bit LSB. - (1, 2): (">HHIIIIIHHH", ">IIIIIIII", (0, 1, 4)), # 32-bit MSB. - (2, 1): ("<HHIQQQIHHH", "<IIQQQQQQ", (0, 2, 5)), # 64-bit LSB. - (2, 2): (">HHIQQQIHHH", ">IIQQQQQQ", (0, 2, 5)), # 64-bit MSB. - }[(self.capacity, self.encoding)] - except KeyError: - raise ELFInvalid( - f"unrecognized capacity ({self.capacity}) or " - f"encoding ({self.encoding})" - ) - - try: - ( - _, - self.machine, # Architecture type. - _, - _, - self._e_phoff, # Offset of program header. - _, - self.flags, # Processor-specific flags. - _, - self._e_phentsize, # Size of section. - self._e_phnum, # Number of sections. - ) = self._read(e_fmt) - except struct.error as e: - raise ELFInvalid("unable to parse machine and section information") from e - - def _read(self, fmt: str) -> Tuple[int, ...]: - return struct.unpack(fmt, self._f.read(struct.calcsize(fmt))) - - @property - def interpreter(self) -> Optional[str]: - """ - The path recorded in the ``PT_INTERP`` section header. - """ - for index in range(self._e_phnum): - self._f.seek(self._e_phoff + self._e_phentsize * index) - try: - data = self._read(self._p_fmt) - except struct.error: - continue - if data[self._p_idx[0]] != 3: # Not PT_INTERP. - continue - self._f.seek(data[self._p_idx[1]]) - return os.fsdecode(self._f.read(data[self._p_idx[2]])).strip("\0") - return None diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py deleted file mode 100644 index ad62505f3ff..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py +++ /dev/null @@ -1,260 +0,0 @@ -import collections -import contextlib -import functools -import os -import re -import sys -import warnings -from typing import Dict, Generator, Iterator, NamedTuple, Optional, Sequence, Tuple - -from ._elffile import EIClass, EIData, ELFFile, EMachine - -EF_ARM_ABIMASK = 0xFF000000 -EF_ARM_ABI_VER5 = 0x05000000 -EF_ARM_ABI_FLOAT_HARD = 0x00000400 - - -# `os.PathLike` not a generic type until Python 3.9, so sticking with `str` -# as the type for `path` until then. -def _parse_elf(path: str) -> Generator[Optional[ELFFile], None, None]: - try: - with open(path, "rb") as f: - yield ELFFile(f) - except (OSError, TypeError, ValueError): - yield None - - -def _is_linux_armhf(executable: str) -> bool: - # hard-float ABI can be detected from the ELF header of the running - # process - # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.Arm - and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5 - and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD - ) - - -def _is_linux_i686(executable: str) -> bool: - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.I386 - ) - - -def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: - if "armv7l" in archs: - return _is_linux_armhf(executable) - if "i686" in archs: - return _is_linux_i686(executable) - allowed_archs = { - "x86_64", - "aarch64", - "ppc64", - "ppc64le", - "s390x", - "loongarch64", - "riscv64", - } - return any(arch in allowed_archs for arch in archs) - - -# If glibc ever changes its major version, we need to know what the last -# minor version was, so we can build the complete list of all versions. -# For now, guess what the highest minor version might be, assume it will -# be 50 for testing. Once this actually happens, update the dictionary -# with the actual value. -_LAST_GLIBC_MINOR: Dict[int, int] = collections.defaultdict(lambda: 50) - - -class _GLibCVersion(NamedTuple): - major: int - minor: int - - -def _glibc_version_string_confstr() -> Optional[str]: - """ - Primary implementation of glibc_version_string using os.confstr. - """ - # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely - # to be broken or missing. This strategy is used in the standard library - # platform module. - # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183 - try: - # Should be a string like "glibc 2.17". - version_string: Optional[str] = os.confstr("CS_GNU_LIBC_VERSION") - assert version_string is not None - _, version = version_string.rsplit() - except (AssertionError, AttributeError, OSError, ValueError): - # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... - return None - return version - - -def _glibc_version_string_ctypes() -> Optional[str]: - """ - Fallback implementation of glibc_version_string using ctypes. - """ - try: - import ctypes - except ImportError: - return None - - # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen - # manpage says, "If filename is NULL, then the returned handle is for the - # main program". This way we can let the linker do the work to figure out - # which libc our process is actually using. - # - # We must also handle the special case where the executable is not a - # dynamically linked executable. This can occur when using musl libc, - # for example. In this situation, dlopen() will error, leading to an - # OSError. Interestingly, at least in the case of musl, there is no - # errno set on the OSError. The single string argument used to construct - # OSError comes from libc itself and is therefore not portable to - # hard code here. In any case, failure to call dlopen() means we - # can proceed, so we bail on our attempt. - try: - process_namespace = ctypes.CDLL(None) - except OSError: - return None - - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return None - - # Call gnu_get_libc_version, which returns a string like "2.5" - gnu_get_libc_version.restype = ctypes.c_char_p - version_str: str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") - - return version_str - - -def _glibc_version_string() -> Optional[str]: - """Returns glibc version string, or None if not using glibc.""" - return _glibc_version_string_confstr() or _glibc_version_string_ctypes() - - -def _parse_glibc_version(version_str: str) -> Tuple[int, int]: - """Parse glibc version. - - We use a regexp instead of str.split because we want to discard any - random junk that might come after the minor version -- this might happen - in patched/forked versions of glibc (e.g. Linaro's version of glibc - uses version strings like "2.20-2014.11"). See gh-3588. - """ - m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) - if not m: - warnings.warn( - f"Expected glibc version with 2 components major.minor," - f" got: {version_str}", - RuntimeWarning, - ) - return -1, -1 - return int(m.group("major")), int(m.group("minor")) - - [email protected]_cache() -def _get_glibc_version() -> Tuple[int, int]: - version_str = _glibc_version_string() - if version_str is None: - return (-1, -1) - return _parse_glibc_version(version_str) - - -# From PEP 513, PEP 600 -def _is_compatible(arch: str, version: _GLibCVersion) -> bool: - sys_glibc = _get_glibc_version() - if sys_glibc < version: - return False - # Check for presence of _manylinux module. - try: - import _manylinux - except ImportError: - return True - if hasattr(_manylinux, "manylinux_compatible"): - result = _manylinux.manylinux_compatible(version[0], version[1], arch) - if result is not None: - return bool(result) - return True - if version == _GLibCVersion(2, 5): - if hasattr(_manylinux, "manylinux1_compatible"): - return bool(_manylinux.manylinux1_compatible) - if version == _GLibCVersion(2, 12): - if hasattr(_manylinux, "manylinux2010_compatible"): - return bool(_manylinux.manylinux2010_compatible) - if version == _GLibCVersion(2, 17): - if hasattr(_manylinux, "manylinux2014_compatible"): - return bool(_manylinux.manylinux2014_compatible) - return True - - -_LEGACY_MANYLINUX_MAP = { - # CentOS 7 w/ glibc 2.17 (PEP 599) - (2, 17): "manylinux2014", - # CentOS 6 w/ glibc 2.12 (PEP 571) - (2, 12): "manylinux2010", - # CentOS 5 w/ glibc 2.5 (PEP 513) - (2, 5): "manylinux1", -} - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate manylinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be manylinux-compatible. - - :returns: An iterator of compatible manylinux tags. - """ - if not _have_compatible_abi(sys.executable, archs): - return - # Oldest glibc to be supported regardless of architecture is (2, 17). - too_old_glibc2 = _GLibCVersion(2, 16) - if set(archs) & {"x86_64", "i686"}: - # On x86/i686 also oldest glibc to be supported is (2, 5). - too_old_glibc2 = _GLibCVersion(2, 4) - current_glibc = _GLibCVersion(*_get_glibc_version()) - glibc_max_list = [current_glibc] - # We can assume compatibility across glibc major versions. - # https://sourceware.org/bugzilla/show_bug.cgi?id=24636 - # - # Build a list of maximum glibc versions so that we can - # output the canonical list of all glibc from current_glibc - # down to too_old_glibc2, including all intermediary versions. - for glibc_major in range(current_glibc.major - 1, 1, -1): - glibc_minor = _LAST_GLIBC_MINOR[glibc_major] - glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor)) - for arch in archs: - for glibc_max in glibc_max_list: - if glibc_max.major == too_old_glibc2.major: - min_minor = too_old_glibc2.minor - else: - # For other glibc major versions oldest supported is (x, 0). - min_minor = -1 - for glibc_minor in range(glibc_max.minor, min_minor, -1): - glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) - tag = "manylinux_{}_{}".format(*glibc_version) - if _is_compatible(arch, glibc_version): - yield f"{tag}_{arch}" - # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. - if glibc_version in _LEGACY_MANYLINUX_MAP: - legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] - if _is_compatible(arch, glibc_version): - yield f"{legacy_tag}_{arch}" diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_musllinux.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_musllinux.py deleted file mode 100644 index 86419df9d70..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_musllinux.py +++ /dev/null @@ -1,83 +0,0 @@ -"""PEP 656 support. - -This module implements logic to detect if the currently running Python is -linked against musl, and what musl version is used. -""" - -import functools -import re -import subprocess -import sys -from typing import Iterator, NamedTuple, Optional, Sequence - -from ._elffile import ELFFile - - -class _MuslVersion(NamedTuple): - major: int - minor: int - - -def _parse_musl_version(output: str) -> Optional[_MuslVersion]: - lines = [n for n in (n.strip() for n in output.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) - - [email protected]_cache() -def _get_musl_version(executable: str) -> Optional[_MuslVersion]: - """Detect currently-running musl runtime version. - - This is done by checking the specified executable's dynamic linking - information, and invoking the loader to parse its output for a version - string. If the loader is musl, the output would be something like:: - - musl libc (x86_64) - Version 1.2.2 - Dynamic Program Loader - """ - try: - with open(executable, "rb") as f: - ld = ELFFile(f).interpreter - except (OSError, TypeError, ValueError): - return None - if ld is None or "musl" not in ld: - return None - proc = subprocess.run([ld], stderr=subprocess.PIPE, text=True) - return _parse_musl_version(proc.stderr) - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate musllinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be musllinux-compatible. - - :returns: An iterator of compatible musllinux tags. - """ - sys_musl = _get_musl_version(sys.executable) - if sys_musl is None: # Python not dynamically linked against musl. - return - for arch in archs: - for minor in range(sys_musl.minor, -1, -1): - yield f"musllinux_{sys_musl.major}_{minor}_{arch}" - - -if __name__ == "__main__": # pragma: no cover - import sysconfig - - plat = sysconfig.get_platform() - assert plat.startswith("linux-"), "not linux" - - print("plat:", plat) - print("musl:", _get_musl_version(sys.executable)) - print("tags:", end=" ") - for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])): - print(t, end="\n ") diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py deleted file mode 100644 index 684df75457c..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py +++ /dev/null @@ -1,356 +0,0 @@ -"""Handwritten parser of dependency specifiers. - -The docstring for each __parse_* function contains ENBF-inspired grammar representing -the implementation. -""" - -import ast -from typing import Any, List, NamedTuple, Optional, Tuple, Union - -from ._tokenizer import DEFAULT_RULES, Tokenizer - - -class Node: - def __init__(self, value: str) -> None: - self.value = value - - def __str__(self) -> str: - return self.value - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" - - def serialize(self) -> str: - raise NotImplementedError - - -class Variable(Node): - def serialize(self) -> str: - return str(self) - - -class Value(Node): - def serialize(self) -> str: - return f'"{self}"' - - -class Op(Node): - def serialize(self) -> str: - return str(self) - - -MarkerVar = Union[Variable, Value] -MarkerItem = Tuple[MarkerVar, Op, MarkerVar] -# MarkerAtom = Union[MarkerItem, List["MarkerAtom"]] -# MarkerList = List[Union["MarkerList", MarkerAtom, str]] -# mypy does not support recursive type definition -# https://github.com/python/mypy/issues/731 -MarkerAtom = Any -MarkerList = List[Any] - - -class ParsedRequirement(NamedTuple): - name: str - url: str - extras: List[str] - specifier: str - marker: Optional[MarkerList] - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for dependency specifier -# -------------------------------------------------------------------------------------- -def parse_requirement(source: str) -> ParsedRequirement: - return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: - """ - requirement = WS? IDENTIFIER WS? extras WS? requirement_details - """ - tokenizer.consume("WS") - - name_token = tokenizer.expect( - "IDENTIFIER", expected="package name at the start of dependency specifier" - ) - name = name_token.text - tokenizer.consume("WS") - - extras = _parse_extras(tokenizer) - tokenizer.consume("WS") - - url, specifier, marker = _parse_requirement_details(tokenizer) - tokenizer.expect("END", expected="end of dependency specifier") - - return ParsedRequirement(name, url, extras, specifier, marker) - - -def _parse_requirement_details( - tokenizer: Tokenizer, -) -> Tuple[str, str, Optional[MarkerList]]: - """ - requirement_details = AT URL (WS requirement_marker?)? - | specifier WS? (requirement_marker)? - """ - - specifier = "" - url = "" - marker = None - - if tokenizer.check("AT"): - tokenizer.read() - tokenizer.consume("WS") - - url_start = tokenizer.position - url = tokenizer.expect("URL", expected="URL after @").text - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - tokenizer.expect("WS", expected="whitespace after URL") - - # The input might end after whitespace. - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, span_start=url_start, after="URL and whitespace" - ) - else: - specifier_start = tokenizer.position - specifier = _parse_specifier(tokenizer) - tokenizer.consume("WS") - - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, - span_start=specifier_start, - after=( - "version specifier" - if specifier - else "name and no valid version specifier" - ), - ) - - return (url, specifier, marker) - - -def _parse_requirement_marker( - tokenizer: Tokenizer, *, span_start: int, after: str -) -> MarkerList: - """ - requirement_marker = SEMICOLON marker WS? - """ - - if not tokenizer.check("SEMICOLON"): - tokenizer.raise_syntax_error( - f"Expected end or semicolon (after {after})", - span_start=span_start, - ) - tokenizer.read() - - marker = _parse_marker(tokenizer) - tokenizer.consume("WS") - - return marker - - -def _parse_extras(tokenizer: Tokenizer) -> List[str]: - """ - extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? - """ - if not tokenizer.check("LEFT_BRACKET", peek=True): - return [] - - with tokenizer.enclosing_tokens( - "LEFT_BRACKET", - "RIGHT_BRACKET", - around="extras", - ): - tokenizer.consume("WS") - extras = _parse_extras_list(tokenizer) - tokenizer.consume("WS") - - return extras - - -def _parse_extras_list(tokenizer: Tokenizer) -> List[str]: - """ - extras_list = identifier (wsp* ',' wsp* identifier)* - """ - extras: List[str] = [] - - if not tokenizer.check("IDENTIFIER"): - return extras - - extras.append(tokenizer.read().text) - - while True: - tokenizer.consume("WS") - if tokenizer.check("IDENTIFIER", peek=True): - tokenizer.raise_syntax_error("Expected comma between extra names") - elif not tokenizer.check("COMMA"): - break - - tokenizer.read() - tokenizer.consume("WS") - - extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") - extras.append(extra_token.text) - - return extras - - -def _parse_specifier(tokenizer: Tokenizer) -> str: - """ - specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS - | WS? version_many WS? - """ - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="version specifier", - ): - tokenizer.consume("WS") - parsed_specifiers = _parse_version_many(tokenizer) - tokenizer.consume("WS") - - return parsed_specifiers - - -def _parse_version_many(tokenizer: Tokenizer) -> str: - """ - version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? - """ - parsed_specifiers = "" - while tokenizer.check("SPECIFIER"): - span_start = tokenizer.position - parsed_specifiers += tokenizer.read().text - if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): - tokenizer.raise_syntax_error( - ".* suffix can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position + 1, - ) - if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): - tokenizer.raise_syntax_error( - "Local version label can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position, - ) - tokenizer.consume("WS") - if not tokenizer.check("COMMA"): - break - parsed_specifiers += tokenizer.read().text - tokenizer.consume("WS") - - return parsed_specifiers - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for marker expression -# -------------------------------------------------------------------------------------- -def parse_marker(source: str) -> MarkerList: - return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList: - retval = _parse_marker(tokenizer) - tokenizer.expect("END", expected="end of marker expression") - return retval - - -def _parse_marker(tokenizer: Tokenizer) -> MarkerList: - """ - marker = marker_atom (BOOLOP marker_atom)+ - """ - expression = [_parse_marker_atom(tokenizer)] - while tokenizer.check("BOOLOP"): - token = tokenizer.read() - expr_right = _parse_marker_atom(tokenizer) - expression.extend((token.text, expr_right)) - return expression - - -def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: - """ - marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? - | WS? marker_item WS? - """ - - tokenizer.consume("WS") - if tokenizer.check("LEFT_PARENTHESIS", peek=True): - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="marker expression", - ): - tokenizer.consume("WS") - marker: MarkerAtom = _parse_marker(tokenizer) - tokenizer.consume("WS") - else: - marker = _parse_marker_item(tokenizer) - tokenizer.consume("WS") - return marker - - -def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: - """ - marker_item = WS? marker_var WS? marker_op WS? marker_var WS? - """ - tokenizer.consume("WS") - marker_var_left = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - marker_op = _parse_marker_op(tokenizer) - tokenizer.consume("WS") - marker_var_right = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - return (marker_var_left, marker_op, marker_var_right) - - -def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: - """ - marker_var = VARIABLE | QUOTED_STRING - """ - if tokenizer.check("VARIABLE"): - return process_env_var(tokenizer.read().text.replace(".", "_")) - elif tokenizer.check("QUOTED_STRING"): - return process_python_str(tokenizer.read().text) - else: - tokenizer.raise_syntax_error( - message="Expected a marker variable or quoted string" - ) - - -def process_env_var(env_var: str) -> Variable: - if env_var in ("platform_python_implementation", "python_implementation"): - return Variable("platform_python_implementation") - else: - return Variable(env_var) - - -def process_python_str(python_str: str) -> Value: - value = ast.literal_eval(python_str) - return Value(str(value)) - - -def _parse_marker_op(tokenizer: Tokenizer) -> Op: - """ - marker_op = IN | NOT IN | OP - """ - if tokenizer.check("IN"): - tokenizer.read() - return Op("in") - elif tokenizer.check("NOT"): - tokenizer.read() - tokenizer.expect("WS", expected="whitespace after 'not'") - tokenizer.expect("IN", expected="'in' after 'not'") - return Op("not in") - elif tokenizer.check("OP"): - return Op(tokenizer.read().text) - else: - return tokenizer.raise_syntax_error( - "Expected marker operator, one of " - "<=, <, !=, ==, >=, >, ~=, ===, in, not in" - ) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_structures.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_structures.py deleted file mode 100644 index 90a6465f968..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_structures.py +++ /dev/null @@ -1,61 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - - -class InfinityType: - def __repr__(self) -> str: - return "Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return False - - def __le__(self, other: object) -> bool: - return False - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return True - - def __ge__(self, other: object) -> bool: - return True - - def __neg__(self: object) -> "NegativeInfinityType": - return NegativeInfinity - - -Infinity = InfinityType() - - -class NegativeInfinityType: - def __repr__(self) -> str: - return "-Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return True - - def __le__(self, other: object) -> bool: - return True - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return False - - def __ge__(self, other: object) -> bool: - return False - - def __neg__(self: object) -> InfinityType: - return Infinity - - -NegativeInfinity = NegativeInfinityType() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_tokenizer.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_tokenizer.py deleted file mode 100644 index dd0d648d49a..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_tokenizer.py +++ /dev/null @@ -1,192 +0,0 @@ -import contextlib -import re -from dataclasses import dataclass -from typing import Dict, Iterator, NoReturn, Optional, Tuple, Union - -from .specifiers import Specifier - - -@dataclass -class Token: - name: str - text: str - position: int - - -class ParserSyntaxError(Exception): - """The provided source text could not be parsed correctly.""" - - def __init__( - self, - message: str, - *, - source: str, - span: Tuple[int, int], - ) -> None: - self.span = span - self.message = message - self.source = source - - super().__init__() - - def __str__(self) -> str: - marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" - return "\n ".join([self.message, self.source, marker]) - - -DEFAULT_RULES: "Dict[str, Union[str, re.Pattern[str]]]" = { - "LEFT_PARENTHESIS": r"\(", - "RIGHT_PARENTHESIS": r"\)", - "LEFT_BRACKET": r"\[", - "RIGHT_BRACKET": r"\]", - "SEMICOLON": r";", - "COMMA": r",", - "QUOTED_STRING": re.compile( - r""" - ( - ('[^']*') - | - ("[^"]*") - ) - """, - re.VERBOSE, - ), - "OP": r"(===|==|~=|!=|<=|>=|<|>)", - "BOOLOP": r"\b(or|and)\b", - "IN": r"\bin\b", - "NOT": r"\bnot\b", - "VARIABLE": re.compile( - r""" - \b( - python_version - |python_full_version - |os[._]name - |sys[._]platform - |platform_(release|system) - |platform[._](version|machine|python_implementation) - |python_implementation - |implementation_(name|version) - |extra - )\b - """, - re.VERBOSE, - ), - "SPECIFIER": re.compile( - Specifier._operator_regex_str + Specifier._version_regex_str, - re.VERBOSE | re.IGNORECASE, - ), - "AT": r"\@", - "URL": r"[^ \t]+", - "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", - "VERSION_PREFIX_TRAIL": r"\.\*", - "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", - "WS": r"[ \t]+", - "END": r"$", -} - - -class Tokenizer: - """Context-sensitive token parsing. - - Provides methods to examine the input stream to check whether the next token - matches. - """ - - def __init__( - self, - source: str, - *, - rules: "Dict[str, Union[str, re.Pattern[str]]]", - ) -> None: - self.source = source - self.rules: Dict[str, re.Pattern[str]] = { - name: re.compile(pattern) for name, pattern in rules.items() - } - self.next_token: Optional[Token] = None - self.position = 0 - - def consume(self, name: str) -> None: - """Move beyond provided token name, if at current position.""" - if self.check(name): - self.read() - - def check(self, name: str, *, peek: bool = False) -> bool: - """Check whether the next token has the provided name. - - By default, if the check succeeds, the token *must* be read before - another check. If `peek` is set to `True`, the token is not loaded and - would need to be checked again. - """ - assert ( - self.next_token is None - ), f"Cannot check for {name!r}, already have {self.next_token!r}" - assert name in self.rules, f"Unknown token name: {name!r}" - - expression = self.rules[name] - - match = expression.match(self.source, self.position) - if match is None: - return False - if not peek: - self.next_token = Token(name, match[0], self.position) - return True - - def expect(self, name: str, *, expected: str) -> Token: - """Expect a certain token name next, failing with a syntax error otherwise. - - The token is *not* read. - """ - if not self.check(name): - raise self.raise_syntax_error(f"Expected {expected}") - return self.read() - - def read(self) -> Token: - """Consume the next token and return it.""" - token = self.next_token - assert token is not None - - self.position += len(token.text) - self.next_token = None - - return token - - def raise_syntax_error( - self, - message: str, - *, - span_start: Optional[int] = None, - span_end: Optional[int] = None, - ) -> NoReturn: - """Raise ParserSyntaxError at the given position.""" - span = ( - self.position if span_start is None else span_start, - self.position if span_end is None else span_end, - ) - raise ParserSyntaxError( - message, - source=self.source, - span=span, - ) - - @contextlib.contextmanager - def enclosing_tokens( - self, open_token: str, close_token: str, *, around: str - ) -> Iterator[None]: - if self.check(open_token): - open_position = self.position - self.read() - else: - open_position = None - - yield - - if open_position is None: - return - - if not self.check(close_token): - self.raise_syntax_error( - f"Expected matching {close_token} for {open_token}, after {around}", - span_start=open_position, - ) - - self.read() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py deleted file mode 100644 index 8b98fca7233..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py +++ /dev/null @@ -1,252 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import operator -import os -import platform -import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -from ._parser import ( - MarkerAtom, - MarkerList, - Op, - Value, - Variable, - parse_marker as _parse_marker, -) -from ._tokenizer import ParserSyntaxError -from .specifiers import InvalidSpecifier, Specifier -from .utils import canonicalize_name - -__all__ = [ - "InvalidMarker", - "UndefinedComparison", - "UndefinedEnvironmentName", - "Marker", - "default_environment", -] - -Operator = Callable[[str, str], bool] - - -class InvalidMarker(ValueError): - """ - An invalid marker was found, users should refer to PEP 508. - """ - - -class UndefinedComparison(ValueError): - """ - An invalid operation was attempted on a value that doesn't support it. - """ - - -class UndefinedEnvironmentName(ValueError): - """ - A name was attempted to be used that does not exist inside of the - environment. - """ - - -def _normalize_extra_values(results: Any) -> Any: - """ - Normalize extra values. - """ - if isinstance(results[0], tuple): - lhs, op, rhs = results[0] - if isinstance(lhs, Variable) and lhs.value == "extra": - normalized_extra = canonicalize_name(rhs.value) - rhs = Value(normalized_extra) - elif isinstance(rhs, Variable) and rhs.value == "extra": - normalized_extra = canonicalize_name(lhs.value) - lhs = Value(normalized_extra) - results[0] = lhs, op, rhs - return results - - -def _format_marker( - marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True -) -> str: - - assert isinstance(marker, (list, tuple, str)) - - # Sometimes we have a structure like [[...]] which is a single item list - # where the single item is itself it's own list. In that case we want skip - # the rest of this function so that we don't get extraneous () on the - # outside. - if ( - isinstance(marker, list) - and len(marker) == 1 - and isinstance(marker[0], (list, tuple)) - ): - return _format_marker(marker[0]) - - if isinstance(marker, list): - inner = (_format_marker(m, first=False) for m in marker) - if first: - return " ".join(inner) - else: - return "(" + " ".join(inner) + ")" - elif isinstance(marker, tuple): - return " ".join([m.serialize() for m in marker]) - else: - return marker - - -_operators: Dict[str, Operator] = { - "in": lambda lhs, rhs: lhs in rhs, - "not in": lambda lhs, rhs: lhs not in rhs, - "<": operator.lt, - "<=": operator.le, - "==": operator.eq, - "!=": operator.ne, - ">=": operator.ge, - ">": operator.gt, -} - - -def _eval_op(lhs: str, op: Op, rhs: str) -> bool: - try: - spec = Specifier("".join([op.serialize(), rhs])) - except InvalidSpecifier: - pass - else: - return spec.contains(lhs, prereleases=True) - - oper: Optional[Operator] = _operators.get(op.serialize()) - if oper is None: - raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.") - - return oper(lhs, rhs) - - -def _normalize(*values: str, key: str) -> Tuple[str, ...]: - # PEP 685 – Comparison of extra names for optional distribution dependencies - # https://peps.python.org/pep-0685/ - # > When comparing extra names, tools MUST normalize the names being - # > compared using the semantics outlined in PEP 503 for names - if key == "extra": - return tuple(canonicalize_name(v) for v in values) - - # other environment markers don't have such standards - return values - - -def _evaluate_markers(markers: MarkerList, environment: Dict[str, str]) -> bool: - groups: List[List[bool]] = [[]] - - for marker in markers: - assert isinstance(marker, (list, tuple, str)) - - if isinstance(marker, list): - groups[-1].append(_evaluate_markers(marker, environment)) - elif isinstance(marker, tuple): - lhs, op, rhs = marker - - if isinstance(lhs, Variable): - environment_key = lhs.value - lhs_value = environment[environment_key] - rhs_value = rhs.value - else: - lhs_value = lhs.value - environment_key = rhs.value - rhs_value = environment[environment_key] - - lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key) - groups[-1].append(_eval_op(lhs_value, op, rhs_value)) - else: - assert marker in ["and", "or"] - if marker == "or": - groups.append([]) - - return any(all(item) for item in groups) - - -def format_full_version(info: "sys._version_info") -> str: - version = "{0.major}.{0.minor}.{0.micro}".format(info) - kind = info.releaselevel - if kind != "final": - version += kind[0] + str(info.serial) - return version - - -def default_environment() -> Dict[str, str]: - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name - return { - "implementation_name": implementation_name, - "implementation_version": iver, - "os_name": os.name, - "platform_machine": platform.machine(), - "platform_release": platform.release(), - "platform_system": platform.system(), - "platform_version": platform.version(), - "python_full_version": platform.python_version(), - "platform_python_implementation": platform.python_implementation(), - "python_version": ".".join(platform.python_version_tuple()[:2]), - "sys_platform": sys.platform, - } - - -class Marker: - def __init__(self, marker: str) -> None: - # Note: We create a Marker object without calling this constructor in - # packaging.requirements.Requirement. If any additional logic is - # added here, make sure to mirror/adapt Requirement. - try: - self._markers = _normalize_extra_values(_parse_marker(marker)) - # The attribute `_markers` can be described in terms of a recursive type: - # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]] - # - # For example, the following expression: - # python_version > "3.6" or (python_version == "3.6" and os_name == "unix") - # - # is parsed into: - # [ - # (<Variable('python_version')>, <Op('>')>, <Value('3.6')>), - # 'and', - # [ - # (<Variable('python_version')>, <Op('==')>, <Value('3.6')>), - # 'or', - # (<Variable('os_name')>, <Op('==')>, <Value('unix')>) - # ] - # ] - except ParserSyntaxError as e: - raise InvalidMarker(str(e)) from e - - def __str__(self) -> str: - return _format_marker(self._markers) - - def __repr__(self) -> str: - return f"<Marker('{self}')>" - - def __hash__(self) -> int: - return hash((self.__class__.__name__, str(self))) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Marker): - return NotImplemented - - return str(self) == str(other) - - def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool: - """Evaluate a marker. - - Return the boolean from evaluating the given marker against the - environment. environment is an optional argument to override all or - part of the determined environment. - - The environment is determined from the current Python process. - """ - current_environment = default_environment() - current_environment["extra"] = "" - if environment is not None: - current_environment.update(environment) - # The API used to allow setting extra to None. We need to handle this - # case for backwards compatibility. - if current_environment["extra"] is None: - current_environment["extra"] = "" - - return _evaluate_markers(self._markers, current_environment) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py deleted file mode 100644 index fb274930799..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py +++ /dev/null @@ -1,825 +0,0 @@ -import email.feedparser -import email.header -import email.message -import email.parser -import email.policy -import sys -import typing -from typing import ( - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - Union, - cast, -) - -from . import requirements, specifiers, utils, version as version_module - -T = typing.TypeVar("T") -if sys.version_info[:2] >= (3, 8): # pragma: no cover - from typing import Literal, TypedDict -else: # pragma: no cover - if typing.TYPE_CHECKING: - from typing_extensions import Literal, TypedDict - else: - try: - from typing_extensions import Literal, TypedDict - except ImportError: - - class Literal: - def __init_subclass__(*_args, **_kwargs): - pass - - class TypedDict: - def __init_subclass__(*_args, **_kwargs): - pass - - -try: - ExceptionGroup -except NameError: # pragma: no cover - - class ExceptionGroup(Exception): # noqa: N818 - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: List[Exception] - - def __init__(self, message: str, exceptions: List[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" - -else: # pragma: no cover - ExceptionGroup = ExceptionGroup - - -class InvalidMetadata(ValueError): - """A metadata field contains invalid data.""" - - field: str - """The name of the field that contains invalid data.""" - - def __init__(self, field: str, message: str) -> None: - self.field = field - super().__init__(message) - - -# The RawMetadata class attempts to make as few assumptions about the underlying -# serialization formats as possible. The idea is that as long as a serialization -# formats offer some very basic primitives in *some* way then we can support -# serializing to and from that format. -class RawMetadata(TypedDict, total=False): - """A dictionary of raw core metadata. - - Each field in core metadata maps to a key of this dictionary (when data is - provided). The key is lower-case and underscores are used instead of dashes - compared to the equivalent core metadata field. Any core metadata field that - can be specified multiple times or can hold multiple values in a single - field have a key with a plural name. See :class:`Metadata` whose attributes - match the keys of this dictionary. - - Core metadata fields that can be specified multiple times are stored as a - list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - - """ - - # Metadata 1.0 - PEP 241 - metadata_version: str - name: str - version: str - platforms: List[str] - summary: str - description: str - keywords: List[str] - home_page: str - author: str - author_email: str - license: str - - # Metadata 1.1 - PEP 314 - supported_platforms: List[str] - download_url: str - classifiers: List[str] - requires: List[str] - provides: List[str] - obsoletes: List[str] - - # Metadata 1.2 - PEP 345 - maintainer: str - maintainer_email: str - requires_dist: List[str] - provides_dist: List[str] - obsoletes_dist: List[str] - requires_python: str - requires_external: List[str] - project_urls: Dict[str, str] - - # Metadata 2.0 - # PEP 426 attempted to completely revamp the metadata format - # but got stuck without ever being able to build consensus on - # it and ultimately ended up withdrawn. - # - # However, a number of tools had started emitting METADATA with - # `2.0` Metadata-Version, so for historical reasons, this version - # was skipped. - - # Metadata 2.1 - PEP 566 - description_content_type: str - provides_extra: List[str] - - # Metadata 2.2 - PEP 643 - dynamic: List[str] - - # Metadata 2.3 - PEP 685 - # No new fields were added in PEP 685, just some edge case were - # tightened up to provide better interoptability. - - -_STRING_FIELDS = { - "author", - "author_email", - "description", - "description_content_type", - "download_url", - "home_page", - "license", - "maintainer", - "maintainer_email", - "metadata_version", - "name", - "requires_python", - "summary", - "version", -} - -_LIST_FIELDS = { - "classifiers", - "dynamic", - "obsoletes", - "obsoletes_dist", - "platforms", - "provides", - "provides_dist", - "provides_extra", - "requires", - "requires_dist", - "requires_external", - "supported_platforms", -} - -_DICT_FIELDS = { - "project_urls", -} - - -def _parse_keywords(data: str) -> List[str]: - """Split a string of comma-separate keyboards into a list of keywords.""" - return [k.strip() for k in data.split(",")] - - -def _parse_project_urls(data: List[str]) -> Dict[str, str]: - """Parse a list of label/URL string pairings separated by a comma.""" - urls = {} - for pair in data: - # Our logic is slightly tricky here as we want to try and do - # *something* reasonable with malformed data. - # - # The main thing that we have to worry about, is data that does - # not have a ',' at all to split the label from the Value. There - # isn't a singular right answer here, and we will fail validation - # later on (if the caller is validating) so it doesn't *really* - # matter, but since the missing value has to be an empty str - # and our return value is dict[str, str], if we let the key - # be the missing value, then they'd have multiple '' values that - # overwrite each other in a accumulating dict. - # - # The other potentional issue is that it's possible to have the - # same label multiple times in the metadata, with no solid "right" - # answer with what to do in that case. As such, we'll do the only - # thing we can, which is treat the field as unparseable and add it - # to our list of unparsed fields. - parts = [p.strip() for p in pair.split(",", 1)] - parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items - - # TODO: The spec doesn't say anything about if the keys should be - # considered case sensitive or not... logically they should - # be case-preserving and case-insensitive, but doing that - # would open up more cases where we might have duplicate - # entries. - label, url = parts - if label in urls: - # The label already exists in our set of urls, so this field - # is unparseable, and we can just add the whole thing to our - # unparseable data and stop processing it. - raise KeyError("duplicate labels in project urls") - urls[label] = url - - return urls - - -def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: - """Get the body of the message.""" - # If our source is a str, then our caller has managed encodings for us, - # and we don't need to deal with it. - if isinstance(source, str): - payload: str = msg.get_payload() - return payload - # If our source is a bytes, then we're managing the encoding and we need - # to deal with it. - else: - bpayload: bytes = msg.get_payload(decode=True) - try: - return bpayload.decode("utf8", "strict") - except UnicodeDecodeError: - raise ValueError("payload in an invalid encoding") - - -# The various parse_FORMAT functions here are intended to be as lenient as -# possible in their parsing, while still returning a correctly typed -# RawMetadata. -# -# To aid in this, we also generally want to do as little touching of the -# data as possible, except where there are possibly some historic holdovers -# that make valid data awkward to work with. -# -# While this is a lower level, intermediate format than our ``Metadata`` -# class, some light touch ups can make a massive difference in usability. - -# Map METADATA fields to RawMetadata. -_EMAIL_TO_RAW_MAPPING = { - "author": "author", - "author-email": "author_email", - "classifier": "classifiers", - "description": "description", - "description-content-type": "description_content_type", - "download-url": "download_url", - "dynamic": "dynamic", - "home-page": "home_page", - "keywords": "keywords", - "license": "license", - "maintainer": "maintainer", - "maintainer-email": "maintainer_email", - "metadata-version": "metadata_version", - "name": "name", - "obsoletes": "obsoletes", - "obsoletes-dist": "obsoletes_dist", - "platform": "platforms", - "project-url": "project_urls", - "provides": "provides", - "provides-dist": "provides_dist", - "provides-extra": "provides_extra", - "requires": "requires", - "requires-dist": "requires_dist", - "requires-external": "requires_external", - "requires-python": "requires_python", - "summary": "summary", - "supported-platform": "supported_platforms", - "version": "version", -} -_RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()} - - -def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: - """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``). - - This function returns a two-item tuple of dicts. The first dict is of - recognized fields from the core metadata specification. Fields that can be - parsed and translated into Python's built-in types are converted - appropriately. All other fields are left as-is. Fields that are allowed to - appear multiple times are stored as lists. - - The second dict contains all other fields from the metadata. This includes - any unrecognized fields. It also includes any fields which are expected to - be parsed into a built-in type but were not formatted appropriately. Finally, - any fields that are expected to appear only once but are repeated are - included in this dict. - - """ - raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} - unparsed: Dict[str, List[str]] = {} - - if isinstance(data, str): - parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) - else: - parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) - - # We have to wrap parsed.keys() in a set, because in the case of multiple - # values for a key (a list), the key will appear multiple times in the - # list of keys, but we're avoiding that by using get_all(). - for name in frozenset(parsed.keys()): - # Header names in RFC are case insensitive, so we'll normalize to all - # lower case to make comparisons easier. - name = name.lower() - - # We use get_all() here, even for fields that aren't multiple use, - # because otherwise someone could have e.g. two Name fields, and we - # would just silently ignore it rather than doing something about it. - headers = parsed.get_all(name) or [] - - # The way the email module works when parsing bytes is that it - # unconditionally decodes the bytes as ascii using the surrogateescape - # handler. When you pull that data back out (such as with get_all() ), - # it looks to see if the str has any surrogate escapes, and if it does - # it wraps it in a Header object instead of returning the string. - # - # As such, we'll look for those Header objects, and fix up the encoding. - value = [] - # Flag if we have run into any issues processing the headers, thus - # signalling that the data belongs in 'unparsed'. - valid_encoding = True - for h in headers: - # It's unclear if this can return more types than just a Header or - # a str, so we'll just assert here to make sure. - assert isinstance(h, (email.header.Header, str)) - - # If it's a header object, we need to do our little dance to get - # the real data out of it. In cases where there is invalid data - # we're going to end up with mojibake, but there's no obvious, good - # way around that without reimplementing parts of the Header object - # ourselves. - # - # That should be fine since, if mojibacked happens, this key is - # going into the unparsed dict anyways. - if isinstance(h, email.header.Header): - # The Header object stores it's data as chunks, and each chunk - # can be independently encoded, so we'll need to check each - # of them. - chunks: List[Tuple[bytes, Optional[str]]] = [] - for bin, encoding in email.header.decode_header(h): - try: - bin.decode("utf8", "strict") - except UnicodeDecodeError: - # Enable mojibake. - encoding = "latin1" - valid_encoding = False - else: - encoding = "utf8" - chunks.append((bin, encoding)) - - # Turn our chunks back into a Header object, then let that - # Header object do the right thing to turn them into a - # string for us. - value.append(str(email.header.make_header(chunks))) - # This is already a string, so just add it. - else: - value.append(h) - - # We've processed all of our values to get them into a list of str, - # but we may have mojibake data, in which case this is an unparsed - # field. - if not valid_encoding: - unparsed[name] = value - continue - - raw_name = _EMAIL_TO_RAW_MAPPING.get(name) - if raw_name is None: - # This is a bit of a weird situation, we've encountered a key that - # we don't know what it means, so we don't know whether it's meant - # to be a list or not. - # - # Since we can't really tell one way or another, we'll just leave it - # as a list, even though it may be a single item list, because that's - # what makes the most sense for email headers. - unparsed[name] = value - continue - - # If this is one of our string fields, then we'll check to see if our - # value is a list of a single item. If it is then we'll assume that - # it was emitted as a single string, and unwrap the str from inside - # the list. - # - # If it's any other kind of data, then we haven't the faintest clue - # what we should parse it as, and we have to just add it to our list - # of unparsed stuff. - if raw_name in _STRING_FIELDS and len(value) == 1: - raw[raw_name] = value[0] - # If this is one of our list of string fields, then we can just assign - # the value, since email *only* has strings, and our get_all() call - # above ensures that this is a list. - elif raw_name in _LIST_FIELDS: - raw[raw_name] = value - # Special Case: Keywords - # The keywords field is implemented in the metadata spec as a str, - # but it conceptually is a list of strings, and is serialized using - # ", ".join(keywords), so we'll do some light data massaging to turn - # this into what it logically is. - elif raw_name == "keywords" and len(value) == 1: - raw[raw_name] = _parse_keywords(value[0]) - # Special Case: Project-URL - # The project urls is implemented in the metadata spec as a list of - # specially-formatted strings that represent a key and a value, which - # is fundamentally a mapping, however the email format doesn't support - # mappings in a sane way, so it was crammed into a list of strings - # instead. - # - # We will do a little light data massaging to turn this into a map as - # it logically should be. - elif raw_name == "project_urls": - try: - raw[raw_name] = _parse_project_urls(value) - except KeyError: - unparsed[name] = value - # Nothing that we've done has managed to parse this, so it'll just - # throw it in our unparseable data and move on. - else: - unparsed[name] = value - - # We need to support getting the Description from the message payload in - # addition to getting it from the the headers. This does mean, though, there - # is the possibility of it being set both ways, in which case we put both - # in 'unparsed' since we don't know which is right. - try: - payload = _get_payload(parsed, data) - except ValueError: - unparsed.setdefault("description", []).append( - parsed.get_payload(decode=isinstance(data, bytes)) - ) - else: - if payload: - # Check to see if we've already got a description, if so then both - # it, and this body move to unparseable. - if "description" in raw: - description_header = cast(str, raw.pop("description")) - unparsed.setdefault("description", []).extend( - [description_header, payload] - ) - elif "description" in unparsed: - unparsed["description"].append(payload) - else: - raw["description"] = payload - - # We need to cast our `raw` to a metadata, because a TypedDict only support - # literal key names, but we're computing our key names on purpose, but the - # way this function is implemented, our `TypedDict` can only have valid key - # names. - return cast(RawMetadata, raw), unparsed - - -_NOT_FOUND = object() - - -# Keep the two values in sync. -_VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] -_MetadataVersion = Literal["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] - -_REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"]) - - -class _Validator(Generic[T]): - """Validate a metadata field. - - All _process_*() methods correspond to a core metadata field. The method is - called with the field's raw value. If the raw value is valid it is returned - in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field). - If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause - as appropriate). - """ - - name: str - raw_name: str - added: _MetadataVersion - - def __init__( - self, - *, - added: _MetadataVersion = "1.0", - ) -> None: - self.added = added - - def __set_name__(self, _owner: "Metadata", name: str) -> None: - self.name = name - self.raw_name = _RAW_TO_EMAIL_MAPPING[name] - - def __get__(self, instance: "Metadata", _owner: Type["Metadata"]) -> T: - # With Python 3.8, the caching can be replaced with functools.cached_property(). - # No need to check the cache as attribute lookup will resolve into the - # instance's __dict__ before __get__ is called. - cache = instance.__dict__ - value = instance._raw.get(self.name) - - # To make the _process_* methods easier, we'll check if the value is None - # and if this field is NOT a required attribute, and if both of those - # things are true, we'll skip the the converter. This will mean that the - # converters never have to deal with the None union. - if self.name in _REQUIRED_ATTRS or value is not None: - try: - converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}") - except AttributeError: - pass - else: - value = converter(value) - - cache[self.name] = value - try: - del instance._raw[self.name] # type: ignore[misc] - except KeyError: - pass - - return cast(T, value) - - def _invalid_metadata( - self, msg: str, cause: Optional[Exception] = None - ) -> InvalidMetadata: - exc = InvalidMetadata( - self.raw_name, msg.format_map({"field": repr(self.raw_name)}) - ) - exc.__cause__ = cause - return exc - - def _process_metadata_version(self, value: str) -> _MetadataVersion: - # Implicitly makes Metadata-Version required. - if value not in _VALID_METADATA_VERSIONS: - raise self._invalid_metadata(f"{value!r} is not a valid metadata version") - return cast(_MetadataVersion, value) - - def _process_name(self, value: str) -> str: - if not value: - raise self._invalid_metadata("{field} is a required field") - # Validate the name as a side-effect. - try: - utils.canonicalize_name(value, validate=True) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - else: - return value - - def _process_version(self, value: str) -> version_module.Version: - if not value: - raise self._invalid_metadata("{field} is a required field") - try: - return version_module.parse(value) - except version_module.InvalidVersion as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_summary(self, value: str) -> str: - """Check the field contains no newlines.""" - if "\n" in value: - raise self._invalid_metadata("{field} must be a single line") - return value - - def _process_description_content_type(self, value: str) -> str: - content_types = {"text/plain", "text/x-rst", "text/markdown"} - message = email.message.EmailMessage() - message["content-type"] = value - - content_type, parameters = ( - # Defaults to `text/plain` if parsing failed. - message.get_content_type().lower(), - message["content-type"].params, - ) - # Check if content-type is valid or defaulted to `text/plain` and thus was - # not parseable. - if content_type not in content_types or content_type not in value.lower(): - raise self._invalid_metadata( - f"{{field}} must be one of {list(content_types)}, not {value!r}" - ) - - charset = parameters.get("charset", "UTF-8") - if charset != "UTF-8": - raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" - ) - - markdown_variants = {"GFM", "CommonMark"} - variant = parameters.get("variant", "GFM") # Use an acceptable default. - if content_type == "text/markdown" and variant not in markdown_variants: - raise self._invalid_metadata( - f"valid Markdown variants for {{field}} are {list(markdown_variants)}, " - f"not {variant!r}", - ) - return value - - def _process_dynamic(self, value: List[str]) -> List[str]: - for dynamic_field in map(str.lower, value): - if dynamic_field in {"name", "version", "metadata-version"}: - raise self._invalid_metadata( - f"{value!r} is not allowed as a dynamic field" - ) - elif dynamic_field not in _EMAIL_TO_RAW_MAPPING: - raise self._invalid_metadata(f"{value!r} is not a valid dynamic field") - return list(map(str.lower, value)) - - def _process_provides_extra( - self, - value: List[str], - ) -> List[utils.NormalizedName]: - normalized_names = [] - try: - for name in value: - normalized_names.append(utils.canonicalize_name(name, validate=True)) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{name!r} is invalid for {{field}}", cause=exc - ) - else: - return normalized_names - - def _process_requires_python(self, value: str) -> specifiers.SpecifierSet: - try: - return specifiers.SpecifierSet(value) - except specifiers.InvalidSpecifier as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_requires_dist( - self, - value: List[str], - ) -> List[requirements.Requirement]: - reqs = [] - try: - for req in value: - reqs.append(requirements.Requirement(req)) - except requirements.InvalidRequirement as exc: - raise self._invalid_metadata(f"{req!r} is invalid for {{field}}", cause=exc) - else: - return reqs - - -class Metadata: - """Representation of distribution metadata. - - Compared to :class:`RawMetadata`, this class provides objects representing - metadata fields instead of only using built-in types. Any invalid metadata - will cause :exc:`InvalidMetadata` to be raised (with a - :py:attr:`~BaseException.__cause__` attribute as appropriate). - """ - - _raw: RawMetadata - - @classmethod - def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> "Metadata": - """Create an instance from :class:`RawMetadata`. - - If *validate* is true, all metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - ins = cls() - ins._raw = data.copy() # Mutations occur due to caching enriched values. - - if validate: - exceptions: List[Exception] = [] - try: - metadata_version = ins.metadata_version - metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None - - # Make sure to check for the fields that are present, the required - # fields (so their absence can be reported). - fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS - # Remove fields that have already been checked. - fields_to_check -= {"metadata_version"} - - for key in fields_to_check: - try: - if metadata_version: - # Can't use getattr() as that triggers descriptor protocol which - # will fail due to no value for the instance argument. - try: - field_metadata_version = cls.__dict__[key].added - except KeyError: - exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) - continue - field_age = _VALID_METADATA_VERSIONS.index( - field_metadata_version - ) - if field_age > metadata_age: - field = _RAW_TO_EMAIL_MAPPING[key] - exc = InvalidMetadata( - field, - "{field} introduced in metadata version " - "{field_metadata_version}, not {metadata_version}", - ) - exceptions.append(exc) - continue - getattr(ins, key) - except InvalidMetadata as exc: - exceptions.append(exc) - - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) - - return ins - - @classmethod - def from_email( - cls, data: Union[bytes, str], *, validate: bool = True - ) -> "Metadata": - """Parse metadata from email headers. - - If *validate* is true, the metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - raw, unparsed = parse_email(data) - - if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) - - try: - return cls.from_raw(raw, validate=validate) - except ExceptionGroup as exc_group: - raise ExceptionGroup( - "invalid or unparsed metadata", exc_group.exceptions - ) from None - - metadata_version: _Validator[_MetadataVersion] = _Validator() - """:external:ref:`core-metadata-metadata-version` - (required; validated to be a valid metadata version)""" - name: _Validator[str] = _Validator() - """:external:ref:`core-metadata-name` - (required; validated using :func:`~packaging.utils.canonicalize_name` and its - *validate* parameter)""" - version: _Validator[version_module.Version] = _Validator() - """:external:ref:`core-metadata-version` (required)""" - dynamic: _Validator[Optional[List[str]]] = _Validator( - added="2.2", - ) - """:external:ref:`core-metadata-dynamic` - (validated against core metadata field names and lowercased)""" - platforms: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-platform`""" - supported_platforms: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-supported-platform`""" - summary: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-summary` (validated to contain no newlines)""" - description: _Validator[Optional[str]] = _Validator() # TODO 2.1: can be in body - """:external:ref:`core-metadata-description`""" - description_content_type: _Validator[Optional[str]] = _Validator(added="2.1") - """:external:ref:`core-metadata-description-content-type` (validated)""" - keywords: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-keywords`""" - home_page: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-home-page`""" - download_url: _Validator[Optional[str]] = _Validator(added="1.1") - """:external:ref:`core-metadata-download-url`""" - author: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author`""" - author_email: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author-email`""" - maintainer: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer`""" - maintainer_email: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer-email`""" - license: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-license`""" - classifiers: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-classifier`""" - requires_dist: _Validator[Optional[List[requirements.Requirement]]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-dist`""" - requires_python: _Validator[Optional[specifiers.SpecifierSet]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-python`""" - # Because `Requires-External` allows for non-PEP 440 version specifiers, we - # don't do any processing on the values. - requires_external: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-requires-external`""" - project_urls: _Validator[Optional[Dict[str, str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-project-url`""" - # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation - # regardless of metadata version. - provides_extra: _Validator[Optional[List[utils.NormalizedName]]] = _Validator( - added="2.1", - ) - """:external:ref:`core-metadata-provides-extra`""" - provides_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-provides-dist`""" - obsoletes_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-obsoletes-dist`""" - requires: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Requires`` (deprecated)""" - provides: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Provides`` (deprecated)""" - obsoletes: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Obsoletes`` (deprecated)""" diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/py.typed b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/requirements.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/requirements.py deleted file mode 100644 index bdc43a7e98d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/requirements.py +++ /dev/null @@ -1,90 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from typing import Any, Iterator, Optional, Set - -from ._parser import parse_requirement as _parse_requirement -from ._tokenizer import ParserSyntaxError -from .markers import Marker, _normalize_extra_values -from .specifiers import SpecifierSet -from .utils import canonicalize_name - - -class InvalidRequirement(ValueError): - """ - An invalid requirement was found, users should refer to PEP 508. - """ - - -class Requirement: - """Parse a requirement. - - Parse a given requirement string into its parts, such as name, specifier, - URL, and extras. Raises InvalidRequirement on a badly-formed requirement - string. - """ - - # TODO: Can we test whether something is contained within a requirement? - # If so how do we do that? Do we need to test against the _name_ of - # the thing as well as the version? What about the markers? - # TODO: Can we normalize the name and extra name? - - def __init__(self, requirement_string: str) -> None: - try: - parsed = _parse_requirement(requirement_string) - except ParserSyntaxError as e: - raise InvalidRequirement(str(e)) from e - - self.name: str = parsed.name - self.url: Optional[str] = parsed.url or None - self.extras: Set[str] = set(parsed.extras or []) - self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) - self.marker: Optional[Marker] = None - if parsed.marker is not None: - self.marker = Marker.__new__(Marker) - self.marker._markers = _normalize_extra_values(parsed.marker) - - def _iter_parts(self, name: str) -> Iterator[str]: - yield name - - if self.extras: - formatted_extras = ",".join(sorted(self.extras)) - yield f"[{formatted_extras}]" - - if self.specifier: - yield str(self.specifier) - - if self.url: - yield f"@ {self.url}" - if self.marker: - yield " " - - if self.marker: - yield f"; {self.marker}" - - def __str__(self) -> str: - return "".join(self._iter_parts(self.name)) - - def __repr__(self) -> str: - return f"<Requirement('{self}')>" - - def __hash__(self) -> int: - return hash( - ( - self.__class__.__name__, - *self._iter_parts(canonicalize_name(self.name)), - ) - ) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Requirement): - return NotImplemented - - return ( - canonicalize_name(self.name) == canonicalize_name(other.name) - and self.extras == other.extras - and self.specifier == other.specifier - and self.url == other.url - and self.marker == other.marker - ) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py deleted file mode 100644 index 2d015bab595..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py +++ /dev/null @@ -1,1017 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.specifiers import Specifier, SpecifierSet, InvalidSpecifier - from packaging.version import Version -""" - -import abc -import itertools -import re -from typing import Callable, Iterable, Iterator, List, Optional, Tuple, TypeVar, Union - -from .utils import canonicalize_version -from .version import Version - -UnparsedVersion = Union[Version, str] -UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) -CallableOperator = Callable[[Version, str], bool] - - -def _coerce_version(version: UnparsedVersion) -> Version: - if not isinstance(version, Version): - version = Version(version) - return version - - -class InvalidSpecifier(ValueError): - """ - Raised when attempting to create a :class:`Specifier` with a specifier - string that is invalid. - - >>> Specifier("lolwat") - Traceback (most recent call last): - ... - packaging.specifiers.InvalidSpecifier: Invalid specifier: 'lolwat' - """ - - -class BaseSpecifier(metaclass=abc.ABCMeta): - @abc.abstractmethod - def __str__(self) -> str: - """ - Returns the str representation of this Specifier-like object. This - should be representative of the Specifier itself. - """ - - @abc.abstractmethod - def __hash__(self) -> int: - """ - Returns a hash value for this Specifier-like object. - """ - - @abc.abstractmethod - def __eq__(self, other: object) -> bool: - """ - Returns a boolean representing whether or not the two Specifier-like - objects are equal. - - :param other: The other object to check against. - """ - - @property - @abc.abstractmethod - def prereleases(self) -> Optional[bool]: - """Whether or not pre-releases as a whole are allowed. - - This can be set to either ``True`` or ``False`` to explicitly enable or disable - prereleases or it can be set to ``None`` (the default) to use default semantics. - """ - - @prereleases.setter - def prereleases(self, value: bool) -> None: - """Setter for :attr:`prereleases`. - - :param value: The value to set. - """ - - @abc.abstractmethod - def contains(self, item: str, prereleases: Optional[bool] = None) -> bool: - """ - Determines if the given item is contained within this specifier. - """ - - @abc.abstractmethod - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """ - Takes an iterable of items and filters them so that only items which - are contained within this specifier are allowed in it. - """ - - -class Specifier(BaseSpecifier): - """This class abstracts handling of version specifiers. - - .. tip:: - - It is generally not required to instantiate this manually. You should instead - prefer to work with :class:`SpecifierSet` instead, which can parse - comma-separated version specifiers (which is what package metadata contains). - """ - - _operator_regex_str = r""" - (?P<operator>(~=|==|!=|<=|>=|<|>|===)) - """ - _version_regex_str = r""" - (?P<version> - (?: - # The identity operators allow for an escape hatch that will - # do an exact string match of the version you wish to install. - # This will not be parsed by PEP 440 and we cannot determine - # any semantic meaning from it. This operator is discouraged - # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator - \s* - [^\s;)]* # The arbitrary version can be just about anything, - # we match everything except for whitespace, a - # semi-colon for marker support, and a closing paren - # since versions can be enclosed in them. - ) - | - (?: - # The (non)equality operators allow for wild card and local - # versions to be specified so we have to define these two - # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - - # You cannot use a wild card and a pre-release, post-release, a dev or - # local version together so group them with a | and make them optional. - (?: - \.\* # Wild card syntax of .* - | - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local - )? - ) - | - (?: - # The compatible operator requires at least two digits in the - # release segment. - (?<=~=) # Only match for the compatible operator - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - | - (?: - # All other operators only allow a sub set of what the - # (non)equality operators do. Specifically they do not allow - # local versions to be specified nor do they allow the prefix - # matching wild cards. - (?<!==|!=|~=) # We have special cases for these - # operators so we want to make sure they - # don't match here. - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - ) - """ - - _regex = re.compile( - r"^\s*" + _operator_regex_str + _version_regex_str + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) - - _operators = { - "~=": "compatible", - "==": "equal", - "!=": "not_equal", - "<=": "less_than_equal", - ">=": "greater_than_equal", - "<": "less_than", - ">": "greater_than", - "===": "arbitrary", - } - - def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: - """Initialize a Specifier instance. - - :param spec: - The string representation of a specifier which will be parsed and - normalized before use. - :param prereleases: - This tells the specifier if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - :raises InvalidSpecifier: - If the given specifier is invalid (i.e. bad syntax). - """ - match = self._regex.search(spec) - if not match: - raise InvalidSpecifier(f"Invalid specifier: '{spec}'") - - self._spec: Tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) - - # Store whether or not this Specifier should accept prereleases - self._prereleases = prereleases - - # https://github.com/python/mypy/pull/13475#pullrequestreview-1079784515 - @property # type: ignore[override] - def prereleases(self) -> bool: - # If there is an explicit prereleases set for this, then we'll just - # blindly use that. - if self._prereleases is not None: - return self._prereleases - - # Look at all of our specifiers and determine if they are inclusive - # operators, and if they are if they are including an explicit - # prerelease. - operator, version = self._spec - if operator in ["==", ">=", "<=", "~=", "==="]: - # The == specifier can include a trailing .*, if it does we - # want to remove before parsing. - if operator == "==" and version.endswith(".*"): - version = version[:-2] - - # Parse the version, and if it is a pre-release than this - # specifier allows pre-releases. - if Version(version).is_prerelease: - return True - - return False - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - @property - def operator(self) -> str: - """The operator of this specifier. - - >>> Specifier("==1.2.3").operator - '==' - """ - return self._spec[0] - - @property - def version(self) -> str: - """The version of this specifier. - - >>> Specifier("==1.2.3").version - '1.2.3' - """ - return self._spec[1] - - def __repr__(self) -> str: - """A representation of the Specifier that shows all internal state. - - >>> Specifier('>=1.0.0') - <Specifier('>=1.0.0')> - >>> Specifier('>=1.0.0', prereleases=False) - <Specifier('>=1.0.0', prereleases=False)> - >>> Specifier('>=1.0.0', prereleases=True) - <Specifier('>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<{self.__class__.__name__}({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the Specifier that can be round-tripped. - - >>> str(Specifier('>=1.0.0')) - '>=1.0.0' - >>> str(Specifier('>=1.0.0', prereleases=False)) - '>=1.0.0' - """ - return "{}{}".format(*self._spec) - - @property - def _canonical_spec(self) -> Tuple[str, str]: - canonical_version = canonicalize_version( - self._spec[1], - strip_trailing_zero=(self._spec[0] != "~="), - ) - return self._spec[0], canonical_version - - def __hash__(self) -> int: - return hash(self._canonical_spec) - - def __eq__(self, other: object) -> bool: - """Whether or not the two Specifier-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> Specifier("==1.2.3") == Specifier("== 1.2.3.0") - True - >>> (Specifier("==1.2.3", prereleases=False) == - ... Specifier("==1.2.3", prereleases=True)) - True - >>> Specifier("==1.2.3") == "==1.2.3" - True - >>> Specifier("==1.2.3") == Specifier("==1.2.4") - False - >>> Specifier("==1.2.3") == Specifier("~=1.2.3") - False - """ - if isinstance(other, str): - try: - other = self.__class__(str(other)) - except InvalidSpecifier: - return NotImplemented - elif not isinstance(other, self.__class__): - return NotImplemented - - return self._canonical_spec == other._canonical_spec - - def _get_operator(self, op: str) -> CallableOperator: - operator_callable: CallableOperator = getattr( - self, f"_compare_{self._operators[op]}" - ) - return operator_callable - - def _compare_compatible(self, prospective: Version, spec: str) -> bool: - - # Compatible releases have an equivalent combination of >= and ==. That - # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to - # implement this in terms of the other specifiers instead of - # implementing it ourselves. The only thing we need to do is construct - # the other specifiers. - - # We want everything but the last item in the version, but we want to - # ignore suffix segments. - prefix = _version_join( - list(itertools.takewhile(_is_not_suffix, _version_split(spec)))[:-1] - ) - - # Add the prefix notation to the end of our string - prefix += ".*" - - return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( - prospective, prefix - ) - - def _compare_equal(self, prospective: Version, spec: str) -> bool: - - # We need special logic to handle prefix matching - if spec.endswith(".*"): - # In the case of prefix matching we want to ignore local segment. - normalized_prospective = canonicalize_version( - prospective.public, strip_trailing_zero=False - ) - # Get the normalized version string ignoring the trailing .* - normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) - # Split the spec out by bangs and dots, and pretend that there is - # an implicit dot in between a release segment and a pre-release segment. - split_spec = _version_split(normalized_spec) - - # Split the prospective version out by bangs and dots, and pretend - # that there is an implicit dot in between a release segment and - # a pre-release segment. - split_prospective = _version_split(normalized_prospective) - - # 0-pad the prospective version before shortening it to get the correct - # shortened version. - padded_prospective, _ = _pad_version(split_prospective, split_spec) - - # Shorten the prospective version to be the same length as the spec - # so that we can determine if the specifier is a prefix of the - # prospective version or not. - shortened_prospective = padded_prospective[: len(split_spec)] - - return shortened_prospective == split_spec - else: - # Convert our spec string into a Version - spec_version = Version(spec) - - # If the specifier does not have a local segment, then we want to - # act as if the prospective version also does not have a local - # segment. - if not spec_version.local: - prospective = Version(prospective.public) - - return prospective == spec_version - - def _compare_not_equal(self, prospective: Version, spec: str) -> bool: - return not self._compare_equal(prospective, spec) - - def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) <= Version(spec) - - def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) >= Version(spec) - - def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is less than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective < spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a pre-release version, that we do not accept pre-release - # versions for the version mentioned in the specifier (e.g. <3.1 should - # not match 3.1.dev0, but should match 3.0.dev0). - if not spec.is_prerelease and prospective.is_prerelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # less than the spec version *and* it's not a pre-release of the same - # version in the spec. - return True - - def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is greater than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective > spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a post-release version, that we do not accept - # post-release versions for the version mentioned in the specifier - # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). - if not spec.is_postrelease and prospective.is_postrelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is technically greater than, to match. - if prospective.local is not None: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # greater than the spec version *and* it's not a pre-release of the - # same version in the spec. - return True - - def _compare_arbitrary(self, prospective: Version, spec: str) -> bool: - return str(prospective).lower() == str(spec).lower() - - def __contains__(self, item: Union[str, Version]) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in Specifier(">=1.2.3") - True - >>> Version("1.2.3") in Specifier(">=1.2.3") - True - >>> "1.0.0" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, item: UnparsedVersion, prereleases: Optional[bool] = None - ) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this Specifier. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> Specifier(">=1.2.3").contains("1.2.3") - True - >>> Specifier(">=1.2.3").contains(Version("1.2.3")) - True - >>> Specifier(">=1.2.3").contains("1.0.0") - False - >>> Specifier(">=1.2.3").contains("1.3.0a1") - False - >>> Specifier(">=1.2.3", prereleases=True).contains("1.3.0a1") - True - >>> Specifier(">=1.2.3").contains("1.3.0a1", prereleases=True) - True - """ - - # Determine if prereleases are to be allowed or not. - if prereleases is None: - prereleases = self.prereleases - - # Normalize item to a Version, this allows us to have a shortcut for - # "2.0" in Specifier(">=2") - normalized_item = _coerce_version(item) - - # Determine if we should be supporting prereleases in this specifier - # or not, if we do not support prereleases than we can short circuit - # logic if this version is a prereleases. - if normalized_item.is_prerelease and not prereleases: - return False - - # Actually do the comparison to determine if this item is contained - # within this Specifier or not. - operator_callable: CallableOperator = self._get_operator(self.operator) - return operator_callable(normalized_item, self.version) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifier. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(Specifier().contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.2.3", "1.3", Version("1.4")])) - ['1.2.3', '1.3', <Version('1.4')>] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.5a1"])) - ['1.5a1'] - >>> list(Specifier(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - """ - - yielded = False - found_prereleases = [] - - kw = {"prereleases": prereleases if prereleases is not None else True} - - # Attempt to iterate over all the values in the iterable and if any of - # them match, yield them. - for version in iterable: - parsed_version = _coerce_version(version) - - if self.contains(parsed_version, **kw): - # If our version is a prerelease, and we were not set to allow - # prereleases, then we'll store it for later in case nothing - # else matches this specifier. - if parsed_version.is_prerelease and not ( - prereleases or self.prereleases - ): - found_prereleases.append(version) - # Either this is not a prerelease, or we should have been - # accepting prereleases from the beginning. - else: - yielded = True - yield version - - # Now that we've iterated over everything, determine if we've yielded - # any values, and if we have not and we have any prereleases stored up - # then we will go ahead and yield the prereleases. - if not yielded and found_prereleases: - for version in found_prereleases: - yield version - - -_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") - - -def _version_split(version: str) -> List[str]: - """Split version into components. - - The split components are intended for version comparison. The logic does - not attempt to retain the original version string, so joining the - components back with :func:`_version_join` may not produce the original - version string. - """ - result: List[str] = [] - - epoch, _, rest = version.rpartition("!") - result.append(epoch or "0") - - for item in rest.split("."): - match = _prefix_regex.search(item) - if match: - result.extend(match.groups()) - else: - result.append(item) - return result - - -def _version_join(components: List[str]) -> str: - """Join split version components into a version string. - - This function assumes the input came from :func:`_version_split`, where the - first component must be the epoch (either empty or numeric), and all other - components numeric. - """ - epoch, *rest = components - return f"{epoch}!{'.'.join(rest)}" - - -def _is_not_suffix(segment: str) -> bool: - return not any( - segment.startswith(prefix) for prefix in ("dev", "a", "b", "rc", "post") - ) - - -def _pad_version(left: List[str], right: List[str]) -> Tuple[List[str], List[str]]: - left_split, right_split = [], [] - - # Get the release segment of our versions - left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) - right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) - - # Get the rest of our versions - left_split.append(left[len(left_split[0]) :]) - right_split.append(right[len(right_split[0]) :]) - - # Insert our padding - left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) - right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - - return ( - list(itertools.chain.from_iterable(left_split)), - list(itertools.chain.from_iterable(right_split)), - ) - - -class SpecifierSet(BaseSpecifier): - """This class abstracts handling of a set of version specifiers. - - It can be passed a single specifier (``>=3.0``), a comma-separated list of - specifiers (``>=3.0,!=3.1``), or no specifier at all. - """ - - def __init__( - self, specifiers: str = "", prereleases: Optional[bool] = None - ) -> None: - """Initialize a SpecifierSet instance. - - :param specifiers: - The string representation of a specifier or a comma-separated list of - specifiers which will be parsed and normalized before use. - :param prereleases: - This tells the SpecifierSet if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - - :raises InvalidSpecifier: - If the given ``specifiers`` are not parseable than this exception will be - raised. - """ - - # Split on `,` to break each individual specifier into it's own item, and - # strip each item to remove leading/trailing whitespace. - split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] - - # Make each individual specifier a Specifier and save in a frozen set for later. - self._specs = frozenset(map(Specifier, split_specifiers)) - - # Store our prereleases value so we can use it later to determine if - # we accept prereleases or not. - self._prereleases = prereleases - - @property - def prereleases(self) -> Optional[bool]: - # If we have been given an explicit prerelease modifier, then we'll - # pass that through here. - if self._prereleases is not None: - return self._prereleases - - # If we don't have any specifiers, and we don't have a forced value, - # then we'll just return None since we don't know if this should have - # pre-releases or not. - if not self._specs: - return None - - # Otherwise we'll see if any of the given specifiers accept - # prereleases, if any of them do we'll return True, otherwise False. - return any(s.prereleases for s in self._specs) - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - def __repr__(self) -> str: - """A representation of the specifier set that shows all internal state. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> SpecifierSet('>=1.0.0,!=2.0.0') - <SpecifierSet('!=2.0.0,>=1.0.0')> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=False) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=False)> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=True) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<SpecifierSet({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the specifier set that can be round-tripped. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> str(SpecifierSet(">=1.0.0,!=1.0.1")) - '!=1.0.1,>=1.0.0' - >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) - '!=1.0.1,>=1.0.0' - """ - return ",".join(sorted(str(s) for s in self._specs)) - - def __hash__(self) -> int: - return hash(self._specs) - - def __and__(self, other: Union["SpecifierSet", str]) -> "SpecifierSet": - """Return a SpecifierSet which is a combination of the two sets. - - :param other: The other object to combine with. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") & '<=2.0.0,!=2.0.1' - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - >>> SpecifierSet(">=1.0.0,!=1.0.1") & SpecifierSet('<=2.0.0,!=2.0.1') - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - """ - if isinstance(other, str): - other = SpecifierSet(other) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - specifier = SpecifierSet() - specifier._specs = frozenset(self._specs | other._specs) - - if self._prereleases is None and other._prereleases is not None: - specifier._prereleases = other._prereleases - elif self._prereleases is not None and other._prereleases is None: - specifier._prereleases = self._prereleases - elif self._prereleases == other._prereleases: - specifier._prereleases = self._prereleases - else: - raise ValueError( - "Cannot combine SpecifierSets with True and False prerelease " - "overrides." - ) - - return specifier - - def __eq__(self, other: object) -> bool: - """Whether or not the two SpecifierSet-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> (SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False) == - ... SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True)) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == ">=1.0.0,!=1.0.1" - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.2") - False - """ - if isinstance(other, (str, Specifier)): - other = SpecifierSet(str(other)) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - return self._specs == other._specs - - def __len__(self) -> int: - """Returns the number of specifiers in this specifier set.""" - return len(self._specs) - - def __iter__(self) -> Iterator[Specifier]: - """ - Returns an iterator over all the underlying :class:`Specifier` instances - in this specifier set. - - >>> sorted(SpecifierSet(">=1.0.0,!=1.0.1"), key=str) - [<Specifier('!=1.0.1')>, <Specifier('>=1.0.0')>] - """ - return iter(self._specs) - - def __contains__(self, item: UnparsedVersion) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> Version("1.2.3") in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> "1.0.1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, - item: UnparsedVersion, - prereleases: Optional[bool] = None, - installed: Optional[bool] = None, - ) -> bool: - """Return whether or not the item is contained in this SpecifierSet. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this SpecifierSet. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.2.3") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains(Version("1.2.3")) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.0.1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True).contains("1.3.0a1") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1", prereleases=True) - True - """ - # Ensure that our item is a Version instance. - if not isinstance(item, Version): - item = Version(item) - - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # We can determine if we're going to allow pre-releases by looking to - # see if any of the underlying items supports them. If none of them do - # and this item is a pre-release then we do not allow it and we can - # short circuit that here. - # Note: This means that 1.0.dev1 would not be contained in something - # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 - if not prereleases and item.is_prerelease: - return False - - if installed and item.is_prerelease: - item = Version(item.base_version) - - # We simply dispatch to the underlying specs here to make sure that the - # given version is contained within all of them. - # Note: This use of all() here means that an empty set of specifiers - # will always return True, this is an explicit design decision. - return all(s.contains(item, prereleases=prereleases) for s in self._specs) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifiers in this set. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(SpecifierSet(...).contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", Version("1.4")])) - ['1.3', <Version('1.4')>] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.5a1"])) - [] - >>> list(SpecifierSet(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - - An "empty" SpecifierSet will filter items based on the presence of prerelease - versions in the set. - - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet("").filter(["1.5a1"])) - ['1.5a1'] - >>> list(SpecifierSet("", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - """ - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # If we have any specifiers, then we want to wrap our iterable in the - # filter method for each one, this will act as a logical AND amongst - # each specifier. - if self._specs: - for spec in self._specs: - iterable = spec.filter(iterable, prereleases=bool(prereleases)) - return iter(iterable) - # If we do not have any specifiers, then we need to have a rough filter - # which will filter out any pre-releases, unless there are no final - # releases. - else: - filtered: List[UnparsedVersionVar] = [] - found_prereleases: List[UnparsedVersionVar] = [] - - for item in iterable: - parsed_version = _coerce_version(item) - - # Store any item which is a pre-release for later unless we've - # already found a final version or we are accepting prereleases - if parsed_version.is_prerelease and not prereleases: - if not filtered: - found_prereleases.append(item) - else: - filtered.append(item) - - # If we've found no items except for pre-releases, then we'll go - # ahead and use the pre-releases - if not filtered and found_prereleases and prereleases is None: - return iter(found_prereleases) - - return iter(filtered) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/tags.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/tags.py deleted file mode 100644 index 89f1926137d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/tags.py +++ /dev/null @@ -1,571 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import logging -import platform -import re -import struct -import subprocess -import sys -import sysconfig -from importlib.machinery import EXTENSION_SUFFIXES -from typing import ( - Dict, - FrozenSet, - Iterable, - Iterator, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) - -from . import _manylinux, _musllinux - -logger = logging.getLogger(__name__) - -PythonVersion = Sequence[int] -MacVersion = Tuple[int, int] - -INTERPRETER_SHORT_NAMES: Dict[str, str] = { - "python": "py", # Generic. - "cpython": "cp", - "pypy": "pp", - "ironpython": "ip", - "jython": "jy", -} - - -_32_BIT_INTERPRETER = struct.calcsize("P") == 4 - - -class Tag: - """ - A representation of the tag triple for a wheel. - - Instances are considered immutable and thus are hashable. Equality checking - is also supported. - """ - - __slots__ = ["_interpreter", "_abi", "_platform", "_hash"] - - def __init__(self, interpreter: str, abi: str, platform: str) -> None: - self._interpreter = interpreter.lower() - self._abi = abi.lower() - self._platform = platform.lower() - # The __hash__ of every single element in a Set[Tag] will be evaluated each time - # that a set calls its `.disjoint()` method, which may be called hundreds of - # times when scanning a page of links for packages with tags matching that - # Set[Tag]. Pre-computing the value here produces significant speedups for - # downstream consumers. - self._hash = hash((self._interpreter, self._abi, self._platform)) - - @property - def interpreter(self) -> str: - return self._interpreter - - @property - def abi(self) -> str: - return self._abi - - @property - def platform(self) -> str: - return self._platform - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Tag): - return NotImplemented - - return ( - (self._hash == other._hash) # Short-circuit ASAP for perf reasons. - and (self._platform == other._platform) - and (self._abi == other._abi) - and (self._interpreter == other._interpreter) - ) - - def __hash__(self) -> int: - return self._hash - - def __str__(self) -> str: - return f"{self._interpreter}-{self._abi}-{self._platform}" - - def __repr__(self) -> str: - return f"<{self} @ {id(self)}>" - - -def parse_tag(tag: str) -> FrozenSet[Tag]: - """ - Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. - - Returning a set is required due to the possibility that the tag is a - compressed tag set. - """ - tags = set() - interpreters, abis, platforms = tag.split("-") - for interpreter in interpreters.split("."): - for abi in abis.split("."): - for platform_ in platforms.split("."): - tags.add(Tag(interpreter, abi, platform_)) - return frozenset(tags) - - -def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]: - value: Union[int, str, None] = sysconfig.get_config_var(name) - if value is None and warn: - logger.debug( - "Config variable '%s' is unset, Python ABI tag may be incorrect", name - ) - return value - - -def _normalize_string(string: str) -> str: - return string.replace(".", "_").replace("-", "_").replace(" ", "_") - - -def _is_threaded_cpython(abis: List[str]) -> bool: - """ - Determine if the ABI corresponds to a threaded (`--disable-gil`) build. - - The threaded builds are indicated by a "t" in the abiflags. - """ - if len(abis) == 0: - return False - # expect e.g., cp313 - m = re.match(r"cp\d+(.*)", abis[0]) - if not m: - return False - abiflags = m.group(1) - return "t" in abiflags - - -def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: - """ - Determine if the Python version supports abi3. - - PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) - builds do not support abi3. - """ - return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading - - -def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> List[str]: - py_version = tuple(py_version) # To allow for version comparison. - abis = [] - version = _version_nodot(py_version[:2]) - threading = debug = pymalloc = ucs4 = "" - with_debug = _get_config_var("Py_DEBUG", warn) - has_refcount = hasattr(sys, "gettotalrefcount") - # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled - # extension modules is the best option. - # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 - has_ext = "_d.pyd" in EXTENSION_SUFFIXES - if with_debug or (with_debug is None and (has_refcount or has_ext)): - debug = "d" - if py_version >= (3, 13) and _get_config_var("Py_GIL_DISABLED", warn): - threading = "t" - if py_version < (3, 8): - with_pymalloc = _get_config_var("WITH_PYMALLOC", warn) - if with_pymalloc or with_pymalloc is None: - pymalloc = "m" - if py_version < (3, 3): - unicode_size = _get_config_var("Py_UNICODE_SIZE", warn) - if unicode_size == 4 or ( - unicode_size is None and sys.maxunicode == 0x10FFFF - ): - ucs4 = "u" - elif debug: - # Debug builds can also load "normal" extension modules. - # We can also assume no UCS-4 or pymalloc requirement. - abis.append(f"cp{version}{threading}") - abis.insert(0, f"cp{version}{threading}{debug}{pymalloc}{ucs4}") - return abis - - -def cpython_tags( - python_version: Optional[PythonVersion] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a CPython interpreter. - - The tags consist of: - - cp<python_version>-<abi>-<platform> - - cp<python_version>-abi3-<platform> - - cp<python_version>-none-<platform> - - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. - - If python_version only specifies a major version then user-provided ABIs and - the 'none' ABItag will be used. - - If 'abi3' or 'none' are specified in 'abis' then they will be yielded at - their normal position and not at the beginning. - """ - if not python_version: - python_version = sys.version_info[:2] - - interpreter = f"cp{_version_nodot(python_version[:2])}" - - if abis is None: - if len(python_version) > 1: - abis = _cpython_abis(python_version, warn) - else: - abis = [] - abis = list(abis) - # 'abi3' and 'none' are explicitly handled later. - for explicit_abi in ("abi3", "none"): - try: - abis.remove(explicit_abi) - except ValueError: - pass - - platforms = list(platforms or platform_tags()) - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - threading = _is_threaded_cpython(abis) - use_abi3 = _abi3_applies(python_version, threading) - if use_abi3: - yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) - yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) - - if use_abi3: - for minor_version in range(python_version[1] - 1, 1, -1): - for platform_ in platforms: - interpreter = "cp{version}".format( - version=_version_nodot((python_version[0], minor_version)) - ) - yield Tag(interpreter, "abi3", platform_) - - -def _generic_abi() -> List[str]: - """ - Return the ABI tag based on EXT_SUFFIX. - """ - # The following are examples of `EXT_SUFFIX`. - # We want to keep the parts which are related to the ABI and remove the - # parts which are related to the platform: - # - linux: '.cpython-310-x86_64-linux-gnu.so' => cp310 - # - mac: '.cpython-310-darwin.so' => cp310 - # - win: '.cp310-win_amd64.pyd' => cp310 - # - win: '.pyd' => cp37 (uses _cpython_abis()) - # - pypy: '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73 - # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib' - # => graalpy_38_native - - ext_suffix = _get_config_var("EXT_SUFFIX", warn=True) - if not isinstance(ext_suffix, str) or ext_suffix[0] != ".": - raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')") - parts = ext_suffix.split(".") - if len(parts) < 3: - # CPython3.7 and earlier uses ".pyd" on Windows. - return _cpython_abis(sys.version_info[:2]) - soabi = parts[1] - if soabi.startswith("cpython"): - # non-windows - abi = "cp" + soabi.split("-")[1] - elif soabi.startswith("cp"): - # windows - abi = soabi.split("-")[0] - elif soabi.startswith("pypy"): - abi = "-".join(soabi.split("-")[:2]) - elif soabi.startswith("graalpy"): - abi = "-".join(soabi.split("-")[:3]) - elif soabi: - # pyston, ironpython, others? - abi = soabi - else: - return [] - return [_normalize_string(abi)] - - -def generic_tags( - interpreter: Optional[str] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a generic interpreter. - - The tags consist of: - - <interpreter>-<abi>-<platform> - - The "none" ABI will be added if it was not explicitly provided. - """ - if not interpreter: - interp_name = interpreter_name() - interp_version = interpreter_version(warn=warn) - interpreter = "".join([interp_name, interp_version]) - if abis is None: - abis = _generic_abi() - else: - abis = list(abis) - platforms = list(platforms or platform_tags()) - if "none" not in abis: - abis.append("none") - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - -def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]: - """ - Yields Python versions in descending order. - - After the latest version, the major-only version will be yielded, and then - all previous versions of that major version. - """ - if len(py_version) > 1: - yield f"py{_version_nodot(py_version[:2])}" - yield f"py{py_version[0]}" - if len(py_version) > 1: - for minor in range(py_version[1] - 1, -1, -1): - yield f"py{_version_nodot((py_version[0], minor))}" - - -def compatible_tags( - python_version: Optional[PythonVersion] = None, - interpreter: Optional[str] = None, - platforms: Optional[Iterable[str]] = None, -) -> Iterator[Tag]: - """ - Yields the sequence of tags that are compatible with a specific version of Python. - - The tags consist of: - - py*-none-<platform> - - <interpreter>-none-any # ... if `interpreter` is provided. - - py*-none-any - """ - if not python_version: - python_version = sys.version_info[:2] - platforms = list(platforms or platform_tags()) - for version in _py_interpreter_range(python_version): - for platform_ in platforms: - yield Tag(version, "none", platform_) - if interpreter: - yield Tag(interpreter, "none", "any") - for version in _py_interpreter_range(python_version): - yield Tag(version, "none", "any") - - -def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str: - if not is_32bit: - return arch - - if arch.startswith("ppc"): - return "ppc" - - return "i386" - - -def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> List[str]: - formats = [cpu_arch] - if cpu_arch == "x86_64": - if version < (10, 4): - return [] - formats.extend(["intel", "fat64", "fat32"]) - - elif cpu_arch == "i386": - if version < (10, 4): - return [] - formats.extend(["intel", "fat32", "fat"]) - - elif cpu_arch == "ppc64": - # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? - if version > (10, 5) or version < (10, 4): - return [] - formats.append("fat64") - - elif cpu_arch == "ppc": - if version > (10, 6): - return [] - formats.extend(["fat32", "fat"]) - - if cpu_arch in {"arm64", "x86_64"}: - formats.append("universal2") - - if cpu_arch in {"x86_64", "i386", "ppc64", "ppc", "intel"}: - formats.append("universal") - - return formats - - -def mac_platforms( - version: Optional[MacVersion] = None, arch: Optional[str] = None -) -> Iterator[str]: - """ - Yields the platform tags for a macOS system. - - The `version` parameter is a two-item tuple specifying the macOS version to - generate platform tags for. The `arch` parameter is the CPU architecture to - generate platform tags for. Both parameters default to the appropriate value - for the current system. - """ - version_str, _, cpu_arch = platform.mac_ver() - if version is None: - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - if version == (10, 16): - # When built against an older macOS SDK, Python will report macOS 10.16 - # instead of the real version. - version_str = subprocess.run( - [ - sys.executable, - "-sS", - "-c", - "import platform; print(platform.mac_ver()[0])", - ], - check=True, - env={"SYSTEM_VERSION_COMPAT": "0"}, - stdout=subprocess.PIPE, - text=True, - ).stdout - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - else: - version = version - if arch is None: - arch = _mac_arch(cpu_arch) - else: - arch = arch - - if (10, 0) <= version and version < (11, 0): - # Prior to Mac OS 11, each yearly release of Mac OS bumped the - # "minor" version number. The major version was always 10. - for minor_version in range(version[1], -1, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=10, minor=minor_version, binary_format=binary_format - ) - - if version >= (11, 0): - # Starting with Mac OS 11, each yearly release bumps the major version - # number. The minor versions are now the midyear updates. - for major_version in range(version[0], 10, -1): - compat_version = major_version, 0 - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=major_version, minor=0, binary_format=binary_format - ) - - if version >= (11, 0): - # Mac OS 11 on x86_64 is compatible with binaries from previous releases. - # Arm64 support was introduced in 11.0, so no Arm binaries from previous - # releases exist. - # - # However, the "universal2" binary format can have a - # macOS version earlier than 11.0 when the x86_64 part of the binary supports - # that version of macOS. - if arch == "x86_64": - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - else: - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_format = "universal2" - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - - -def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: - linux = _normalize_string(sysconfig.get_platform()) - if not linux.startswith("linux_"): - # we should never be here, just yield the sysconfig one and return - yield linux - return - if is_32bit: - if linux == "linux_x86_64": - linux = "linux_i686" - elif linux == "linux_aarch64": - linux = "linux_armv8l" - _, arch = linux.split("_", 1) - archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch]) - yield from _manylinux.platform_tags(archs) - yield from _musllinux.platform_tags(archs) - for arch in archs: - yield f"linux_{arch}" - - -def _generic_platforms() -> Iterator[str]: - yield _normalize_string(sysconfig.get_platform()) - - -def platform_tags() -> Iterator[str]: - """ - Provides the platform tags for this installation. - """ - if platform.system() == "Darwin": - return mac_platforms() - elif platform.system() == "Linux": - return _linux_platforms() - else: - return _generic_platforms() - - -def interpreter_name() -> str: - """ - Returns the name of the running interpreter. - - Some implementations have a reserved, two-letter abbreviation which will - be returned when appropriate. - """ - name = sys.implementation.name - return INTERPRETER_SHORT_NAMES.get(name) or name - - -def interpreter_version(*, warn: bool = False) -> str: - """ - Returns the version of the running interpreter. - """ - version = _get_config_var("py_version_nodot", warn=warn) - if version: - version = str(version) - else: - version = _version_nodot(sys.version_info[:2]) - return version - - -def _version_nodot(version: PythonVersion) -> str: - return "".join(map(str, version)) - - -def sys_tags(*, warn: bool = False) -> Iterator[Tag]: - """ - Returns the sequence of tag triples for the running interpreter. - - The order of the sequence corresponds to priority order for the - interpreter, from most to least important. - """ - - interp_name = interpreter_name() - if interp_name == "cp": - yield from cpython_tags(warn=warn) - else: - yield from generic_tags() - - if interp_name == "pp": - interp = "pp3" - elif interp_name == "cp": - interp = "cp" + interpreter_version(warn=warn) - else: - interp = None - yield from compatible_tags(interpreter=interp) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/utils.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/utils.py deleted file mode 100644 index c2c2f75aa80..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/utils.py +++ /dev/null @@ -1,172 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import re -from typing import FrozenSet, NewType, Tuple, Union, cast - -from .tags import Tag, parse_tag -from .version import InvalidVersion, Version - -BuildTag = Union[Tuple[()], Tuple[int, str]] -NormalizedName = NewType("NormalizedName", str) - - -class InvalidName(ValueError): - """ - An invalid distribution name; users should refer to the packaging user guide. - """ - - -class InvalidWheelFilename(ValueError): - """ - An invalid wheel filename was found, users should refer to PEP 427. - """ - - -class InvalidSdistFilename(ValueError): - """ - An invalid sdist filename was found, users should refer to the packaging user guide. - """ - - -# Core metadata spec for `Name` -_validate_regex = re.compile( - r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE -) -_canonicalize_regex = re.compile(r"[-_.]+") -_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") -# PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") - - -def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: - if validate and not _validate_regex.match(name): - raise InvalidName(f"name is invalid: {name!r}") - # This is taken from PEP 503. - value = _canonicalize_regex.sub("-", name).lower() - return cast(NormalizedName, value) - - -def is_normalized_name(name: str) -> bool: - return _normalized_regex.match(name) is not None - - -def canonicalize_version( - version: Union[Version, str], *, strip_trailing_zero: bool = True -) -> str: - """ - This is very similar to Version.__str__, but has one subtle difference - with the way it handles the release segment. - """ - if isinstance(version, str): - try: - parsed = Version(version) - except InvalidVersion: - # Legacy versions cannot be normalized - return version - else: - parsed = version - - parts = [] - - # Epoch - if parsed.epoch != 0: - parts.append(f"{parsed.epoch}!") - - # Release segment - release_segment = ".".join(str(x) for x in parsed.release) - if strip_trailing_zero: - # NB: This strips trailing '.0's to normalize - release_segment = re.sub(r"(\.0)+$", "", release_segment) - parts.append(release_segment) - - # Pre-release - if parsed.pre is not None: - parts.append("".join(str(x) for x in parsed.pre)) - - # Post-release - if parsed.post is not None: - parts.append(f".post{parsed.post}") - - # Development release - if parsed.dev is not None: - parts.append(f".dev{parsed.dev}") - - # Local version segment - if parsed.local is not None: - parts.append(f"+{parsed.local}") - - return "".join(parts) - - -def parse_wheel_filename( - filename: str, -) -> Tuple[NormalizedName, Version, BuildTag, FrozenSet[Tag]]: - if not filename.endswith(".whl"): - raise InvalidWheelFilename( - f"Invalid wheel filename (extension must be '.whl'): {filename}" - ) - - filename = filename[:-4] - dashes = filename.count("-") - if dashes not in (4, 5): - raise InvalidWheelFilename( - f"Invalid wheel filename (wrong number of parts): {filename}" - ) - - parts = filename.split("-", dashes - 2) - name_part = parts[0] - # See PEP 427 for the rules on escaping the project name. - if "__" in name_part or re.match(r"^[\w\d._]*$", name_part, re.UNICODE) is None: - raise InvalidWheelFilename(f"Invalid project name: {filename}") - name = canonicalize_name(name_part) - - try: - version = Version(parts[1]) - except InvalidVersion as e: - raise InvalidWheelFilename( - f"Invalid wheel filename (invalid version): {filename}" - ) from e - - if dashes == 5: - build_part = parts[2] - build_match = _build_tag_regex.match(build_part) - if build_match is None: - raise InvalidWheelFilename( - f"Invalid build number: {build_part} in '{filename}'" - ) - build = cast(BuildTag, (int(build_match.group(1)), build_match.group(2))) - else: - build = () - tags = parse_tag(parts[-1]) - return (name, version, build, tags) - - -def parse_sdist_filename(filename: str) -> Tuple[NormalizedName, Version]: - if filename.endswith(".tar.gz"): - file_stem = filename[: -len(".tar.gz")] - elif filename.endswith(".zip"): - file_stem = filename[: -len(".zip")] - else: - raise InvalidSdistFilename( - f"Invalid sdist filename (extension must be '.tar.gz' or '.zip'):" - f" {filename}" - ) - - # We are requiring a PEP 440 version, which cannot contain dashes, - # so we split on the last dash. - name_part, sep, version_part = file_stem.rpartition("-") - if not sep: - raise InvalidSdistFilename(f"Invalid sdist filename: {filename}") - - name = canonicalize_name(name_part) - - try: - version = Version(version_part) - except InvalidVersion as e: - raise InvalidSdistFilename( - f"Invalid sdist filename (invalid version): {filename}" - ) from e - - return (name, version) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/version.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/version.py deleted file mode 100644 index 5faab9bd0dc..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/version.py +++ /dev/null @@ -1,563 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.version import parse, Version -""" - -import itertools -import re -from typing import Any, Callable, NamedTuple, Optional, SupportsInt, Tuple, Union - -from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType - -__all__ = ["VERSION_PATTERN", "parse", "Version", "InvalidVersion"] - -LocalType = Tuple[Union[int, str], ...] - -CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] -CmpLocalType = Union[ - NegativeInfinityType, - Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], -] -CmpKey = Tuple[ - int, - Tuple[int, ...], - CmpPrePostDevType, - CmpPrePostDevType, - CmpPrePostDevType, - CmpLocalType, -] -VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] - - -class _Version(NamedTuple): - epoch: int - release: Tuple[int, ...] - dev: Optional[Tuple[str, int]] - pre: Optional[Tuple[str, int]] - post: Optional[Tuple[str, int]] - local: Optional[LocalType] - - -def parse(version: str) -> "Version": - """Parse the given version string. - - >>> parse('1.0.dev1') - <Version('1.0.dev1')> - - :param version: The version string to parse. - :raises InvalidVersion: When the version string is not a valid version. - """ - return Version(version) - - -class InvalidVersion(ValueError): - """Raised when a version string is not a valid version. - - >>> Version("invalid") - Traceback (most recent call last): - ... - packaging.version.InvalidVersion: Invalid version: 'invalid' - """ - - -class _BaseVersion: - _key: Tuple[Any, ...] - - def __hash__(self) -> int: - return hash(self._key) - - # Please keep the duplicated `isinstance` check - # in the six comparisons hereunder - # unless you find a way to avoid adding overhead function calls. - def __lt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key < other._key - - def __le__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key <= other._key - - def __eq__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key == other._key - - def __ge__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key >= other._key - - def __gt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key > other._key - - def __ne__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key != other._key - - -# Deliberately not anchored to the start and end of the string, to make it -# easier for 3rd party code to reuse -_VERSION_PATTERN = r""" - v? - (?: - (?:(?P<epoch>[0-9]+)!)? # epoch - (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment - (?P<pre> # pre-release - [-_\.]? - (?P<pre_l>alpha|a|beta|b|preview|pre|c|rc) - [-_\.]? - (?P<pre_n>[0-9]+)? - )? - (?P<post> # post release - (?:-(?P<post_n1>[0-9]+)) - | - (?: - [-_\.]? - (?P<post_l>post|rev|r) - [-_\.]? - (?P<post_n2>[0-9]+)? - ) - )? - (?P<dev> # dev release - [-_\.]? - (?P<dev_l>dev) - [-_\.]? - (?P<dev_n>[0-9]+)? - )? - ) - (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version -""" - -VERSION_PATTERN = _VERSION_PATTERN -""" -A string containing the regular expression used to match a valid version. - -The pattern is not anchored at either end, and is intended for embedding in larger -expressions (for example, matching a version number as part of a file name). The -regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` -flags set. - -:meta hide-value: -""" - - -class Version(_BaseVersion): - """This class abstracts handling of a project's versions. - - A :class:`Version` instance is comparison aware and can be compared and - sorted using the standard Python interfaces. - - >>> v1 = Version("1.0a5") - >>> v2 = Version("1.0") - >>> v1 - <Version('1.0a5')> - >>> v2 - <Version('1.0')> - >>> v1 < v2 - True - >>> v1 == v2 - False - >>> v1 > v2 - False - >>> v1 >= v2 - False - >>> v1 <= v2 - True - """ - - _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) - _key: CmpKey - - def __init__(self, version: str) -> None: - """Initialize a Version object. - - :param version: - The string representation of a version which will be parsed and normalized - before use. - :raises InvalidVersion: - If the ``version`` does not conform to PEP 440 in any way then this - exception will be raised. - """ - - # Validate the version and parse it into pieces - match = self._regex.search(version) - if not match: - raise InvalidVersion(f"Invalid version: '{version}'") - - # Store the parsed out pieces of the version - self._version = _Version( - epoch=int(match.group("epoch")) if match.group("epoch") else 0, - release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), - post=_parse_letter_version( - match.group("post_l"), match.group("post_n1") or match.group("post_n2") - ), - dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), - local=_parse_local_version(match.group("local")), - ) - - # Generate a key which will be used for sorting - self._key = _cmpkey( - self._version.epoch, - self._version.release, - self._version.pre, - self._version.post, - self._version.dev, - self._version.local, - ) - - def __repr__(self) -> str: - """A representation of the Version that shows all internal state. - - >>> Version('1.0.0') - <Version('1.0.0')> - """ - return f"<Version('{self}')>" - - def __str__(self) -> str: - """A string representation of the version that can be rounded-tripped. - - >>> str(Version("1.0a5")) - '1.0a5' - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - # Pre-release - if self.pre is not None: - parts.append("".join(str(x) for x in self.pre)) - - # Post-release - if self.post is not None: - parts.append(f".post{self.post}") - - # Development release - if self.dev is not None: - parts.append(f".dev{self.dev}") - - # Local version segment - if self.local is not None: - parts.append(f"+{self.local}") - - return "".join(parts) - - @property - def epoch(self) -> int: - """The epoch of the version. - - >>> Version("2.0.0").epoch - 0 - >>> Version("1!2.0.0").epoch - 1 - """ - return self._version.epoch - - @property - def release(self) -> Tuple[int, ...]: - """The components of the "release" segment of the version. - - >>> Version("1.2.3").release - (1, 2, 3) - >>> Version("2.0.0").release - (2, 0, 0) - >>> Version("1!2.0.0.post0").release - (2, 0, 0) - - Includes trailing zeroes but not the epoch or any pre-release / development / - post-release suffixes. - """ - return self._version.release - - @property - def pre(self) -> Optional[Tuple[str, int]]: - """The pre-release segment of the version. - - >>> print(Version("1.2.3").pre) - None - >>> Version("1.2.3a1").pre - ('a', 1) - >>> Version("1.2.3b1").pre - ('b', 1) - >>> Version("1.2.3rc1").pre - ('rc', 1) - """ - return self._version.pre - - @property - def post(self) -> Optional[int]: - """The post-release number of the version. - - >>> print(Version("1.2.3").post) - None - >>> Version("1.2.3.post1").post - 1 - """ - return self._version.post[1] if self._version.post else None - - @property - def dev(self) -> Optional[int]: - """The development number of the version. - - >>> print(Version("1.2.3").dev) - None - >>> Version("1.2.3.dev1").dev - 1 - """ - return self._version.dev[1] if self._version.dev else None - - @property - def local(self) -> Optional[str]: - """The local version segment of the version. - - >>> print(Version("1.2.3").local) - None - >>> Version("1.2.3+abc").local - 'abc' - """ - if self._version.local: - return ".".join(str(x) for x in self._version.local) - else: - return None - - @property - def public(self) -> str: - """The public portion of the version. - - >>> Version("1.2.3").public - '1.2.3' - >>> Version("1.2.3+abc").public - '1.2.3' - >>> Version("1.2.3+abc.dev1").public - '1.2.3' - """ - return str(self).split("+", 1)[0] - - @property - def base_version(self) -> str: - """The "base version" of the version. - - >>> Version("1.2.3").base_version - '1.2.3' - >>> Version("1.2.3+abc").base_version - '1.2.3' - >>> Version("1!1.2.3+abc.dev1").base_version - '1!1.2.3' - - The "base version" is the public version of the project without any pre or post - release markers. - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - return "".join(parts) - - @property - def is_prerelease(self) -> bool: - """Whether this version is a pre-release. - - >>> Version("1.2.3").is_prerelease - False - >>> Version("1.2.3a1").is_prerelease - True - >>> Version("1.2.3b1").is_prerelease - True - >>> Version("1.2.3rc1").is_prerelease - True - >>> Version("1.2.3dev1").is_prerelease - True - """ - return self.dev is not None or self.pre is not None - - @property - def is_postrelease(self) -> bool: - """Whether this version is a post-release. - - >>> Version("1.2.3").is_postrelease - False - >>> Version("1.2.3.post1").is_postrelease - True - """ - return self.post is not None - - @property - def is_devrelease(self) -> bool: - """Whether this version is a development release. - - >>> Version("1.2.3").is_devrelease - False - >>> Version("1.2.3.dev1").is_devrelease - True - """ - return self.dev is not None - - @property - def major(self) -> int: - """The first item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").major - 1 - """ - return self.release[0] if len(self.release) >= 1 else 0 - - @property - def minor(self) -> int: - """The second item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").minor - 2 - >>> Version("1").minor - 0 - """ - return self.release[1] if len(self.release) >= 2 else 0 - - @property - def micro(self) -> int: - """The third item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").micro - 3 - >>> Version("1").micro - 0 - """ - return self.release[2] if len(self.release) >= 3 else 0 - - -def _parse_letter_version( - letter: Optional[str], number: Union[str, bytes, SupportsInt, None] -) -> Optional[Tuple[str, int]]: - - if letter: - # We consider there to be an implicit 0 in a pre-release if there is - # not a numeral associated with it. - if number is None: - number = 0 - - # We normalize any letters to their lower case form - letter = letter.lower() - - # We consider some words to be alternate spellings of other words and - # in those cases we want to normalize the spellings to our preferred - # spelling. - if letter == "alpha": - letter = "a" - elif letter == "beta": - letter = "b" - elif letter in ["c", "pre", "preview"]: - letter = "rc" - elif letter in ["rev", "r"]: - letter = "post" - - return letter, int(number) - if not letter and number: - # We assume if we are given a number, but we are not given a letter - # then this is using the implicit post release syntax (e.g. 1.0-1) - letter = "post" - - return letter, int(number) - - return None - - -_local_version_separators = re.compile(r"[\._-]") - - -def _parse_local_version(local: Optional[str]) -> Optional[LocalType]: - """ - Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). - """ - if local is not None: - return tuple( - part.lower() if not part.isdigit() else int(part) - for part in _local_version_separators.split(local) - ) - return None - - -def _cmpkey( - epoch: int, - release: Tuple[int, ...], - pre: Optional[Tuple[str, int]], - post: Optional[Tuple[str, int]], - dev: Optional[Tuple[str, int]], - local: Optional[LocalType], -) -> CmpKey: - - # When we compare a release version, we want to compare it with all of the - # trailing zeros removed. So we'll use a reverse the list, drop all the now - # leading zeros until we come to something non zero, then take the rest - # re-reverse it back into the correct order and make it a tuple and use - # that for our sorting key. - _release = tuple( - reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) - ) - - # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. - # We'll do this by abusing the pre segment, but we _only_ want to do this - # if there is not a pre or a post segment. If we have one of those then - # the normal sorting rules will handle this case correctly. - if pre is None and post is None and dev is not None: - _pre: CmpPrePostDevType = NegativeInfinity - # Versions without a pre-release (except as noted above) should sort after - # those with one. - elif pre is None: - _pre = Infinity - else: - _pre = pre - - # Versions without a post segment should sort before those with one. - if post is None: - _post: CmpPrePostDevType = NegativeInfinity - - else: - _post = post - - # Versions without a development segment should sort after those with one. - if dev is None: - _dev: CmpPrePostDevType = Infinity - - else: - _dev = dev - - if local is None: - # Versions without a local segment should sort before those with one. - _local: CmpLocalType = NegativeInfinity - else: - # Versions with a local segment need that segment parsed to implement - # the sorting rules in PEP440. - # - Alpha numeric segments sort before numeric segments - # - Alpha numeric segments sort lexicographically - # - Numeric segments sort numerically - # - Shorter versions sort before longer versions when the prefixes - # match exactly - _local = tuple( - (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local - ) - - return epoch, _release, _pre, _post, _dev, _local diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py deleted file mode 100644 index aef2821b83f..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -Utilities for determining application-specific dirs. See <https://github.com/platformdirs/platformdirs> for details and -usage. -""" -from __future__ import annotations - -import os -import sys -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no cover (py38+) - from typing import Literal -else: # pragma: no cover (py38+) - from ..typing_extensions import Literal - -from .api import PlatformDirsABC -from .version import __version__ -from .version import __version_tuple__ as __version_info__ - - -def _set_platform_dir_class() -> type[PlatformDirsABC]: - if sys.platform == "win32": - from .windows import Windows as Result - elif sys.platform == "darwin": - from .macos import MacOS as Result - else: - from .unix import Unix as Result - - if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system": - - if os.getenv("SHELL") or os.getenv("PREFIX"): - return Result - - from .android import _android_folder - - if _android_folder() is not None: - from .android import Android - - return Android # return to avoid redefinition of result - - return Result - - -PlatformDirs = _set_platform_dir_class() #: Currently active platform -AppDirs = PlatformDirs #: Backwards compatibility with appdirs - - -def user_data_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: data directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_dir - - -def site_data_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: data directory shared by users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_dir - - -def user_config_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: config directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_dir - - -def site_config_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: config directory shared by the users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_dir - - -def user_cache_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: cache directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_dir - - -def user_state_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: state directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_dir - - -def user_log_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: log directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_dir - - -def user_documents_dir() -> str: - """ - :returns: documents directory tied to the user - """ - return PlatformDirs().user_documents_dir - - -def user_runtime_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: runtime directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_dir - - -def user_data_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: data path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_path - - -def site_data_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `multipath <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: data path shared by users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_path - - -def user_config_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: config path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_path - - -def site_config_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: config path shared by the users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_path - - -def user_cache_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: cache path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_path - - -def user_state_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: state path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_path - - -def user_log_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: log path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_path - - -def user_documents_path() -> Path: - """ - :returns: documents path tied to the user - """ - return PlatformDirs().user_documents_path - - -def user_runtime_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: runtime path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_path - - -__all__ = [ - "__version__", - "__version_info__", - "PlatformDirs", - "AppDirs", - "PlatformDirsABC", - "user_data_dir", - "user_config_dir", - "user_cache_dir", - "user_state_dir", - "user_log_dir", - "user_documents_dir", - "user_runtime_dir", - "site_data_dir", - "site_config_dir", - "user_data_path", - "user_config_path", - "user_cache_path", - "user_state_path", - "user_log_path", - "user_documents_path", - "user_runtime_path", - "site_data_path", - "site_config_path", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py deleted file mode 100644 index 0fc1edd59cf..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from platformdirs import PlatformDirs, __version__ - -PROPS = ( - "user_data_dir", - "user_config_dir", - "user_cache_dir", - "user_state_dir", - "user_log_dir", - "user_documents_dir", - "user_runtime_dir", - "site_data_dir", - "site_config_dir", -) - - -def main() -> None: - app_name = "MyApp" - app_author = "MyCompany" - - print(f"-- platformdirs {__version__} --") - - print("-- app dirs (with optional 'version')") - dirs = PlatformDirs(app_name, app_author, version="1.0") - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (without optional 'version')") - dirs = PlatformDirs(app_name, app_author) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (without optional 'appauthor')") - dirs = PlatformDirs(app_name) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (with disabled 'appauthor')") - dirs = PlatformDirs(app_name, appauthor=False) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - -if __name__ == "__main__": - main() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py deleted file mode 100644 index eda80935123..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py +++ /dev/null @@ -1,120 +0,0 @@ -from __future__ import annotations - -import os -import re -import sys -from functools import lru_cache -from typing import cast - -from .api import PlatformDirsABC - - -class Android(PlatformDirsABC): - """ - Follows the guidance `from here <https://android.stackexchange.com/a/216132>`_. Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>` and - `version <platformdirs.api.PlatformDirsABC.version>`. - """ - - @property - def user_data_dir(self) -> str: - """:return: data directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/files/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "files") - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_config_dir(self) -> str: - """ - :return: config directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/shared_prefs/<AppName>`` - """ - return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, same as `user_config_dir`""" - return self.user_config_dir - - @property - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user, e.g. e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "cache") - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_cache_dir` if not opinionated else ``log`` in it, - e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/log`` - """ - path = self.user_cache_dir - if self.opinion: - path = os.path.join(path, "log") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents`` - """ - return _android_documents_folder() - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, same as `user_cache_dir` if not opinionated else ``tmp`` in it, - e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/tmp`` - """ - path = self.user_cache_dir - if self.opinion: - path = os.path.join(path, "tmp") - return path - - -@lru_cache(maxsize=1) -def _android_folder() -> str | None: - """:return: base folder for the Android OS or None if cannot be found""" - try: - # First try to get path to android app via pyjnius - from jnius import autoclass - - Context = autoclass("android.content.Context") # noqa: N806 - result: str | None = Context.getFilesDir().getParentFile().getAbsolutePath() - except Exception: - # if fails find an android folder looking path on the sys.path - pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files") - for path in sys.path: - if pattern.match(path): - result = path.split("/files")[0] - break - else: - result = None - return result - - -@lru_cache(maxsize=1) -def _android_documents_folder() -> str: - """:return: documents folder for the Android OS""" - # Get directories with pyjnius - try: - from jnius import autoclass - - Context = autoclass("android.content.Context") # noqa: N806 - Environment = autoclass("android.os.Environment") # noqa: N806 - documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath() - except Exception: - documents_dir = "/storage/emulated/0/Documents" - - return documents_dir - - -__all__ = [ - "Android", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py deleted file mode 100644 index 6f6e2c2c69d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import os -import sys -from abc import ABC, abstractmethod -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no branch - from typing import Literal # pragma: no cover - - -class PlatformDirsABC(ABC): - """ - Abstract base class for platform directories. - """ - - def __init__( - self, - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, - multipath: bool = False, - opinion: bool = True, - ): - """ - Create a new platform directory. - - :param appname: See `appname`. - :param appauthor: See `appauthor`. - :param version: See `version`. - :param roaming: See `roaming`. - :param multipath: See `multipath`. - :param opinion: See `opinion`. - """ - self.appname = appname #: The name of application. - self.appauthor = appauthor - """ - The name of the app author or distributing body for this application. Typically, it is the owning company name. - Defaults to `appname`. You may pass ``False`` to disable it. - """ - self.version = version - """ - An optional version path element to append to the path. You might want to use this if you want multiple versions - of your app to be able to run independently. If used, this would typically be ``<major>.<minor>``. - """ - self.roaming = roaming - """ - Whether to use the roaming appdata directory on Windows. That means that for users on a Windows network setup - for roaming profiles, this user data will be synced on login (see - `here <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>`_). - """ - self.multipath = multipath - """ - An optional parameter only applicable to Unix/Linux which indicates that the entire list of data dirs should be - returned. By default, the first item would only be returned. - """ - self.opinion = opinion #: A flag to indicating to use opinionated values. - - def _append_app_name_and_version(self, *base: str) -> str: - params = list(base[1:]) - if self.appname: - params.append(self.appname) - if self.version: - params.append(self.version) - return os.path.join(base[0], *params) - - @property - @abstractmethod - def user_data_dir(self) -> str: - """:return: data directory tied to the user""" - - @property - @abstractmethod - def site_data_dir(self) -> str: - """:return: data directory shared by users""" - - @property - @abstractmethod - def user_config_dir(self) -> str: - """:return: config directory tied to the user""" - - @property - @abstractmethod - def site_config_dir(self) -> str: - """:return: config directory shared by the users""" - - @property - @abstractmethod - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user""" - - @property - @abstractmethod - def user_state_dir(self) -> str: - """:return: state directory tied to the user""" - - @property - @abstractmethod - def user_log_dir(self) -> str: - """:return: log directory tied to the user""" - - @property - @abstractmethod - def user_documents_dir(self) -> str: - """:return: documents directory tied to the user""" - - @property - @abstractmethod - def user_runtime_dir(self) -> str: - """:return: runtime directory tied to the user""" - - @property - def user_data_path(self) -> Path: - """:return: data path tied to the user""" - return Path(self.user_data_dir) - - @property - def site_data_path(self) -> Path: - """:return: data path shared by users""" - return Path(self.site_data_dir) - - @property - def user_config_path(self) -> Path: - """:return: config path tied to the user""" - return Path(self.user_config_dir) - - @property - def site_config_path(self) -> Path: - """:return: config path shared by the users""" - return Path(self.site_config_dir) - - @property - def user_cache_path(self) -> Path: - """:return: cache path tied to the user""" - return Path(self.user_cache_dir) - - @property - def user_state_path(self) -> Path: - """:return: state path tied to the user""" - return Path(self.user_state_dir) - - @property - def user_log_path(self) -> Path: - """:return: log path tied to the user""" - return Path(self.user_log_dir) - - @property - def user_documents_path(self) -> Path: - """:return: documents path tied to the user""" - return Path(self.user_documents_dir) - - @property - def user_runtime_path(self) -> Path: - """:return: runtime path tied to the user""" - return Path(self.user_runtime_dir) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py deleted file mode 100644 index a01337c7764..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import os - -from .api import PlatformDirsABC - - -class MacOS(PlatformDirsABC): - """ - Platform directories for the macOS operating system. Follows the guidance from `Apple documentation - <https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html>`_. - Makes use of the `appname <platformdirs.api.PlatformDirsABC.appname>` and - `version <platformdirs.api.PlatformDirsABC.version>`. - """ - - @property - def user_data_dir(self) -> str: - """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support/")) - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, e.g. ``/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version("/Library/Application Support") - - @property - def user_config_dir(self) -> str: - """:return: config directory tied to the user, e.g. ``~/Library/Preferences/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Preferences/")) - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, e.g. ``/Library/Preferences/$appname``""" - return self._append_app_name_and_version("/Library/Preferences") - - @property - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) - - @property - def user_documents_dir(self) -> str: - """:return: documents directory tied to the user, e.g. ``~/Documents``""" - return os.path.expanduser("~/Documents") - - @property - def user_runtime_dir(self) -> str: - """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) - - -__all__ = [ - "MacOS", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/py.typed b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py deleted file mode 100644 index 9aca5a03054..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import os -import sys -from configparser import ConfigParser -from pathlib import Path - -from .api import PlatformDirsABC - -if sys.platform.startswith("linux"): # pragma: no branch # no op check, only to please the type checker - from os import getuid -else: - - def getuid() -> int: - raise RuntimeError("should only be used on Linux") - - -class Unix(PlatformDirsABC): - """ - On Unix/Linux, we follow the - `XDG Basedir Spec <https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html>`_. The spec allows - overriding directories with environment variables. The examples show are the default values, alongside the name of - the environment variable that overrides them. Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>`, - `version <platformdirs.api.PlatformDirsABC.version>`, - `multipath <platformdirs.api.PlatformDirsABC.multipath>`, - `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - """ - - @property - def user_data_dir(self) -> str: - """ - :return: data directory tied to the user, e.g. ``~/.local/share/$appname/$version`` or - ``$XDG_DATA_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_DATA_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.local/share") - return self._append_app_name_and_version(path) - - @property - def site_data_dir(self) -> str: - """ - :return: data directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` is - enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS - path separator), e.g. ``/usr/local/share/$appname/$version`` or ``/usr/share/$appname/$version`` - """ - # XDG default for $XDG_DATA_DIRS; only first, if multipath is False - path = os.environ.get("XDG_DATA_DIRS", "") - if not path.strip(): - path = f"/usr/local/share{os.pathsep}/usr/share" - return self._with_multi_path(path) - - def _with_multi_path(self, path: str) -> str: - path_list = path.split(os.pathsep) - if not self.multipath: - path_list = path_list[0:1] - path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] - return os.pathsep.join(path_list) - - @property - def user_config_dir(self) -> str: - """ - :return: config directory tied to the user, e.g. ``~/.config/$appname/$version`` or - ``$XDG_CONFIG_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_CONFIG_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.config") - return self._append_app_name_and_version(path) - - @property - def site_config_dir(self) -> str: - """ - :return: config directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` - is enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS - path separator), e.g. ``/etc/xdg/$appname/$version`` - """ - # XDG default for $XDG_CONFIG_DIRS only first, if multipath is False - path = os.environ.get("XDG_CONFIG_DIRS", "") - if not path.strip(): - path = "/etc/xdg" - return self._with_multi_path(path) - - @property - def user_cache_dir(self) -> str: - """ - :return: cache directory tied to the user, e.g. ``~/.cache/$appname/$version`` or - ``~/$XDG_CACHE_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_CACHE_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.cache") - return self._append_app_name_and_version(path) - - @property - def user_state_dir(self) -> str: - """ - :return: state directory tied to the user, e.g. ``~/.local/state/$appname/$version`` or - ``$XDG_STATE_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_STATE_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.local/state") - return self._append_app_name_and_version(path) - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it - """ - path = self.user_state_dir - if self.opinion: - path = os.path.join(path, "log") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user, e.g. ``~/Documents`` - """ - documents_dir = _get_user_dirs_folder("XDG_DOCUMENTS_DIR") - if documents_dir is None: - documents_dir = os.environ.get("XDG_DOCUMENTS_DIR", "").strip() - if not documents_dir: - documents_dir = os.path.expanduser("~/Documents") - - return documents_dir - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or - ``$XDG_RUNTIME_DIR/$appname/$version`` - """ - path = os.environ.get("XDG_RUNTIME_DIR", "") - if not path.strip(): - path = f"/run/user/{getuid()}" - return self._append_app_name_and_version(path) - - @property - def site_data_path(self) -> Path: - """:return: data path shared by users. Only return first item, even if ``multipath`` is set to ``True``""" - return self._first_item_as_path_if_multipath(self.site_data_dir) - - @property - def site_config_path(self) -> Path: - """:return: config path shared by the users. Only return first item, even if ``multipath`` is set to ``True``""" - return self._first_item_as_path_if_multipath(self.site_config_dir) - - def _first_item_as_path_if_multipath(self, directory: str) -> Path: - if self.multipath: - # If multipath is True, the first path is returned. - directory = directory.split(os.pathsep)[0] - return Path(directory) - - -def _get_user_dirs_folder(key: str) -> str | None: - """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/""" - user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs") - if os.path.exists(user_dirs_config_path): - parser = ConfigParser() - - with open(user_dirs_config_path) as stream: - # Add fake section header, so ConfigParser doesn't complain - parser.read_string(f"[top]\n{stream.read()}") - - if key not in parser["top"]: - return None - - path = parser["top"][key].strip('"') - # Handle relative home paths - path = path.replace("$HOME", os.path.expanduser("~")) - return path - - return None - - -__all__ = [ - "Unix", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py deleted file mode 100644 index 9f6eb98e8f0..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py +++ /dev/null @@ -1,4 +0,0 @@ -# file generated by setuptools_scm -# don't change, don't track in version control -__version__ = version = '2.6.2' -__version_tuple__ = version_tuple = (2, 6, 2) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py deleted file mode 100644 index d5c27b34140..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py +++ /dev/null @@ -1,184 +0,0 @@ -from __future__ import annotations - -import ctypes -import os -import sys -from functools import lru_cache -from typing import Callable - -from .api import PlatformDirsABC - - -class Windows(PlatformDirsABC): - """`MSDN on where to store app data files - <http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120>`_. - Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>`, - `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`, - `version <platformdirs.api.PlatformDirsABC.version>`, - `roaming <platformdirs.api.PlatformDirsABC.roaming>`, - `opinion <platformdirs.api.PlatformDirsABC.opinion>`.""" - - @property - def user_data_dir(self) -> str: - """ - :return: data directory tied to the user, e.g. - ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname`` (not roaming) or - ``%USERPROFILE%\\AppData\\Roaming\\$appauthor\\$appname`` (roaming) - """ - const = "CSIDL_APPDATA" if self.roaming else "CSIDL_LOCAL_APPDATA" - path = os.path.normpath(get_win_folder(const)) - return self._append_parts(path) - - def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str: - params = [] - if self.appname: - if self.appauthor is not False: - author = self.appauthor or self.appname - params.append(author) - params.append(self.appname) - if opinion_value is not None and self.opinion: - params.append(opinion_value) - if self.version: - params.append(self.version) - return os.path.join(path, *params) - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, e.g. ``C:\\ProgramData\\$appauthor\\$appname``""" - path = os.path.normpath(get_win_folder("CSIDL_COMMON_APPDATA")) - return self._append_parts(path) - - @property - def user_config_dir(self) -> str: - """:return: config directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, same as `site_data_dir`""" - return self.site_data_dir - - @property - def user_cache_dir(self) -> str: - """ - :return: cache directory tied to the user (if opinionated with ``Cache`` folder within ``$appname``) e.g. - ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname\\Cache\\$version`` - """ - path = os.path.normpath(get_win_folder("CSIDL_LOCAL_APPDATA")) - return self._append_parts(path, opinion_value="Cache") - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it - """ - path = self.user_data_dir - if self.opinion: - path = os.path.join(path, "Logs") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents`` - """ - return os.path.normpath(get_win_folder("CSIDL_PERSONAL")) - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, e.g. - ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname`` - """ - path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) - return self._append_parts(path) - - -def get_win_folder_from_env_vars(csidl_name: str) -> str: - """Get folder from environment variables.""" - if csidl_name == "CSIDL_PERSONAL": # does not have an environment name - return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") - - env_var_name = { - "CSIDL_APPDATA": "APPDATA", - "CSIDL_COMMON_APPDATA": "ALLUSERSPROFILE", - "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA", - }.get(csidl_name) - if env_var_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - result = os.environ.get(env_var_name) - if result is None: - raise ValueError(f"Unset environment variable: {env_var_name}") - return result - - -def get_win_folder_from_registry(csidl_name: str) -> str: - """Get folder from the registry. - - This is a fallback technique at best. I'm not sure if using the - registry for this guarantees us the correct answer for all CSIDL_* - names. - """ - shell_folder_name = { - "CSIDL_APPDATA": "AppData", - "CSIDL_COMMON_APPDATA": "Common AppData", - "CSIDL_LOCAL_APPDATA": "Local AppData", - "CSIDL_PERSONAL": "Personal", - }.get(csidl_name) - if shell_folder_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - if sys.platform != "win32": # only needed for mypy type checker to know that this code runs only on Windows - raise NotImplementedError - import winreg - - key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") - directory, _ = winreg.QueryValueEx(key, shell_folder_name) - return str(directory) - - -def get_win_folder_via_ctypes(csidl_name: str) -> str: - """Get folder with ctypes.""" - csidl_const = { - "CSIDL_APPDATA": 26, - "CSIDL_COMMON_APPDATA": 35, - "CSIDL_LOCAL_APPDATA": 28, - "CSIDL_PERSONAL": 5, - }.get(csidl_name) - if csidl_const is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - - buf = ctypes.create_unicode_buffer(1024) - windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker - windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) - - # Downgrade to short path name if it has highbit chars. - if any(ord(c) > 255 for c in buf): - buf2 = ctypes.create_unicode_buffer(1024) - if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): - buf = buf2 - - return buf.value - - -def _pick_get_win_folder() -> Callable[[str], str]: - if hasattr(ctypes, "windll"): - return get_win_folder_via_ctypes - try: - import winreg # noqa: F401 - except ImportError: - return get_win_folder_from_env_vars - else: - return get_win_folder_from_registry - - -get_win_folder = lru_cache(maxsize=None)(_pick_get_win_folder()) - -__all__ = [ - "Windows", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py deleted file mode 100644 index 26b723c1fd3..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py +++ /dev/null @@ -1,329 +0,0 @@ -import io -import posixpath -import zipfile -import itertools -import contextlib -import sys -import pathlib - -if sys.version_info < (3, 7): - from collections import OrderedDict -else: - OrderedDict = dict - - -__all__ = ['Path'] - - -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = OrderedDict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(zipfile.ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, zipfile.ZipFile): - return cls(_pathlib_compat(source)) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -def _pathlib_compat(path): - """ - For path-like objects, convert to a filename for compatibility - on Python 3.6.1 and earlier. - """ - try: - return path.__fspath__() - except AttributeError: - return str(path) - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = zipfile.ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix - - @property - def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes - - @property - def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *map(_pathlib_compat, other)) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) diff --git a/contrib/python/setuptools/py3/pkg_resources/api_tests.txt b/contrib/python/setuptools/py3/pkg_resources/api_tests.txt new file mode 100644 index 00000000000..d72b85aa375 --- /dev/null +++ b/contrib/python/setuptools/py3/pkg_resources/api_tests.txt @@ -0,0 +1,424 @@ +Pluggable Distributions of Python Software +========================================== + +Distributions +------------- + +A "Distribution" is a collection of files that represent a "Release" of a +"Project" as of a particular point in time, denoted by a +"Version":: + + >>> import sys, pkg_resources + >>> from pkg_resources import Distribution + >>> Distribution(project_name="Foo", version="1.2") + Foo 1.2 + +Distributions have a location, which can be a filename, URL, or really anything +else you care to use:: + + >>> dist = Distribution( + ... location="http://example.com/something", + ... project_name="Bar", version="0.9" + ... ) + + >>> dist + Bar 0.9 (http://example.com/something) + + +Distributions have various introspectable attributes:: + + >>> dist.location + 'http://example.com/something' + + >>> dist.project_name + 'Bar' + + >>> dist.version + '0.9' + + >>> dist.py_version == '{}.{}'.format(*sys.version_info) + True + + >>> print(dist.platform) + None + +Including various computed attributes:: + + >>> from pkg_resources import parse_version + >>> dist.parsed_version == parse_version(dist.version) + True + + >>> dist.key # case-insensitive form of the project name + 'bar' + +Distributions are compared (and hashed) by version first:: + + >>> Distribution(version='1.0') == Distribution(version='1.0') + True + >>> Distribution(version='1.0') == Distribution(version='1.1') + False + >>> Distribution(version='1.0') < Distribution(version='1.1') + True + +but also by project name (case-insensitive), platform, Python version, +location, etc.:: + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="Foo",version="1.0") + True + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="foo",version="1.0") + True + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="Foo",version="1.1") + False + + >>> Distribution(project_name="Foo",py_version="2.3",version="1.0") == \ + ... Distribution(project_name="Foo",py_version="2.4",version="1.0") + False + + >>> Distribution(location="spam",version="1.0") == \ + ... Distribution(location="spam",version="1.0") + True + + >>> Distribution(location="spam",version="1.0") == \ + ... Distribution(location="baz",version="1.0") + False + + + +Hash and compare distribution by prio/plat + +Get version from metadata +provider capabilities +egg_name() +as_requirement() +from_location, from_filename (w/path normalization) + +Releases may have zero or more "Requirements", which indicate +what releases of another project the release requires in order to +function. A Requirement names the other project, expresses some criteria +as to what releases of that project are acceptable, and lists any "Extras" +that the requiring release may need from that project. (An Extra is an +optional feature of a Release, that can only be used if its additional +Requirements are satisfied.) + + + +The Working Set +--------------- + +A collection of active distributions is called a Working Set. Note that a +Working Set can contain any importable distribution, not just pluggable ones. +For example, the Python standard library is an importable distribution that +will usually be part of the Working Set, even though it is not pluggable. +Similarly, when you are doing development work on a project, the files you are +editing are also a Distribution. (And, with a little attention to the +directory names used, and including some additional metadata, such a +"development distribution" can be made pluggable as well.) + + >>> from pkg_resources import WorkingSet + +A working set's entries are the sys.path entries that correspond to the active +distributions. By default, the working set's entries are the items on +``sys.path``:: + + >>> ws = WorkingSet() + >>> ws.entries == sys.path + True + +But you can also create an empty working set explicitly, and add distributions +to it:: + + >>> ws = WorkingSet([]) + >>> ws.add(dist) + >>> ws.entries + ['http://example.com/something'] + >>> dist in ws + True + >>> Distribution('foo',version="") in ws + False + +And you can iterate over its distributions:: + + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +Adding the same distribution more than once is a no-op:: + + >>> ws.add(dist) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +For that matter, adding multiple distributions for the same project also does +nothing, because a working set can only hold one active distribution per +project -- the first one added to it:: + + >>> ws.add( + ... Distribution( + ... 'http://example.com/something', project_name="Bar", + ... version="7.2" + ... ) + ... ) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +You can append a path entry to a working set using ``add_entry()``:: + + >>> ws.entries + ['http://example.com/something'] + >>> ws.add_entry(pkg_resources.__file__) + >>> ws.entries + ['http://example.com/something', '...pkg_resources...'] + +Multiple additions result in multiple entries, even if the entry is already in +the working set (because ``sys.path`` can contain the same entry more than +once):: + + >>> ws.add_entry(pkg_resources.__file__) + >>> ws.entries + ['...example.com...', '...pkg_resources...', '...pkg_resources...'] + +And you can specify the path entry a distribution was found under, using the +optional second parameter to ``add()``:: + + >>> ws = WorkingSet([]) + >>> ws.add(dist,"foo") + >>> ws.entries + ['foo'] + +But even if a distribution is found under multiple path entries, it still only +shows up once when iterating the working set: + + >>> ws.add_entry(ws.entries[0]) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +You can ask a WorkingSet to ``find()`` a distribution matching a requirement:: + + >>> from pkg_resources import Requirement + >>> print(ws.find(Requirement.parse("Foo==1.0"))) # no match, return None + None + + >>> ws.find(Requirement.parse("Bar==0.9")) # match, return distribution + Bar 0.9 (http://example.com/something) + +Note that asking for a conflicting version of a distribution already in a +working set triggers a ``pkg_resources.VersionConflict`` error: + + >>> try: + ... ws.find(Requirement.parse("Bar==1.0")) + ... except pkg_resources.VersionConflict as exc: + ... print(str(exc)) + ... else: + ... raise AssertionError("VersionConflict was not raised") + (Bar 0.9 (http://example.com/something), Requirement.parse('Bar==1.0')) + +You can subscribe a callback function to receive notifications whenever a new +distribution is added to a working set. The callback is immediately invoked +once for each existing distribution in the working set, and then is called +again for new distributions added thereafter:: + + >>> def added(dist): print("Added %s" % dist) + >>> ws.subscribe(added) + Added Bar 0.9 + >>> foo12 = Distribution(project_name="Foo", version="1.2", location="f12") + >>> ws.add(foo12) + Added Foo 1.2 + +Note, however, that only the first distribution added for a given project name +will trigger a callback, even during the initial ``subscribe()`` callback:: + + >>> foo14 = Distribution(project_name="Foo", version="1.4", location="f14") + >>> ws.add(foo14) # no callback, because Foo 1.2 is already active + + >>> ws = WorkingSet([]) + >>> ws.add(foo12) + >>> ws.add(foo14) + >>> ws.subscribe(added) + Added Foo 1.2 + +And adding a callback more than once has no effect, either:: + + >>> ws.subscribe(added) # no callbacks + + # and no double-callbacks on subsequent additions, either + >>> just_a_test = Distribution(project_name="JustATest", version="0.99") + >>> ws.add(just_a_test) + Added JustATest 0.99 + + +Finding Plugins +--------------- + +``WorkingSet`` objects can be used to figure out what plugins in an +``Environment`` can be loaded without any resolution errors:: + + >>> from pkg_resources import Environment + + >>> plugins = Environment([]) # normally, a list of plugin directories + >>> plugins.add(foo12) + >>> plugins.add(foo14) + >>> plugins.add(just_a_test) + +In the simplest case, we just get the newest version of each distribution in +the plugin environment:: + + >>> ws = WorkingSet([]) + >>> ws.find_plugins(plugins) + ([JustATest 0.99, Foo 1.4 (f14)], {}) + +But if there's a problem with a version conflict or missing requirements, the +method falls back to older versions, and the error info dict will contain an +exception instance for each unloadable plugin:: + + >>> ws.add(foo12) # this will conflict with Foo 1.4 + >>> ws.find_plugins(plugins) + ([JustATest 0.99, Foo 1.2 (f12)], {Foo 1.4 (f14): VersionConflict(...)}) + +But if you disallow fallbacks, the failed plugin will be skipped instead of +trying older versions:: + + >>> ws.find_plugins(plugins, fallback=False) + ([JustATest 0.99], {Foo 1.4 (f14): VersionConflict(...)}) + + + +Platform Compatibility Rules +---------------------------- + +On the Mac, there are potential compatibility issues for modules compiled +on newer versions of macOS than what the user is running. Additionally, +macOS will soon have two platforms to contend with: Intel and PowerPC. + +Basic equality works as on other platforms:: + + >>> from pkg_resources import compatible_platforms as cp + >>> reqd = 'macosx-10.4-ppc' + >>> cp(reqd, reqd) + True + >>> cp("win32", reqd) + False + +Distributions made on other machine types are not compatible:: + + >>> cp("macosx-10.4-i386", reqd) + False + +Distributions made on earlier versions of the OS are compatible, as +long as they are from the same top-level version. The patchlevel version +number does not matter:: + + >>> cp("macosx-10.4-ppc", reqd) + True + >>> cp("macosx-10.3-ppc", reqd) + True + >>> cp("macosx-10.5-ppc", reqd) + False + >>> cp("macosx-9.5-ppc", reqd) + False + +Backwards compatibility for packages made via earlier versions of +setuptools is provided as well:: + + >>> cp("darwin-8.2.0-Power_Macintosh", reqd) + True + >>> cp("darwin-7.2.0-Power_Macintosh", reqd) + True + >>> cp("darwin-8.2.0-Power_Macintosh", "macosx-10.3-ppc") + False + + +Environment Markers +------------------- + + >>> from pkg_resources import invalid_marker as im, evaluate_marker as em + >>> import os + + >>> print(im("sys_platform")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + sys_platform + ^ + + >>> print(im("sys_platform==")) + Expected a marker variable or quoted string + sys_platform== + ^ + + >>> print(im("sys_platform=='win32'")) + False + + >>> print(im("sys=='x'")) + Expected a marker variable or quoted string + sys=='x' + ^ + + >>> print(im("(extra)")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + (extra) + ^ + + >>> print(im("(extra")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + (extra + ^ + + >>> print(im("os.open('foo')=='y'")) + Expected a marker variable or quoted string + os.open('foo')=='y' + ^ + + >>> print(im("'x'=='y' and os.open('foo')=='y'")) # no short-circuit! + Expected a marker variable or quoted string + 'x'=='y' and os.open('foo')=='y' + ^ + + >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! + Expected a marker variable or quoted string + 'x'=='x' or os.open('foo')=='y' + ^ + + >>> print(im("r'x'=='x'")) + Expected a marker variable or quoted string + r'x'=='x' + ^ + + >>> print(im("'''x'''=='x'")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + '''x'''=='x' + ^ + + >>> print(im('"""x"""=="x"')) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + """x"""=="x" + ^ + + >>> print(im(r"x\n=='x'")) + Expected a marker variable or quoted string + x\n=='x' + ^ + + >>> print(im("os.open=='y'")) + Expected a marker variable or quoted string + os.open=='y' + ^ + + >>> em("sys_platform=='win32'") == (sys.platform=='win32') + True + + >>> em("python_version >= '2.7'") + True + + >>> em("python_version > '2.6'") + True + + >>> im("implementation_name=='cpython'") + False + + >>> im("platform_python_implementation=='CPython'") + False + + >>> im("implementation_version=='3.5.1'") + False diff --git a/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py b/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py deleted file mode 100644 index daa978ff728..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations -from importlib.machinery import ModuleSpec -import importlib.util -import sys -from types import ModuleType -from typing import Iterable, Sequence - - -class VendorImporter: - """ - A PEP 302 meta path importer for finding optionally-vendored - or otherwise naturally-installed packages from root_name. - """ - - def __init__( - self, - root_name: str, - vendored_names: Iterable[str] = (), - vendor_pkg: str | None = None, - ): - self.root_name = root_name - self.vendored_names = set(vendored_names) - self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') - - @property - def search_path(self): - """ - Search first the vendor package then as a natural package. - """ - yield self.vendor_pkg + '.' - yield '' - - def _module_matches_namespace(self, fullname): - """Figure out if the target module is vendored.""" - root, base, target = fullname.partition(self.root_name + '.') - return not root and any(map(target.startswith, self.vendored_names)) - - def load_module(self, fullname: str): - """ - Iterate over the search path to locate and load fullname. - """ - root, base, target = fullname.partition(self.root_name + '.') - for prefix in self.search_path: - extant = prefix + target - try: - __import__(extant) - except ImportError: - continue - mod = sys.modules[extant] - sys.modules[fullname] = mod - return mod - else: - raise ImportError( - "The '{target}' package is required; " - "normally this is bundled with this package so if you get " - "this warning, consult the packager of your " - "distribution.".format(**locals()) - ) - - def create_module(self, spec: ModuleSpec): - return self.load_module(spec.name) - - def exec_module(self, module: ModuleType): - pass - - def find_spec( - self, - fullname: str, - path: Sequence[str] | None = None, - target: ModuleType | None = None, - ): - """Return a module spec for vendored names.""" - return ( - # This should fix itself next mypy release https://github.com/python/typeshed/pull/11890 - importlib.util.spec_from_loader(fullname, self) # type: ignore[arg-type] - if self._module_matches_namespace(fullname) - else None - ) - - def install(self): - """ - Install this importer into sys.meta_path if not already present. - """ - if self not in sys.meta_path: - sys.meta_path.append(self) - - -# [[[cog -# import cog -# from tools.vendored import yield_top_level -# names = "\n".join(f" {x!r}," for x in yield_top_level('pkg_resources')) -# cog.outl(f"names = (\n{names}\n)") -# ]]] -names = ( - 'backports', - 'importlib_resources', - 'jaraco', - 'more_itertools', - 'packaging', - 'platformdirs', - 'zipp', -) -# [[[end]]] -VendorImporter(__name__, names).install() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/py.typed b/contrib/python/setuptools/py3/pkg_resources/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/py.typed +++ b/contrib/python/setuptools/py3/pkg_resources/py.typed diff --git a/contrib/python/setuptools/py3/setuptools/__init__.py b/contrib/python/setuptools/py3/setuptools/__init__.py index bf03f37b779..afca08be9c5 100644 --- a/contrib/python/setuptools/py3/setuptools/__init__.py +++ b/contrib/python/setuptools/py3/setuptools/__init__.py @@ -3,8 +3,13 @@ import functools import os import re +import sys from typing import TYPE_CHECKING +sys.path.extend(((vendor_path := os.path.join(os.path.dirname(os.path.dirname(__file__)), 'setuptools', '_vendor')) not in sys.path) * [vendor_path]) # fmt: skip +# workaround for #4476 +sys.modules.pop('backports', None) + import _distutils_hack.override # noqa: F401 import distutils.core from distutils.errors import DistutilsOptionError diff --git a/contrib/python/setuptools/py3/setuptools/_core_metadata.py b/contrib/python/setuptools/py3/setuptools/_core_metadata.py index 45aae7d70be..82ec19fc755 100644 --- a/contrib/python/setuptools/py3/setuptools/_core_metadata.py +++ b/contrib/python/setuptools/py3/setuptools/_core_metadata.py @@ -16,10 +16,10 @@ from tempfile import NamedTemporaryFile from distutils.util import rfc822_escape from . import _normalization, _reqs -from .extern.packaging.markers import Marker -from .extern.packaging.requirements import Requirement -from .extern.packaging.utils import canonicalize_name, canonicalize_version -from .extern.packaging.version import Version +from packaging.markers import Marker +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name, canonicalize_version +from packaging.version import Version from .warnings import SetuptoolsDeprecationWarning diff --git a/contrib/python/setuptools/py3/setuptools/_entry_points.py b/contrib/python/setuptools/py3/setuptools/_entry_points.py index b244e78387b..5de12582beb 100644 --- a/contrib/python/setuptools/py3/setuptools/_entry_points.py +++ b/contrib/python/setuptools/py3/setuptools/_entry_points.py @@ -3,11 +3,11 @@ import operator import itertools from .errors import OptionError -from .extern.jaraco.text import yield_lines -from .extern.jaraco.functools import pass_none +from jaraco.text import yield_lines +from jaraco.functools import pass_none from ._importlib import metadata from ._itertools import ensure_unique -from .extern.more_itertools import consume +from more_itertools import consume def ensure_valid(ep): diff --git a/contrib/python/setuptools/py3/setuptools/_importlib.py b/contrib/python/setuptools/py3/setuptools/_importlib.py index 8e52888d6f7..b2d5b5b84af 100644 --- a/contrib/python/setuptools/py3/setuptools/_importlib.py +++ b/contrib/python/setuptools/py3/setuptools/_importlib.py @@ -1,51 +1,13 @@ import sys -def disable_importlib_metadata_finder(metadata): - """ - Ensure importlib_metadata doesn't provide older, incompatible - Distributions. - - Workaround for #3102. - """ - try: - import importlib_metadata - except ImportError: - return - except AttributeError: - from .warnings import SetuptoolsWarning - - SetuptoolsWarning.emit( - "Incompatibility problem.", - """ - `importlib-metadata` version is incompatible with `setuptools`. - This problem is likely to be solved by installing an updated version of - `importlib-metadata`. - """, - see_url="https://github.com/python/importlib_metadata/issues/396", - ) # Ensure a descriptive message is shown. - raise # This exception can be suppressed by _distutils_hack - - if importlib_metadata is metadata: - return - to_remove = [ - ob - for ob in sys.meta_path - if isinstance(ob, importlib_metadata.MetadataPathFinder) - ] - for item in to_remove: - sys.meta_path.remove(item) - - if sys.version_info < (3, 10): - from setuptools.extern import importlib_metadata as metadata - - disable_importlib_metadata_finder(metadata) + import importlib_metadata as metadata # pragma: no cover else: - import importlib.metadata as metadata + import importlib.metadata as metadata # noqa: F401 if sys.version_info < (3, 9): - from setuptools.extern import importlib_resources as resources + import importlib_resources as resources # pragma: no cover else: import importlib.resources as resources # noqa: F401 diff --git a/contrib/python/setuptools/py3/setuptools/_itertools.py b/contrib/python/setuptools/py3/setuptools/_itertools.py index b8bf6d210ae..d6ca841353c 100644 --- a/contrib/python/setuptools/py3/setuptools/_itertools.py +++ b/contrib/python/setuptools/py3/setuptools/_itertools.py @@ -1,4 +1,4 @@ -from setuptools.extern.more_itertools import consume # noqa: F401 +from more_itertools import consume # noqa: F401 # copied from jaraco.itertools 6.1 diff --git a/contrib/python/setuptools/py3/setuptools/_normalization.py b/contrib/python/setuptools/py3/setuptools/_normalization.py index e858052ccd3..467b643d463 100644 --- a/contrib/python/setuptools/py3/setuptools/_normalization.py +++ b/contrib/python/setuptools/py3/setuptools/_normalization.py @@ -5,7 +5,7 @@ and core metadata import re -from .extern import packaging +import packaging # https://packaging.python.org/en/latest/specifications/core-metadata/#name _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I) @@ -54,7 +54,7 @@ def safe_version(version: str) -> str: >>> safe_version("ubuntu lts") Traceback (most recent call last): ... - setuptools.extern.packaging.version.InvalidVersion: Invalid version: 'ubuntu.lts' + packaging.version.InvalidVersion: Invalid version: 'ubuntu.lts' """ v = version.replace(' ', '.') try: diff --git a/contrib/python/setuptools/py3/setuptools/_reqs.py b/contrib/python/setuptools/py3/setuptools/_reqs.py index 9f83437033c..1b64d9df792 100644 --- a/contrib/python/setuptools/py3/setuptools/_reqs.py +++ b/contrib/python/setuptools/py3/setuptools/_reqs.py @@ -1,8 +1,8 @@ from functools import lru_cache from typing import Callable, Iterable, Iterator, TypeVar, Union, overload -import setuptools.extern.jaraco.text as text -from setuptools.extern.packaging.requirements import Requirement +import jaraco.text as text +from packaging.requirements import Requirement _T = TypeVar("_T") _StrOrIter = Union[str, Iterable[str]] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py b/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py deleted file mode 100644 index a7a9a6e7b94..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py +++ /dev/null @@ -1,2900 +0,0 @@ -#!/usr/bin/env python3 -#------------------------------------------------------------------- -# tarfile.py -#------------------------------------------------------------------- -# Copyright (C) 2002 Lars Gustaebel <[email protected]> -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -"""Read from and write to tar format archives. -""" - -version = "0.9.0" -__author__ = "Lars Gust\u00e4bel ([email protected])" -__credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." - -#--------- -# Imports -#--------- -from builtins import open as bltn_open -import sys -import os -import io -import shutil -import stat -import time -import struct -import copy -import re -import warnings - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -# os.symlink on Windows prior to 6.0 raises NotImplementedError -# OSError (winerror=1314) will be raised if the caller does not hold the -# SeCreateSymbolicLinkPrivilege privilege -symlink_exception = (AttributeError, NotImplementedError, OSError) - -# from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", - "CompressionError", "StreamError", "ExtractError", "HeaderError", - "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", - "tar_filter", "FilterError", "AbsoluteLinkError", - "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] - - -#--------------------------------------------------------- -# tar constants -#--------------------------------------------------------- -NUL = b"\0" # the null character -BLOCKSIZE = 512 # length of processing blocks -RECORDSIZE = BLOCKSIZE * 20 # length of records -GNU_MAGIC = b"ustar \0" # magic gnu tar string -POSIX_MAGIC = b"ustar\x0000" # magic posix tar string - -LENGTH_NAME = 100 # maximum length of a filename -LENGTH_LINK = 100 # maximum length of a linkname -LENGTH_PREFIX = 155 # maximum length of the prefix field - -REGTYPE = b"0" # regular file -AREGTYPE = b"\0" # regular file -LNKTYPE = b"1" # link (inside tarfile) -SYMTYPE = b"2" # symbolic link -CHRTYPE = b"3" # character special device -BLKTYPE = b"4" # block special device -DIRTYPE = b"5" # directory -FIFOTYPE = b"6" # fifo special device -CONTTYPE = b"7" # contiguous file - -GNUTYPE_LONGNAME = b"L" # GNU tar longname -GNUTYPE_LONGLINK = b"K" # GNU tar longlink -GNUTYPE_SPARSE = b"S" # GNU tar sparse file - -XHDTYPE = b"x" # POSIX.1-2001 extended header -XGLTYPE = b"g" # POSIX.1-2001 global header -SOLARIS_XHDTYPE = b"X" # Solaris extended header - -USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format -GNU_FORMAT = 1 # GNU tar format -PAX_FORMAT = 2 # POSIX.1-2001 (pax) format -DEFAULT_FORMAT = PAX_FORMAT - -#--------------------------------------------------------- -# tarfile constants -#--------------------------------------------------------- -# File types that tarfile supports: -SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, - SYMTYPE, DIRTYPE, FIFOTYPE, - CONTTYPE, CHRTYPE, BLKTYPE, - GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# File types that will be treated as a regular file. -REGULAR_TYPES = (REGTYPE, AREGTYPE, - CONTTYPE, GNUTYPE_SPARSE) - -# File types that are part of the GNU tar format. -GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# Fields from a pax header that override a TarInfo attribute. -PAX_FIELDS = ("path", "linkpath", "size", "mtime", - "uid", "gid", "uname", "gname") - -# Fields from a pax header that are affected by hdrcharset. -PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"} - -# Fields in a pax header that are numbers, all other fields -# are treated as strings. -PAX_NUMBER_FIELDS = { - "atime": float, - "ctime": float, - "mtime": float, - "uid": int, - "gid": int, - "size": int -} - -#--------------------------------------------------------- -# initialization -#--------------------------------------------------------- -if os.name == "nt": - ENCODING = "utf-8" -else: - ENCODING = sys.getfilesystemencoding() - -#--------------------------------------------------------- -# Some useful functions -#--------------------------------------------------------- - -def stn(s, length, encoding, errors): - """Convert a string to a null-terminated bytes object. - """ - if s is None: - raise ValueError("metadata cannot contain None") - s = s.encode(encoding, errors) - return s[:length] + (length - len(s)) * NUL - -def nts(s, encoding, errors): - """Convert a null-terminated bytes object to a string. - """ - p = s.find(b"\0") - if p != -1: - s = s[:p] - return s.decode(encoding, errors) - -def nti(s): - """Convert a number field to a python number. - """ - # There are two possible encodings for a number field, see - # itn() below. - if s[0] in (0o200, 0o377): - n = 0 - for i in range(len(s) - 1): - n <<= 8 - n += s[i + 1] - if s[0] == 0o377: - n = -(256 ** (len(s) - 1) - n) - else: - try: - s = nts(s, "ascii", "strict") - n = int(s.strip() or "0", 8) - except ValueError: - raise InvalidHeaderError("invalid header") - return n - -def itn(n, digits=8, format=DEFAULT_FORMAT): - """Convert a python number to a number field. - """ - # POSIX 1003.1-1988 requires numbers to be encoded as a string of - # octal digits followed by a null-byte, this allows values up to - # (8**(digits-1))-1. GNU tar allows storing numbers greater than - # that if necessary. A leading 0o200 or 0o377 byte indicate this - # particular encoding, the following digits-1 bytes are a big-endian - # base-256 representation. This allows values up to (256**(digits-1))-1. - # A 0o200 byte indicates a positive number, a 0o377 byte a negative - # number. - original_n = n - n = int(n) - if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL - elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): - if n >= 0: - s = bytearray([0o200]) - else: - s = bytearray([0o377]) - n = 256 ** digits + n - - for i in range(digits - 1): - s.insert(1, n & 0o377) - n >>= 8 - else: - raise ValueError("overflow in number field") - - return s - -def calc_chksums(buf): - """Calculate the checksum for a member's header by summing up all - characters except for the chksum field which is treated as if - it was filled with spaces. According to the GNU tar sources, - some tars (Sun and NeXT) calculate chksum with signed char, - which will be different if there are chars in the buffer with - the high bit set. So we calculate two checksums, unsigned and - signed. - """ - unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf)) - signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf)) - return unsigned_chksum, signed_chksum - -def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): - """Copy length bytes from fileobj src to fileobj dst. - If length is None, copy the entire content. - """ - bufsize = bufsize or 16 * 1024 - if length == 0: - return - if length is None: - shutil.copyfileobj(src, dst, bufsize) - return - - blocks, remainder = divmod(length, bufsize) - for b in range(blocks): - buf = src.read(bufsize) - if len(buf) < bufsize: - raise exception("unexpected end of data") - dst.write(buf) - - if remainder != 0: - buf = src.read(remainder) - if len(buf) < remainder: - raise exception("unexpected end of data") - dst.write(buf) - return - -def _safe_print(s): - encoding = getattr(sys.stdout, 'encoding', None) - if encoding is not None: - s = s.encode(encoding, 'backslashreplace').decode(encoding) - print(s, end=' ') - - -class TarError(Exception): - """Base exception.""" - pass -class ExtractError(TarError): - """General exception for extract errors.""" - pass -class ReadError(TarError): - """Exception for unreadable tar archives.""" - pass -class CompressionError(TarError): - """Exception for unavailable compression methods.""" - pass -class StreamError(TarError): - """Exception for unsupported operations on stream-like TarFiles.""" - pass -class HeaderError(TarError): - """Base exception for header errors.""" - pass -class EmptyHeaderError(HeaderError): - """Exception for empty headers.""" - pass -class TruncatedHeaderError(HeaderError): - """Exception for truncated headers.""" - pass -class EOFHeaderError(HeaderError): - """Exception for end of file headers.""" - pass -class InvalidHeaderError(HeaderError): - """Exception for invalid headers.""" - pass -class SubsequentHeaderError(HeaderError): - """Exception for missing and invalid extended headers.""" - pass - -#--------------------------- -# internal stream interface -#--------------------------- -class _LowLevelFile: - """Low-level file object. Supports reading and writing. - It is used instead of a regular file object for streaming - access. - """ - - def __init__(self, name, mode): - mode = { - "r": os.O_RDONLY, - "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - }[mode] - if hasattr(os, "O_BINARY"): - mode |= os.O_BINARY - self.fd = os.open(name, mode, 0o666) - - def close(self): - os.close(self.fd) - - def read(self, size): - return os.read(self.fd, size) - - def write(self, s): - os.write(self.fd, s) - -class _Stream: - """Class that serves as an adapter between TarFile and - a stream-like object. The stream-like object only - needs to have a read() or write() method that works with bytes, - and the method is accessed blockwise. - Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin.buffer, - sys.stdout.buffer, a socket, a tape device etc. - - _Stream is intended to be used only internally. - """ - - def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel): - """Construct a _Stream object. - """ - self._extfileobj = True - if fileobj is None: - fileobj = _LowLevelFile(name, mode) - self._extfileobj = False - - if comptype == '*': - # Enable transparent compression detection for the - # stream interface - fileobj = _StreamProxy(fileobj) - comptype = fileobj.getcomptype() - - self.name = name or "" - self.mode = mode - self.comptype = comptype - self.fileobj = fileobj - self.bufsize = bufsize - self.buf = b"" - self.pos = 0 - self.closed = False - - try: - if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") from None - self.zlib = zlib - self.crc = zlib.crc32(b"") - if mode == "r": - self.exception = zlib.error - self._init_read_gz() - else: - self._init_write_gz(compresslevel) - - elif comptype == "bz2": - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = bz2.BZ2Decompressor() - self.exception = OSError - else: - self.cmp = bz2.BZ2Compressor(compresslevel) - - elif comptype == "xz": - try: - import lzma - except ImportError: - raise CompressionError("lzma module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = lzma.LZMADecompressor() - self.exception = lzma.LZMAError - else: - self.cmp = lzma.LZMACompressor() - - elif comptype != "tar": - raise CompressionError("unknown compression type %r" % comptype) - - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - def __del__(self): - if hasattr(self, "closed") and not self.closed: - self.close() - - def _init_write_gz(self, compresslevel): - """Initialize for writing with gzip compression. - """ - self.cmp = self.zlib.compressobj(compresslevel, - self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) - timestamp = struct.pack("<L", int(time.time())) - self.__write(b"\037\213\010\010" + timestamp + b"\002\377") - if self.name.endswith(".gz"): - self.name = self.name[:-3] - # Honor "directory components removed" from RFC1952 - self.name = os.path.basename(self.name) - # RFC1952 says we must use ISO-8859-1 for the FNAME field. - self.__write(self.name.encode("iso-8859-1", "replace") + NUL) - - def write(self, s): - """Write string s to the stream. - """ - if self.comptype == "gz": - self.crc = self.zlib.crc32(s, self.crc) - self.pos += len(s) - if self.comptype != "tar": - s = self.cmp.compress(s) - self.__write(s) - - def __write(self, s): - """Write string s to the stream if a whole new block - is ready to be written. - """ - self.buf += s - while len(self.buf) > self.bufsize: - self.fileobj.write(self.buf[:self.bufsize]) - self.buf = self.buf[self.bufsize:] - - def close(self): - """Close the _Stream object. No operation should be - done on it afterwards. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode == "w" and self.comptype != "tar": - self.buf += self.cmp.flush() - - if self.mode == "w" and self.buf: - self.fileobj.write(self.buf) - self.buf = b"" - if self.comptype == "gz": - self.fileobj.write(struct.pack("<L", self.crc)) - self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def _init_read_gz(self): - """Initialize for reading a gzip compressed fileobj. - """ - self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" - - # taken from gzip.GzipFile with some alterations - if self.__read(2) != b"\037\213": - raise ReadError("not a gzip file") - if self.__read(1) != b"\010": - raise CompressionError("unsupported compression method") - - flag = ord(self.__read(1)) - self.__read(6) - - if flag & 4: - xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) - if flag & 8: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 16: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 2: - self.__read(2) - - def tell(self): - """Return the stream's file pointer position. - """ - return self.pos - - def seek(self, pos=0): - """Set the stream's file pointer to pos. Negative seeking - is forbidden. - """ - if pos - self.pos >= 0: - blocks, remainder = divmod(pos - self.pos, self.bufsize) - for i in range(blocks): - self.read(self.bufsize) - self.read(remainder) - else: - raise StreamError("seeking backwards is not allowed") - return self.pos - - def read(self, size): - """Return the next size number of bytes from the stream.""" - assert size is not None - buf = self._read(size) - self.pos += len(buf) - return buf - - def _read(self, size): - """Return size bytes from the stream. - """ - if self.comptype == "tar": - return self.__read(size) - - c = len(self.dbuf) - t = [self.dbuf] - while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" - else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] - - def __read(self, size): - """Return size bytes from stream. If internal buffer is empty, - read another block from the stream. - """ - c = len(self.buf) - t = [self.buf] - while c < size: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - t.append(buf) - c += len(buf) - t = b"".join(t) - self.buf = t[size:] - return t[:size] -# class _Stream - -class _StreamProxy(object): - """Small proxy class that enables transparent compression - detection for the Stream interface (mode 'r|*'). - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - self.buf = self.fileobj.read(BLOCKSIZE) - - def read(self, size): - self.read = self.fileobj.read - return self.buf - - def getcomptype(self): - if self.buf.startswith(b"\x1f\x8b\x08"): - return "gz" - elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY": - return "bz2" - elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): - return "xz" - else: - return "tar" - - def close(self): - self.fileobj.close() -# class StreamProxy - -#------------------------ -# Extraction file object -#------------------------ -class _FileInFile(object): - """A thin wrapper around an existing file object that - provides a part of its data as an individual file - object. - """ - - def __init__(self, fileobj, offset, size, name, blockinfo=None): - self.fileobj = fileobj - self.offset = offset - self.size = size - self.position = 0 - self.name = name - self.closed = False - - if blockinfo is None: - blockinfo = [(0, size)] - - # Construct a map with data and zero blocks. - self.map_index = 0 - self.map = [] - lastpos = 0 - realpos = self.offset - for offset, size in blockinfo: - if offset > lastpos: - self.map.append((False, lastpos, offset, None)) - self.map.append((True, offset, offset + size, realpos)) - realpos += size - lastpos = offset + size - if lastpos < self.size: - self.map.append((False, lastpos, self.size, None)) - - def flush(self): - pass - - def readable(self): - return True - - def writable(self): - return False - - def seekable(self): - return self.fileobj.seekable() - - def tell(self): - """Return the current file position. - """ - return self.position - - def seek(self, position, whence=io.SEEK_SET): - """Seek to a position in the file. - """ - if whence == io.SEEK_SET: - self.position = min(max(position, 0), self.size) - elif whence == io.SEEK_CUR: - if position < 0: - self.position = max(self.position + position, 0) - else: - self.position = min(self.position + position, self.size) - elif whence == io.SEEK_END: - self.position = max(min(self.size + position, self.size), 0) - else: - raise ValueError("Invalid argument") - return self.position - - def read(self, size=None): - """Read data from the file. - """ - if size is None: - size = self.size - self.position - else: - size = min(size, self.size - self.position) - - buf = b"" - while size > 0: - while True: - data, start, stop, offset = self.map[self.map_index] - if start <= self.position < stop: - break - else: - self.map_index += 1 - if self.map_index == len(self.map): - self.map_index = 0 - length = min(size, stop - self.position) - if data: - self.fileobj.seek(offset + (self.position - start)) - b = self.fileobj.read(length) - if len(b) != length: - raise ReadError("unexpected end of data") - buf += b - else: - buf += NUL * length - size -= length - self.position += length - return buf - - def readinto(self, b): - buf = self.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def close(self): - self.closed = True -#class _FileInFile - -class ExFileObject(io.BufferedReader): - - def __init__(self, tarfile, tarinfo): - fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data, - tarinfo.size, tarinfo.name, tarinfo.sparse) - super().__init__(fileobj) -#class ExFileObject - - -#----------------------------- -# extraction filters (PEP 706) -#----------------------------- - -class FilterError(TarError): - pass - -class AbsolutePathError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'member {tarinfo.name!r} has an absolute path') - -class OutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, ' - + 'which is outside the destination') - -class SpecialFileError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a special file') - -class AbsoluteLinkError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a link to an absolute path') - -class LinkOutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' - + 'which is outside the destination') - -def _get_filtered_attrs(member, dest_path, for_data=True): - new_attrs = {} - name = member.name - dest_path = os.path.realpath(dest_path) - # Strip leading / (tar's directory separator) from filenames. - # Include os.sep (target OS directory separator) as well. - if name.startswith(('/', os.sep)): - name = new_attrs['name'] = member.path.lstrip('/' + os.sep) - if os.path.isabs(name): - # Path is absolute even after stripping. - # For example, 'C:/foo' on Windows. - raise AbsolutePathError(member) - # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise OutsideDestinationError(member, target_path) - # Limit permissions (no high bits, and go-w) - mode = member.mode - if mode is not None: - # Strip high bits & group/other write bits - mode = mode & 0o755 - if for_data: - # For data, handle permissions & file types - if member.isreg() or member.islnk(): - if not mode & 0o100: - # Clear executable bits if not executable by user - mode &= ~0o111 - # Ensure owner can read & write - mode |= 0o600 - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Reject special files - raise SpecialFileError(member) - if mode != member.mode: - new_attrs['mode'] = mode - if for_data: - # Ignore ownership for 'data' - if member.uid is not None: - new_attrs['uid'] = None - if member.gid is not None: - new_attrs['gid'] = None - if member.uname is not None: - new_attrs['uname'] = None - if member.gname is not None: - new_attrs['gname'] = None - # Check link destination for 'data' - if member.islnk() or member.issym(): - if os.path.isabs(member.linkname): - raise AbsoluteLinkError(member) - if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) - else: - target_path = os.path.join(dest_path, - member.linkname) - target_path = os.path.realpath(target_path) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise LinkOutsideDestinationError(member, target_path) - return new_attrs - -def fully_trusted_filter(member, dest_path): - return member - -def tar_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, False) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -def data_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, True) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -_NAMED_FILTERS = { - "fully_trusted": fully_trusted_filter, - "tar": tar_filter, - "data": data_filter, -} - -#------------------ -# Exported Classes -#------------------ - -# Sentinel for replace() defaults, meaning "don't change the attribute" -_KEEP = object() - -class TarInfo(object): - """Informational class which holds the details about an - archive member given by a tar header block. - TarInfo objects are returned by TarFile.getmember(), - TarFile.getmembers() and TarFile.gettarinfo() and are - usually created internally. - """ - - __slots__ = dict( - name = 'Name of the archive member.', - mode = 'Permission bits.', - uid = 'User ID of the user who originally stored this member.', - gid = 'Group ID of the user who originally stored this member.', - size = 'Size in bytes.', - mtime = 'Time of last modification.', - chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), - linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), - uname = 'User name.', - gname = 'Group name.', - devmajor = 'Device major number.', - devminor = 'Device minor number.', - offset = 'The tar header starts here.', - offset_data = "The file's data starts here.", - pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), - sparse = 'Sparse member information.', - tarfile = None, - _sparse_structs = None, - _link_target = None, - ) - - def __init__(self, name=""): - """Construct a TarInfo object. name is the optional name - of the member. - """ - self.name = name # member name - self.mode = 0o644 # file permissions - self.uid = 0 # user id - self.gid = 0 # group id - self.size = 0 # file size - self.mtime = 0 # modification time - self.chksum = 0 # header checksum - self.type = REGTYPE # member type - self.linkname = "" # link name - self.uname = "" # user name - self.gname = "" # group name - self.devmajor = 0 # device major number - self.devminor = 0 # device minor number - - self.offset = 0 # the tar header starts here - self.offset_data = 0 # the file's data starts here - - self.sparse = None # sparse member information - self.pax_headers = {} # pax header information - - @property - def path(self): - 'In pax headers, "name" is called "path".' - return self.name - - @path.setter - def path(self, name): - self.name = name - - @property - def linkpath(self): - 'In pax headers, "linkname" is called "linkpath".' - return self.linkname - - @linkpath.setter - def linkpath(self, linkname): - self.linkname = linkname - - def __repr__(self): - return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) - - def replace(self, *, - name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, - uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, - deep=True, _KEEP=_KEEP): - """Return a deep copy of self with the given attributes replaced. - """ - if deep: - result = copy.deepcopy(self) - else: - result = copy.copy(self) - if name is not _KEEP: - result.name = name - if mtime is not _KEEP: - result.mtime = mtime - if mode is not _KEEP: - result.mode = mode - if linkname is not _KEEP: - result.linkname = linkname - if uid is not _KEEP: - result.uid = uid - if gid is not _KEEP: - result.gid = gid - if uname is not _KEEP: - result.uname = uname - if gname is not _KEEP: - result.gname = gname - return result - - def get_info(self): - """Return the TarInfo's attributes as a dictionary. - """ - if self.mode is None: - mode = None - else: - mode = self.mode & 0o7777 - info = { - "name": self.name, - "mode": mode, - "uid": self.uid, - "gid": self.gid, - "size": self.size, - "mtime": self.mtime, - "chksum": self.chksum, - "type": self.type, - "linkname": self.linkname, - "uname": self.uname, - "gname": self.gname, - "devmajor": self.devmajor, - "devminor": self.devminor - } - - if info["type"] == DIRTYPE and not info["name"].endswith("/"): - info["name"] += "/" - - return info - - def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): - """Return a tar header as a string of 512 byte blocks. - """ - info = self.get_info() - for name, value in info.items(): - if value is None: - raise ValueError("%s may not be None" % name) - - if format == USTAR_FORMAT: - return self.create_ustar_header(info, encoding, errors) - elif format == GNU_FORMAT: - return self.create_gnu_header(info, encoding, errors) - elif format == PAX_FORMAT: - return self.create_pax_header(info, encoding) - else: - raise ValueError("invalid format") - - def create_ustar_header(self, info, encoding, errors): - """Return the object as a ustar header block. - """ - info["magic"] = POSIX_MAGIC - - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - raise ValueError("linkname is too long") - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - info["prefix"], info["name"] = self._posix_split_name(info["name"], encoding, errors) - - return self._create_header(info, USTAR_FORMAT, encoding, errors) - - def create_gnu_header(self, info, encoding, errors): - """Return the object as a GNU header block sequence. - """ - info["magic"] = GNU_MAGIC - - buf = b"" - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) - - return buf + self._create_header(info, GNU_FORMAT, encoding, errors) - - def create_pax_header(self, info, encoding): - """Return the object as a ustar header block. If it cannot be - represented this way, prepend a pax extended header sequence - with supplement information. - """ - info["magic"] = POSIX_MAGIC - pax_headers = self.pax_headers.copy() - - # Test string fields for values that exceed the field length or cannot - # be represented in ASCII encoding. - for name, hname, length in ( - ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), - ("uname", "uname", 32), ("gname", "gname", 32)): - - if hname in pax_headers: - # The pax header has priority. - continue - - # Try to encode the string as ASCII. - try: - info[name].encode("ascii", "strict") - except UnicodeEncodeError: - pax_headers[hname] = info[name] - continue - - if len(info[name]) > length: - pax_headers[hname] = info[name] - - # Test number fields for values that exceed the field limit or values - # that like to be stored as float. - for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): - needs_pax = False - - val = info[name] - val_is_float = isinstance(val, float) - val_int = round(val) if val_is_float else val - if not 0 <= val_int < 8 ** (digits - 1): - # Avoid overflow. - info[name] = 0 - needs_pax = True - elif val_is_float: - # Put rounded value in ustar header, and full - # precision value in pax header. - info[name] = val_int - needs_pax = True - - # The existing pax header has priority. - if needs_pax and name not in pax_headers: - pax_headers[name] = str(val) - - # Create a pax extended header if necessary. - if pax_headers: - buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) - else: - buf = b"" - - return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") - - @classmethod - def create_pax_global_header(cls, pax_headers): - """Return the object as a pax global header block sequence. - """ - return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") - - def _posix_split_name(self, name, encoding, errors): - """Split a name longer than 100 chars into a prefix - and a name part. - """ - components = name.split("/") - for i in range(1, len(components)): - prefix = "/".join(components[:i]) - name = "/".join(components[i:]) - if len(prefix.encode(encoding, errors)) <= LENGTH_PREFIX and \ - len(name.encode(encoding, errors)) <= LENGTH_NAME: - break - else: - raise ValueError("name is too long") - - return prefix, name - - @staticmethod - def _create_header(info, format, encoding, errors): - """Return a header block. info is a dictionary with file - information, format must be one of the *_FORMAT constants. - """ - has_device_fields = info.get("type") in (CHRTYPE, BLKTYPE) - if has_device_fields: - devmajor = itn(info.get("devmajor", 0), 8, format) - devminor = itn(info.get("devminor", 0), 8, format) - else: - devmajor = stn("", 8, encoding, errors) - devminor = stn("", 8, encoding, errors) - - # None values in metadata should cause ValueError. - # itn()/stn() do this for all fields except type. - filetype = info.get("type", REGTYPE) - if filetype is None: - raise ValueError("TarInfo.type must not be None") - - parts = [ - stn(info.get("name", ""), 100, encoding, errors), - itn(info.get("mode", 0) & 0o7777, 8, format), - itn(info.get("uid", 0), 8, format), - itn(info.get("gid", 0), 8, format), - itn(info.get("size", 0), 12, format), - itn(info.get("mtime", 0), 12, format), - b" ", # checksum field - filetype, - stn(info.get("linkname", ""), 100, encoding, errors), - info.get("magic", POSIX_MAGIC), - stn(info.get("uname", ""), 32, encoding, errors), - stn(info.get("gname", ""), 32, encoding, errors), - devmajor, - devminor, - stn(info.get("prefix", ""), 155, encoding, errors) - ] - - buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) - chksum = calc_chksums(buf[-BLOCKSIZE:])[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - - @staticmethod - def _create_payload(payload): - """Return the string payload filled with zero bytes - up to the next 512 byte border. - """ - blocks, remainder = divmod(len(payload), BLOCKSIZE) - if remainder > 0: - payload += (BLOCKSIZE - remainder) * NUL - return payload - - @classmethod - def _create_gnu_long_header(cls, name, type, encoding, errors): - """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence - for name. - """ - name = name.encode(encoding, errors) + NUL - - info = {} - info["name"] = "././@LongLink" - info["type"] = type - info["size"] = len(name) - info["magic"] = GNU_MAGIC - - # create extended header + name blocks. - return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ - cls._create_payload(name) - - @classmethod - def _create_pax_generic_header(cls, pax_headers, type, encoding): - """Return a POSIX.1-2008 extended or global header sequence - that contains a list of keyword, value pairs. The values - must be strings. - """ - # Check if one of the fields contains surrogate characters and thereby - # forces hdrcharset=BINARY, see _proc_pax() for more information. - binary = False - for keyword, value in pax_headers.items(): - try: - value.encode("utf-8", "strict") - except UnicodeEncodeError: - binary = True - break - - records = b"" - if binary: - # Put the hdrcharset field at the beginning of the header. - records += b"21 hdrcharset=BINARY\n" - - for keyword, value in pax_headers.items(): - keyword = keyword.encode("utf-8") - if binary: - # Try to restore the original byte representation of `value'. - # Needless to say, that the encoding must match the string. - value = value.encode(encoding, "surrogateescape") - else: - value = value.encode("utf-8") - - l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' - n = p = 0 - while True: - n = l + len(str(p)) - if n == p: - break - p = n - records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" - - # We use a hardcoded "././@PaxHeader" name like star does - # instead of the one that POSIX recommends. - info = {} - info["name"] = "././@PaxHeader" - info["type"] = type - info["size"] = len(records) - info["magic"] = POSIX_MAGIC - - # Create pax header + record blocks. - return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ - cls._create_payload(records) - - @classmethod - def frombuf(cls, buf, encoding, errors): - """Construct a TarInfo object from a 512 byte bytes object. - """ - if len(buf) == 0: - raise EmptyHeaderError("empty header") - if len(buf) != BLOCKSIZE: - raise TruncatedHeaderError("truncated header") - if buf.count(NUL) == BLOCKSIZE: - raise EOFHeaderError("end of file header") - - chksum = nti(buf[148:156]) - if chksum not in calc_chksums(buf): - raise InvalidHeaderError("bad checksum") - - obj = cls() - obj.name = nts(buf[0:100], encoding, errors) - obj.mode = nti(buf[100:108]) - obj.uid = nti(buf[108:116]) - obj.gid = nti(buf[116:124]) - obj.size = nti(buf[124:136]) - obj.mtime = nti(buf[136:148]) - obj.chksum = chksum - obj.type = buf[156:157] - obj.linkname = nts(buf[157:257], encoding, errors) - obj.uname = nts(buf[265:297], encoding, errors) - obj.gname = nts(buf[297:329], encoding, errors) - obj.devmajor = nti(buf[329:337]) - obj.devminor = nti(buf[337:345]) - prefix = nts(buf[345:500], encoding, errors) - - # Old V7 tar format represents a directory as a regular - # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): - obj.type = DIRTYPE - - # The old GNU sparse format occupies some of the unused - # space in the buffer for up to 4 sparse structures. - # Save them for later processing in _proc_sparse(). - if obj.type == GNUTYPE_SPARSE: - pos = 386 - structs = [] - for i in range(4): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[482]) - origsize = nti(buf[483:495]) - obj._sparse_structs = (structs, isextended, origsize) - - # Remove redundant slashes from directories. - if obj.isdir(): - obj.name = obj.name.rstrip("/") - - # Reconstruct a ustar longname. - if prefix and obj.type not in GNU_TYPES: - obj.name = prefix + "/" + obj.name - return obj - - @classmethod - def fromtarfile(cls, tarfile): - """Return the next TarInfo object from TarFile object - tarfile. - """ - buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) - obj.offset = tarfile.fileobj.tell() - BLOCKSIZE - return obj._proc_member(tarfile) - - #-------------------------------------------------------------------------- - # The following are methods that are called depending on the type of a - # member. The entry point is _proc_member() which can be overridden in a - # subclass to add custom _proc_*() methods. A _proc_*() method MUST - # implement the following - # operations: - # 1. Set self.offset_data to the position where the data blocks begin, - # if there is data that follows. - # 2. Set tarfile.offset to the position where the next member's header will - # begin. - # 3. Return self or another valid TarInfo object. - def _proc_member(self, tarfile): - """Choose the right processing method depending on - the type and call it. - """ - if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): - return self._proc_gnulong(tarfile) - elif self.type == GNUTYPE_SPARSE: - return self._proc_sparse(tarfile) - elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): - return self._proc_pax(tarfile) - else: - return self._proc_builtin(tarfile) - - def _proc_builtin(self, tarfile): - """Process a builtin type or an unknown type which - will be treated as a regular file. - """ - self.offset_data = tarfile.fileobj.tell() - offset = self.offset_data - if self.isreg() or self.type not in SUPPORTED_TYPES: - # Skip the following data blocks. - offset += self._block(self.size) - tarfile.offset = offset - - # Patch the TarInfo object with saved global - # header information. - self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if self.isdir(): - self.name = self.name.rstrip("/") - - return self - - def _proc_gnulong(self, tarfile): - """Process the blocks that hold a GNU longname - or longlink member. - """ - buf = tarfile.fileobj.read(self._block(self.size)) - - # Fetch the next header and process it. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Patch the TarInfo object from the next header with - # the longname information. - next.offset = self.offset - if self.type == GNUTYPE_LONGNAME: - next.name = nts(buf, tarfile.encoding, tarfile.errors) - elif self.type == GNUTYPE_LONGLINK: - next.linkname = nts(buf, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if next.isdir(): - next.name = next.name.removesuffix("/") - - return next - - def _proc_sparse(self, tarfile): - """Process a GNU sparse header plus extra headers. - """ - # We already collected some sparse structures in frombuf(). - structs, isextended, origsize = self._sparse_structs - del self._sparse_structs - - # Collect sparse structures from extended header blocks. - while isextended: - buf = tarfile.fileobj.read(BLOCKSIZE) - pos = 0 - for i in range(21): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset and numbytes: - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[504]) - self.sparse = structs - - self.offset_data = tarfile.fileobj.tell() - tarfile.offset = self.offset_data + self._block(self.size) - self.size = origsize - return self - - def _proc_pax(self, tarfile): - """Process an extended or global header as described in - POSIX.1-2008. - """ - # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) - - # A pax header stores supplemental information for either - # the following file (extended) or all following files - # (global). - if self.type == XGLTYPE: - pax_headers = tarfile.pax_headers - else: - pax_headers = tarfile.pax_headers.copy() - - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - - # Parse pax header information. A record looks like that: - # "%d %s=%s\n" % (length, keyword, value). length is the size - # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") - pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: - raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] - - # Normally, we could just use "utf-8" as the encoding and "strict" - # as the error handler, but we better not take the risk. For - # example, GNU tar <= 1.23 is known to store filenames it cannot - # translate to UTF-8 as raw strings (unfortunately without a - # hdrcharset=BINARY header). - # We first try the strict standard encoding, and if that fails we - # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", - tarfile.errors) - if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, - tarfile.errors) - else: - value = self._decode_pax_field(value, "utf-8", "utf-8", - tarfile.errors) - - pax_headers[keyword] = value - pos += length - - # Fetch the next header. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Process GNU sparse information. - if "GNU.sparse.map" in pax_headers: - # GNU extended sparse format version 0.1. - self._proc_gnusparse_01(next, pax_headers) - - elif "GNU.sparse.size" in pax_headers: - # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) - - elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": - # GNU extended sparse format version 1.0. - self._proc_gnusparse_10(next, pax_headers, tarfile) - - if self.type in (XHDTYPE, SOLARIS_XHDTYPE): - # Patch the TarInfo object with the extended header info. - next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset - - if "size" in pax_headers: - # If the extended header replaces the size field, - # we need to recalculate the offset where the next - # header starts. - offset = next.offset_data - if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) - tarfile.offset = offset - - return next - - def _proc_gnusparse_00(self, next, pax_headers, buf): - """Process a GNU tar extended sparse header, version 0.0. - """ - offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) - numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) - next.sparse = list(zip(offsets, numbytes)) - - def _proc_gnusparse_01(self, next, pax_headers): - """Process a GNU tar extended sparse header, version 0.1. - """ - sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _proc_gnusparse_10(self, next, pax_headers, tarfile): - """Process a GNU tar extended sparse header, version 1.0. - """ - fields = None - sparse = [] - buf = tarfile.fileobj.read(BLOCKSIZE) - fields, buf = buf.split(b"\n", 1) - fields = int(fields) - while len(sparse) < fields * 2: - if b"\n" not in buf: - buf += tarfile.fileobj.read(BLOCKSIZE) - number, buf = buf.split(b"\n", 1) - sparse.append(int(number)) - next.offset_data = tarfile.fileobj.tell() - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _apply_pax_info(self, pax_headers, encoding, errors): - """Replace fields with supplemental information from a previous - pax extended or global header. - """ - for keyword, value in pax_headers.items(): - if keyword == "GNU.sparse.name": - setattr(self, "path", value) - elif keyword == "GNU.sparse.size": - setattr(self, "size", int(value)) - elif keyword == "GNU.sparse.realsize": - setattr(self, "size", int(value)) - elif keyword in PAX_FIELDS: - if keyword in PAX_NUMBER_FIELDS: - try: - value = PAX_NUMBER_FIELDS[keyword](value) - except ValueError: - value = 0 - if keyword == "path": - value = value.rstrip("/") - setattr(self, keyword, value) - - self.pax_headers = pax_headers.copy() - - def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): - """Decode a single field from a pax record. - """ - try: - return value.decode(encoding, "strict") - except UnicodeDecodeError: - return value.decode(fallback_encoding, fallback_errors) - - def _block(self, count): - """Round up a byte count by BLOCKSIZE and return it, - e.g. _block(834) => 1024. - """ - blocks, remainder = divmod(count, BLOCKSIZE) - if remainder: - blocks += 1 - return blocks * BLOCKSIZE - - def isreg(self): - 'Return True if the Tarinfo object is a regular file.' - return self.type in REGULAR_TYPES - - def isfile(self): - 'Return True if the Tarinfo object is a regular file.' - return self.isreg() - - def isdir(self): - 'Return True if it is a directory.' - return self.type == DIRTYPE - - def issym(self): - 'Return True if it is a symbolic link.' - return self.type == SYMTYPE - - def islnk(self): - 'Return True if it is a hard link.' - return self.type == LNKTYPE - - def ischr(self): - 'Return True if it is a character device.' - return self.type == CHRTYPE - - def isblk(self): - 'Return True if it is a block device.' - return self.type == BLKTYPE - - def isfifo(self): - 'Return True if it is a FIFO.' - return self.type == FIFOTYPE - - def issparse(self): - return self.sparse is not None - - def isdev(self): - 'Return True if it is one of character device, block device or FIFO.' - return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) -# class TarInfo - -class TarFile(object): - """The TarFile Class provides an interface to tar archives. - """ - - debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) - - dereference = False # If true, add content of linked file to the - # tar file, else the link. - - ignore_zeros = False # If true, skips empty or invalid blocks and - # continues processing. - - errorlevel = 1 # If 0, fatal errors only appear in debug - # messages (if debug >= 0). If > 0, errors - # are passed to the caller as exceptions. - - format = DEFAULT_FORMAT # The format to use when creating an archive. - - encoding = ENCODING # Encoding for 8-bit character strings. - - errors = None # Error handler for unicode conversion. - - tarinfo = TarInfo # The default TarInfo class to use. - - fileobject = ExFileObject # The file-object for extractfile(). - - extraction_filter = None # The default filter for extraction. - - def __init__(self, name=None, mode="r", fileobj=None, format=None, - tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, - errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to - read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' - defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. - """ - modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} - if mode not in modes: - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - self.mode = mode - self._mode = modes[mode] - - if not fileobj: - if self.mode == "a" and not os.path.exists(name): - # Create nonexistent files in append mode. - self.mode = "w" - self._mode = "wb" - fileobj = bltn_open(name, self._mode) - self._extfileobj = False - else: - if (name is None and hasattr(fileobj, "name") and - isinstance(fileobj.name, (str, bytes))): - name = fileobj.name - if hasattr(fileobj, "mode"): - self._mode = fileobj.mode - self._extfileobj = True - self.name = os.path.abspath(name) if name else None - self.fileobj = fileobj - - # Init attributes. - if format is not None: - self.format = format - if tarinfo is not None: - self.tarinfo = tarinfo - if dereference is not None: - self.dereference = dereference - if ignore_zeros is not None: - self.ignore_zeros = ignore_zeros - if encoding is not None: - self.encoding = encoding - self.errors = errors - - if pax_headers is not None and self.format == PAX_FORMAT: - self.pax_headers = pax_headers - else: - self.pax_headers = {} - - if debug is not None: - self.debug = debug - if errorlevel is not None: - self.errorlevel = errorlevel - - # Init datastructures. - self.copybufsize = copybufsize - self.closed = False - self.members = [] # list of members as TarInfo objects - self._loaded = False # flag if all members have been read - self.offset = self.fileobj.tell() - # current position in the archive file - self.inodes = {} # dictionary caching the inodes of - # archive members already added - - try: - if self.mode == "r": - self.firstmember = None - self.firstmember = self.next() - - if self.mode == "a": - # Move to the end of the archive, - # before the first empty block. - while True: - self.fileobj.seek(self.offset) - try: - tarinfo = self.tarinfo.fromtarfile(self) - self.members.append(tarinfo) - except EOFHeaderError: - self.fileobj.seek(self.offset) - break - except HeaderError as e: - raise ReadError(str(e)) from None - - if self.mode in ("a", "w", "x"): - self._loaded = True - - if self.pax_headers: - buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) - self.fileobj.write(buf) - self.offset += len(buf) - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - #-------------------------------------------------------------------------- - # Below are the classmethods which act as alternate constructors to the - # TarFile class. The open() method is the only one that is needed for - # public use; it is the "super"-constructor and is able to select an - # adequate "sub"-constructor for a particular compression using the mapping - # from OPEN_METH. - # - # This concept allows one to subclass TarFile without losing the comfort of - # the super-constructor. A sub-constructor is registered and made available - # by adding it to the mapping in OPEN_METH. - - @classmethod - def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): - r"""Open a tar archive for reading, writing or appending. Return - an appropriate TarFile class. - - mode: - 'r' or 'r:\*' open for reading with transparent compression - 'r:' open for reading exclusively uncompressed - 'r:gz' open for reading with gzip compression - 'r:bz2' open for reading with bzip2 compression - 'r:xz' open for reading with lzma compression - 'a' or 'a:' open for appending, creating the file if necessary - 'w' or 'w:' open for writing without compression - 'w:gz' open for writing with gzip compression - 'w:bz2' open for writing with bzip2 compression - 'w:xz' open for writing with lzma compression - - 'x' or 'x:' create a tarfile exclusively without compression, raise - an exception if the file is already created - 'x:gz' create a gzip compressed tarfile, raise an exception - if the file is already created - 'x:bz2' create a bzip2 compressed tarfile, raise an exception - if the file is already created - 'x:xz' create an lzma compressed tarfile, raise an exception - if the file is already created - - 'r|\*' open a stream of tar blocks with transparent compression - 'r|' open an uncompressed stream of tar blocks for reading - 'r|gz' open a gzip compressed stream of tar blocks - 'r|bz2' open a bzip2 compressed stream of tar blocks - 'r|xz' open an lzma compressed stream of tar blocks - 'w|' open an uncompressed stream for writing - 'w|gz' open a gzip compressed stream for writing - 'w|bz2' open a bzip2 compressed stream for writing - 'w|xz' open an lzma compressed stream for writing - """ - - if not name and not fileobj: - raise ValueError("nothing to open") - - if mode in ("r", "r:*"): - # Find out which *open() is appropriate for opening the file. - def not_compressed(comptype): - return cls.OPEN_METH[comptype] == 'taropen' - error_msgs = [] - for comptype in sorted(cls.OPEN_METH, key=not_compressed): - func = getattr(cls, cls.OPEN_METH[comptype]) - if fileobj is not None: - saved_pos = fileobj.tell() - try: - return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError) as e: - error_msgs.append(f'- method {comptype}: {e!r}') - if fileobj is not None: - fileobj.seek(saved_pos) - continue - error_msgs_summary = '\n'.join(error_msgs) - raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}") - - elif ":" in mode: - filemode, comptype = mode.split(":", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - # Select the *open() function according to - # given compression. - if comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - else: - raise CompressionError("unknown compression type %r" % comptype) - return func(name, filemode, fileobj, **kwargs) - - elif "|" in mode: - filemode, comptype = mode.split("|", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - if filemode not in ("r", "w"): - raise ValueError("mode must be 'r' or 'w'") - - compresslevel = kwargs.pop("compresslevel", 9) - stream = _Stream(name, filemode, comptype, fileobj, bufsize, - compresslevel) - try: - t = cls(name, filemode, stream, **kwargs) - except: - stream.close() - raise - t._extfileobj = False - return t - - elif mode in ("a", "w", "x"): - return cls.taropen(name, mode, fileobj, **kwargs) - - raise ValueError("undiscernible mode") - - @classmethod - def taropen(cls, name, mode="r", fileobj=None, **kwargs): - """Open uncompressed tar archive name for reading or writing. - """ - if mode not in ("r", "a", "w", "x"): - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - return cls(name, mode, fileobj, **kwargs) - - @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open gzip compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from gzip import GzipFile - except ImportError: - raise CompressionError("gzip module is not available") from None - - try: - fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) - except OSError as e: - if fileobj is not None and mode == 'r': - raise ReadError("not a gzip file") from e - raise - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except OSError as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a gzip file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open bzip2 compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from bz2 import BZ2File - except ImportError: - raise CompressionError("bz2 module is not available") from None - - fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (OSError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a bzip2 file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): - """Open lzma compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from lzma import LZMAFile, LZMAError - except ImportError: - raise CompressionError("lzma module is not available") from None - - fileobj = LZMAFile(fileobj or name, mode, preset=preset) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (LZMAError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not an lzma file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - # All *open() methods are registered here. - OPEN_METH = { - "tar": "taropen", # uncompressed tar - "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar - } - - #-------------------------------------------------------------------------- - # The public methods which TarFile provides: - - def close(self): - """Close the TarFile. In write-mode, two finishing zero blocks are - appended to the archive. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode in ("a", "w", "x"): - self.fileobj.write(NUL * (BLOCKSIZE * 2)) - self.offset += (BLOCKSIZE * 2) - # fill up the end with zero-blocks - # (like option -b20 for tar does) - blocks, remainder = divmod(self.offset, RECORDSIZE) - if remainder > 0: - self.fileobj.write(NUL * (RECORDSIZE - remainder)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def getmember(self, name): - """Return a TarInfo object for member ``name``. If ``name`` can not be - found in the archive, KeyError is raised. If a member occurs more - than once in the archive, its last occurrence is assumed to be the - most up-to-date version. - """ - tarinfo = self._getmember(name.rstrip('/')) - if tarinfo is None: - raise KeyError("filename %r not found" % name) - return tarinfo - - def getmembers(self): - """Return the members of the archive as a list of TarInfo objects. The - list has the same order as the members in the archive. - """ - self._check() - if not self._loaded: # if we want to obtain a list of - self._load() # all members, we first have to - # scan the whole archive. - return self.members - - def getnames(self): - """Return the members of the archive as a list of their names. It has - the same order as the list returned by getmembers(). - """ - return [tarinfo.name for tarinfo in self.getmembers()] - - def gettarinfo(self, name=None, arcname=None, fileobj=None): - """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by ``name``, or - specified as a file object ``fileobj`` with a file descriptor. If - given, ``arcname`` specifies an alternative name for the file in the - archive, otherwise, the name is taken from the 'name' attribute of - 'fileobj', or the 'name' argument. The name should be a text - string. - """ - self._check("awx") - - # When fileobj is given, replace name by - # fileobj's real name. - if fileobj is not None: - name = fileobj.name - - # Building the name of the member in the archive. - # Backward slashes are converted to forward slashes, - # Absolute paths are turned to relative paths. - if arcname is None: - arcname = name - drv, arcname = os.path.splitdrive(arcname) - arcname = arcname.replace(os.sep, "/") - arcname = arcname.lstrip("/") - - # Now, fill the TarInfo object with - # information specific for the file. - tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed - - # Use os.stat or os.lstat, depending on if symlinks shall be resolved. - if fileobj is None: - if not self.dereference: - statres = os.lstat(name) - else: - statres = os.stat(name) - else: - statres = os.fstat(fileobj.fileno()) - linkname = "" - - stmd = statres.st_mode - if stat.S_ISREG(stmd): - inode = (statres.st_ino, statres.st_dev) - if not self.dereference and statres.st_nlink > 1 and \ - inode in self.inodes and arcname != self.inodes[inode]: - # Is it a hardlink to an already - # archived file? - type = LNKTYPE - linkname = self.inodes[inode] - else: - # The inode is added only if its valid. - # For win32 it is always 0. - type = REGTYPE - if inode[0]: - self.inodes[inode] = arcname - elif stat.S_ISDIR(stmd): - type = DIRTYPE - elif stat.S_ISFIFO(stmd): - type = FIFOTYPE - elif stat.S_ISLNK(stmd): - type = SYMTYPE - linkname = os.readlink(name) - elif stat.S_ISCHR(stmd): - type = CHRTYPE - elif stat.S_ISBLK(stmd): - type = BLKTYPE - else: - return None - - # Fill the TarInfo object with all - # information we can get. - tarinfo.name = arcname - tarinfo.mode = stmd - tarinfo.uid = statres.st_uid - tarinfo.gid = statres.st_gid - if type == REGTYPE: - tarinfo.size = statres.st_size - else: - tarinfo.size = 0 - tarinfo.mtime = statres.st_mtime - tarinfo.type = type - tarinfo.linkname = linkname - if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if type in (CHRTYPE, BLKTYPE): - if hasattr(os, "major") and hasattr(os, "minor"): - tarinfo.devmajor = os.major(statres.st_rdev) - tarinfo.devminor = os.minor(statres.st_rdev) - return tarinfo - - def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If ``verbose`` is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. ``members`` is optional and must be a subset of the - list returned by getmembers(). - """ - self._check() - - if members is None: - members = self - for tarinfo in members: - if verbose: - if tarinfo.mode is None: - _safe_print("??????????") - else: - _safe_print(stat.filemode(tarinfo.mode)) - _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid)) - if tarinfo.ischr() or tarinfo.isblk(): - _safe_print("%10s" % - ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) - else: - _safe_print("%10d" % tarinfo.size) - if tarinfo.mtime is None: - _safe_print("????-??-?? ??:??:??") - else: - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) - - _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) - - if verbose: - if tarinfo.issym(): - _safe_print("-> " + tarinfo.linkname) - if tarinfo.islnk(): - _safe_print("link to " + tarinfo.linkname) - print() - - def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file ``name`` to the archive. ``name`` may be any type of file - (directory, fifo, symbolic link, etc.). If given, ``arcname`` - specifies an alternative name for the file in the archive. - Directories are added recursively by default. This can be avoided by - setting ``recursive`` to False. ``filter`` is a function - that expects a TarInfo object argument and returns the changed - TarInfo object, if it returns None the TarInfo object will be - excluded from the archive. - """ - self._check("awx") - - if arcname is None: - arcname = name - - # Skip if somebody tries to archive the archive... - if self.name is not None and os.path.abspath(name) == self.name: - self._dbg(2, "tarfile: Skipped %r" % name) - return - - self._dbg(1, name) - - # Create a TarInfo object from the file. - tarinfo = self.gettarinfo(name, arcname) - - if tarinfo is None: - self._dbg(1, "tarfile: Unsupported type %r" % name) - return - - # Change or exclude the TarInfo object. - if filter is not None: - tarinfo = filter(tarinfo) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Append the tar header and data to the archive. - if tarinfo.isreg(): - with bltn_open(name, "rb") as f: - self.addfile(tarinfo, f) - - elif tarinfo.isdir(): - self.addfile(tarinfo) - if recursive: - for f in sorted(os.listdir(name)): - self.add(os.path.join(name, f), os.path.join(arcname, f), - recursive, filter=filter) - - else: - self.addfile(tarinfo) - - def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object ``tarinfo`` to the archive. If ``fileobj`` is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). - """ - self._check("awx") - - tarinfo = copy.copy(tarinfo) - - buf = tarinfo.tobuf(self.format, self.encoding, self.errors) - self.fileobj.write(buf) - self.offset += len(buf) - bufsize=self.copybufsize - # If there's data to follow, append it. - if fileobj is not None: - copyfileobj(fileobj, self.fileobj, tarinfo.size, bufsize=bufsize) - blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) - if remainder > 0: - self.fileobj.write(NUL * (BLOCKSIZE - remainder)) - blocks += 1 - self.offset += blocks * BLOCKSIZE - - self.members.append(tarinfo) - - def _get_filter_function(self, filter): - if filter is None: - filter = self.extraction_filter - if filter is None: - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter - if isinstance(filter, str): - raise TypeError( - 'String names are not supported for ' - + 'TarFile.extraction_filter. Use a function such as ' - + 'tarfile.data_filter directly.') - return filter - if callable(filter): - return filter - try: - return _NAMED_FILTERS[filter] - except KeyError: - raise ValueError(f"filter {filter!r} not found") from None - - def extractall(self, path=".", members=None, *, numeric_owner=False, - filter=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only - the numbers for user/group names are used and not the names. - - The `filter` function will be called on each member just - before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - directories = [] - - filter_function = self._get_filter_function(filter) - if members is None: - members = self - - for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is None: - continue - if tarinfo.isdir(): - # For directories, delay setting attributes until later, - # since permissions can interfere with extraction and - # extracting contents can reset mtime. - directories.append(tarinfo) - self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) - - # Reverse sort directories. - directories.sort(key=lambda a: a.name, reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, - filter=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` - is True, only the numbers for user/group names are used and not - the names. - - The `filter` function will be called before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) - - def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - unfiltered = tarinfo - try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo - - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" - self._check("r") - - try: - self._extract_member(tarinfo, os.path.join(path, tarinfo.name), - set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def _handle_nonfatal_error(self, e): - """Handle non-fatal error (ExtractError) according to errorlevel""" - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def _handle_fatal_error(self, e): - """Handle "fatal" error according to self.errorlevel""" - if self.errorlevel > 0: - raise - elif isinstance(e, OSError): - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) - else: - self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) - - def extractfile(self, member): - """Extract a member from the archive as a file object. ``member`` may be - a filename or a TarInfo object. If ``member`` is a regular file or - a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If ``member`` does not appear - in the archive, KeyError is raised. - """ - self._check("r") - - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES: - # Members with unknown types are treated as regular files. - return self.fileobject(self, tarinfo) - - elif tarinfo.islnk() or tarinfo.issym(): - if isinstance(self.fileobj, _Stream): - # A small but ugly workaround for the case that someone tries - # to extract a (sym)link as a file-object from a non-seekable - # stream of tar blocks. - raise StreamError("cannot extract (sym)link as file object") - else: - # A (sym)link's file object is its target's file object. - return self.extractfile(self._find_link_target(tarinfo)) - else: - # If there's no data associated with the member (directory, chrdev, - # blkdev, etc.), return None instead of a file object. - return None - - def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical - file called targetpath. - """ - # Fetch the TarInfo object for the given name - # and build the destination pathname, replacing - # forward slashes to platform specific separators. - targetpath = targetpath.rstrip("/") - targetpath = targetpath.replace("/", os.sep) - - # Create all upper directories. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - # Create directories that are not part of the archive with - # default permissions. - os.makedirs(upperdirs) - - if tarinfo.islnk() or tarinfo.issym(): - self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) - else: - self._dbg(1, tarinfo.name) - - if tarinfo.isreg(): - self.makefile(tarinfo, targetpath) - elif tarinfo.isdir(): - self.makedir(tarinfo, targetpath) - elif tarinfo.isfifo(): - self.makefifo(tarinfo, targetpath) - elif tarinfo.ischr() or tarinfo.isblk(): - self.makedev(tarinfo, targetpath) - elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) - elif tarinfo.type not in SUPPORTED_TYPES: - self.makeunknown(tarinfo, targetpath) - else: - self.makefile(tarinfo, targetpath) - - if set_attrs: - self.chown(tarinfo, targetpath, numeric_owner) - if not tarinfo.issym(): - self.chmod(tarinfo, targetpath) - self.utime(tarinfo, targetpath) - - #-------------------------------------------------------------------------- - # Below are the different file methods. They are called via - # _extract_member() when extract() is called. They can be replaced in a - # subclass to implement other functionality. - - def makedir(self, tarinfo, targetpath): - """Make a directory called targetpath. - """ - try: - if tarinfo.mode is None: - # Use the system's default mode - os.mkdir(targetpath) - else: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) - except FileExistsError: - if not os.path.isdir(targetpath): - raise - - def makefile(self, tarinfo, targetpath): - """Make a file called targetpath. - """ - source = self.fileobj - source.seek(tarinfo.offset_data) - bufsize = self.copybufsize - with bltn_open(targetpath, "wb") as target: - if tarinfo.sparse is not None: - for offset, size in tarinfo.sparse: - target.seek(offset) - copyfileobj(source, target, size, ReadError, bufsize) - target.seek(tarinfo.size) - target.truncate() - else: - copyfileobj(source, target, tarinfo.size, ReadError, bufsize) - - def makeunknown(self, tarinfo, targetpath): - """Make a file from a TarInfo object with an unknown type - at targetpath. - """ - self.makefile(tarinfo, targetpath) - self._dbg(1, "tarfile: Unknown file type %r, " \ - "extracted as regular file." % tarinfo.type) - - def makefifo(self, tarinfo, targetpath): - """Make a fifo called targetpath. - """ - if hasattr(os, "mkfifo"): - os.mkfifo(targetpath) - else: - raise ExtractError("fifo not supported by system") - - def makedev(self, tarinfo, targetpath): - """Make a character or block device called targetpath. - """ - if not hasattr(os, "mknod") or not hasattr(os, "makedev"): - raise ExtractError("special devices not supported by system") - - mode = tarinfo.mode - if mode is None: - # Use mknod's default - mode = 0o600 - if tarinfo.isblk(): - mode |= stat.S_IFBLK - else: - mode |= stat.S_IFCHR - - os.mknod(targetpath, mode, - os.makedev(tarinfo.devmajor, tarinfo.devminor)) - - def makelink(self, tarinfo, targetpath): - """Make a (symbolic) link called targetpath. If it cannot be created - (platform limitation), we try to make a copy of the referenced file - instead of a link. - """ - try: - # For systems that support symbolic and hard links. - if tarinfo.issym(): - if os.path.lexists(targetpath): - # Avoid FileExistsError on following os.symlink. - os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) - else: - if os.path.exists(tarinfo._link_target): - os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except symlink_exception: - try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None - - def chown(self, tarinfo, targetpath, numeric_owner): - """Set owner of targetpath according to tarinfo. If numeric_owner - is True, use .gid/.uid instead of .gname/.uname. If numeric_owner - is False, fall back to .gid/.uid when the search based on name - fails. - """ - if hasattr(os, "geteuid") and os.geteuid() == 0: - # We have to be root to do so. - g = tarinfo.gid - u = tarinfo.uid - if not numeric_owner: - try: - if grp and tarinfo.gname: - g = grp.getgrnam(tarinfo.gname)[2] - except KeyError: - pass - try: - if pwd and tarinfo.uname: - u = pwd.getpwnam(tarinfo.uname)[2] - except KeyError: - pass - if g is None: - g = -1 - if u is None: - u = -1 - try: - if tarinfo.issym() and hasattr(os, "lchown"): - os.lchown(targetpath, u, g) - else: - os.chown(targetpath, u, g) - except OSError as e: - raise ExtractError("could not change owner") from e - - def chmod(self, tarinfo, targetpath): - """Set file permissions of targetpath according to tarinfo. - """ - if tarinfo.mode is None: - return - try: - os.chmod(targetpath, tarinfo.mode) - except OSError as e: - raise ExtractError("could not change mode") from e - - def utime(self, tarinfo, targetpath): - """Set modification time of targetpath according to tarinfo. - """ - mtime = tarinfo.mtime - if mtime is None: - return - if not hasattr(os, 'utime'): - return - try: - os.utime(targetpath, (mtime, mtime)) - except OSError as e: - raise ExtractError("could not change modification time") from e - - #-------------------------------------------------------------------------- - def next(self): - """Return the next member of the archive as a TarInfo object, when - TarFile is opened for reading. Return None if there is no more - available. - """ - self._check("ra") - if self.firstmember is not None: - m = self.firstmember - self.firstmember = None - return m - - # Advance the file pointer. - if self.offset != self.fileobj.tell(): - if self.offset == 0: - return None - self.fileobj.seek(self.offset - 1) - if not self.fileobj.read(1): - raise ReadError("unexpected end of data") - - # Read the next block. - tarinfo = None - while True: - try: - tarinfo = self.tarinfo.fromtarfile(self) - except EOFHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - except InvalidHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - elif self.offset == 0: - raise ReadError(str(e)) from None - except EmptyHeaderError: - if self.offset == 0: - raise ReadError("empty file") from None - except TruncatedHeaderError as e: - if self.offset == 0: - raise ReadError(str(e)) from None - except SubsequentHeaderError as e: - raise ReadError(str(e)) from None - except Exception as e: - try: - import zlib - if isinstance(e, zlib.error): - raise ReadError(f'zlib error: {e}') from None - else: - raise e - except ImportError: - raise e - break - - if tarinfo is not None: - self.members.append(tarinfo) - else: - self._loaded = True - - return tarinfo - - #-------------------------------------------------------------------------- - # Little helper methods: - - def _getmember(self, name, tarinfo=None, normalize=False): - """Find an archive member by name from bottom to top. - If tarinfo is given, it is used as the starting point. - """ - # Ensure that all members have been loaded. - members = self.getmembers() - - # Limit the member search list up to tarinfo. - skipping = False - if tarinfo is not None: - try: - index = members.index(tarinfo) - except ValueError: - # The given starting point might be a (modified) copy. - # We'll later skip members until we find an equivalent. - skipping = True - else: - # Happy fast path - members = members[:index] - - if normalize: - name = os.path.normpath(name) - - for member in reversed(members): - if skipping: - if tarinfo.offset == member.offset: - skipping = False - continue - if normalize: - member_name = os.path.normpath(member.name) - else: - member_name = member.name - - if name == member_name: - return member - - if skipping: - # Starting point was not found - raise ValueError(tarinfo) - - def _load(self): - """Read through the entire archive file and look for readable - members. - """ - while self.next() is not None: - pass - self._loaded = True - - def _check(self, mode=None): - """Check if TarFile is still open, and if the operation's mode - corresponds to TarFile's mode. - """ - if self.closed: - raise OSError("%s is closed" % self.__class__.__name__) - if mode is not None and self.mode not in mode: - raise OSError("bad operation for mode %r" % self.mode) - - def _find_link_target(self, tarinfo): - """Find the target member of a symlink or hardlink member in the - archive. - """ - if tarinfo.issym(): - # Always search the entire archive. - linkname = "/".join(filter(None, (os.path.dirname(tarinfo.name), tarinfo.linkname))) - limit = None - else: - # Search the archive before the link, because a hard link is - # just a reference to an already archived file. - linkname = tarinfo.linkname - limit = tarinfo - - member = self._getmember(linkname, tarinfo=limit, normalize=True) - if member is None: - raise KeyError("linkname %r not found" % linkname) - return member - - def __iter__(self): - """Provide an iterator object. - """ - if self._loaded: - yield from self.members - return - - # Yield items using TarFile's next() method. - # When all members have been read, set TarFile as _loaded. - index = 0 - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will have already exhausted the next() method. - if self.firstmember is not None: - tarinfo = self.next() - index += 1 - yield tarinfo - - while True: - if index < len(self.members): - tarinfo = self.members[index] - elif not self._loaded: - tarinfo = self.next() - if not tarinfo: - self._loaded = True - return - else: - return - index += 1 - yield tarinfo - - def _dbg(self, level, msg): - """Write debugging output to sys.stderr. - """ - if level <= self.debug: - print(msg, file=sys.stderr) - - def __enter__(self): - self._check() - return self - - def __exit__(self, type, value, traceback): - if type is None: - self.close() - else: - # An exception occurred. We must not call close() because - # it would try to write end-of-archive blocks and padding. - if not self._extfileobj: - self.fileobj.close() - self.closed = True - -#-------------------- -# exported functions -#-------------------- - -def is_tarfile(name): - """Return True if name points to a tar archive that we - are able to handle, else return False. - - 'name' should be a string, file, or file-like object. - """ - try: - if hasattr(name, "read"): - pos = name.tell() - t = open(fileobj=name) - name.seek(pos) - else: - t = open(name) - t.close() - return True - except TarError: - return False - -open = TarFile.open - - -def main(): - import argparse - - description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='Verbose output') - parser.add_argument('--filter', metavar='<filtername>', - choices=_NAMED_FILTERS, - help='Filter for extraction') - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='<tarfile>', - help='Show listing of a tarfile') - group.add_argument('-e', '--extract', nargs='+', - metavar=('<tarfile>', '<output_dir>'), - help='Extract tarfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('<name>', '<file>'), - help='Create tarfile from sources') - group.add_argument('-t', '--test', metavar='<tarfile>', - help='Test if a tarfile is valid') - - args = parser.parse_args() - - if args.filter and args.extract is None: - parser.exit(1, '--filter is only valid for extraction\n') - - if args.test is not None: - src = args.test - if is_tarfile(src): - with open(src, 'r') as tar: - tar.getmembers() - print(tar.getmembers(), file=sys.stderr) - if args.verbose: - print('{!r} is a tar archive.'.format(src)) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.list is not None: - src = args.list - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.list(verbose=args.verbose) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.extract is not None: - if len(args.extract) == 1: - src = args.extract[0] - curdir = os.curdir - elif len(args.extract) == 2: - src, curdir = args.extract - else: - parser.exit(1, parser.format_help()) - - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir, filter=args.filter) - if args.verbose: - if curdir == '.': - msg = '{!r} file is extracted.'.format(src) - else: - msg = ('{!r} file is extracted ' - 'into {!r} directory.').format(src, curdir) - print(msg) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.create is not None: - tar_name = args.create.pop(0) - _, ext = os.path.splitext(tar_name) - compressions = { - # gz - '.gz': 'gz', - '.tgz': 'gz', - # xz - '.xz': 'xz', - '.txz': 'xz', - # bz2 - '.bz2': 'bz2', - '.tbz': 'bz2', - '.tbz2': 'bz2', - '.tb2': 'bz2', - } - tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' - tar_files = args.create - - with TarFile.open(tar_name, tar_mode) as tf: - for file_name in tar_files: - tf.add(file_name) - - if args.verbose: - print('{!r} file created.'.format(tar_name)) - -if __name__ == '__main__': - main() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py deleted file mode 100644 index 88642143755..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py +++ /dev/null @@ -1,904 +0,0 @@ -import os -import re -import abc -import csv -import sys -from .. import zipp -import email -import pathlib -import operator -import textwrap -import warnings -import functools -import itertools -import posixpath -import collections - -from . import _adapters, _meta, _py39compat -from ._collections import FreezableDefaultDict, Pair -from ._compat import ( - NullFinder, - install, - pypy_partial, -) -from ._functools import method_cache, pass_none -from ._itertools import always_iterable, unique_everseen -from ._meta import PackageMetadata, SimplePath - -from contextlib import suppress -from importlib import import_module -from importlib.abc import MetaPathFinder -from itertools import starmap -from typing import List, Mapping, Optional - - -__all__ = [ - 'Distribution', - 'DistributionFinder', - 'PackageMetadata', - 'PackageNotFoundError', - 'distribution', - 'distributions', - 'entry_points', - 'files', - 'metadata', - 'packages_distributions', - 'requires', - 'version', -] - - -class PackageNotFoundError(ModuleNotFoundError): - """The package was not found.""" - - def __str__(self): - return f"No package metadata was found for {self.name}" - - @property - def name(self): - (name,) = self.args - return name - - -class Sectioned: - """ - A simple entry point config parser for performance - - >>> for item in Sectioned.read(Sectioned._sample): - ... print(item) - Pair(name='sec1', value='# comments ignored') - Pair(name='sec1', value='a = 1') - Pair(name='sec1', value='b = 2') - Pair(name='sec2', value='a = 2') - - >>> res = Sectioned.section_pairs(Sectioned._sample) - >>> item = next(res) - >>> item.name - 'sec1' - >>> item.value - Pair(name='a', value='1') - >>> item = next(res) - >>> item.value - Pair(name='b', value='2') - >>> item = next(res) - >>> item.name - 'sec2' - >>> item.value - Pair(name='a', value='2') - >>> list(res) - [] - """ - - _sample = textwrap.dedent( - """ - [sec1] - # comments ignored - a = 1 - b = 2 - - [sec2] - a = 2 - """ - ).lstrip() - - @classmethod - def section_pairs(cls, text): - return ( - section._replace(value=Pair.parse(section.value)) - for section in cls.read(text, filter_=cls.valid) - if section.name is not None - ) - - @staticmethod - def read(text, filter_=None): - lines = filter(filter_, map(str.strip, text.splitlines())) - name = None - for value in lines: - section_match = value.startswith('[') and value.endswith(']') - if section_match: - name = value.strip('[]') - continue - yield Pair(name, value) - - @staticmethod - def valid(line): - return line and not line.startswith('#') - - -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=pypy_partial(2), - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): - """An entry point as defined by Python packaging conventions. - - See `the packaging docs on entry points - <https://packaging.python.org/specifications/entry-points/>`_ - for more information. - - >>> ep = EntryPoint( - ... name=None, group=None, value='package.module:attr [extra1, extra2]') - >>> ep.module - 'package.module' - >>> ep.attr - 'attr' - >>> ep.extras - ['extra1', 'extra2'] - """ - - pattern = re.compile( - r'(?P<module>[\w.]+)\s*' - r'(:\s*(?P<attr>[\w.]+)\s*)?' - r'((?P<extras>\[.*\])\s*)?$' - ) - """ - A regular expression describing the syntax for an entry point, - which might look like: - - - module - - package.module - - package.module:attribute - - package.module:object.attribute - - package.module:attr [extra1, extra2] - - Other combinations are possible as well. - - The expression is lenient about whitespace around the ':', - following the attr, and following any extras. - """ - - name: str - value: str - group: str - - dist: Optional['Distribution'] = None - - def __init__(self, name, value, group): - vars(self).update(name=name, value=value, group=group) - - def load(self): - """Load the entry point from its definition. If only a module - is indicated by the value, return that module. Otherwise, - return the named object. - """ - match = self.pattern.match(self.value) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) - return functools.reduce(getattr, attrs, module) - - @property - def module(self): - match = self.pattern.match(self.value) - return match.group('module') - - @property - def attr(self): - match = self.pattern.match(self.value) - return match.group('attr') - - @property - def extras(self): - match = self.pattern.match(self.value) - return re.findall(r'\w+', match.group('extras') or '') - - def _for(self, dist): - vars(self).update(dist=dist) - return self - - def matches(self, **params): - """ - EntryPoint matches the given parameters. - - >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]') - >>> ep.matches(group='foo') - True - >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]') - True - >>> ep.matches(group='foo', name='other') - False - >>> ep.matches() - True - >>> ep.matches(extras=['extra1', 'extra2']) - True - >>> ep.matches(module='bing') - True - >>> ep.matches(attr='bong') - True - """ - attrs = (getattr(self, param) for param in params) - return all(map(operator.eq, params.values(), attrs)) - - def _key(self): - return self.name, self.value, self.group - - def __lt__(self, other): - return self._key() < other._key() - - def __eq__(self, other): - return self._key() == other._key() - - def __setattr__(self, name, value): - raise AttributeError("EntryPoint objects are immutable.") - - def __repr__(self): - return ( - f'EntryPoint(name={self.name!r}, value={self.value!r}, ' - f'group={self.group!r})' - ) - - def __hash__(self): - return hash(self._key()) - - -class EntryPoints(tuple): - """ - An immutable collection of selectable EntryPoint objects. - """ - - __slots__ = () - - def __getitem__(self, name): # -> EntryPoint: - """ - Get the EntryPoint in self matching name. - """ - try: - return next(iter(self.select(name=name))) - except StopIteration: - raise KeyError(name) - - def select(self, **params): - """ - Select entry points from self that match the - given parameters (typically group and/or name). - """ - return EntryPoints(ep for ep in self if _py39compat.ep_matches(ep, **params)) - - @property - def names(self): - """ - Return the set of all names of all entry points. - """ - return {ep.name for ep in self} - - @property - def groups(self): - """ - Return the set of all groups of all entry points. - """ - return {ep.group for ep in self} - - @classmethod - def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in cls._from_text(text)) - - @staticmethod - def _from_text(text): - return ( - EntryPoint(name=item.value.name, value=item.value.value, group=item.name) - for item in Sectioned.section_pairs(text or '') - ) - - -class PackagePath(pathlib.PurePosixPath): - """A reference to a path in a package""" - - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() - - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() - - def locate(self): - """Return a path-like object for this path""" - return self.dist.locate_file(self) - - -class FileHash: - def __init__(self, spec): - self.mode, _, self.value = spec.partition('=') - - def __repr__(self): - return f'<FileHash mode: {self.mode} value: {self.value}>' - - -class Distribution(metaclass=abc.ABCMeta): - """A Python distribution package.""" - - @abc.abstractmethod - def read_text(self, filename): - """Attempt to load metadata file given by the name. - - :param filename: The name of the file in the distribution info. - :return: The text if found, otherwise None. - """ - - @abc.abstractmethod - def locate_file(self, path): - """ - Given a path to a file in this distribution, return a path - to it. - """ - - @classmethod - def from_name(cls, name: str): - """Return the Distribution for the given package name. - - :param name: The name of the distribution package to search for. - :return: The Distribution instance (or subclass thereof) for the named - package, if found. - :raises PackageNotFoundError: When the named package's distribution - metadata cannot be found. - :raises ValueError: When an invalid value is supplied for name. - """ - if not name: - raise ValueError("A distribution name is required.") - try: - return next(cls.discover(name=name)) - except StopIteration: - raise PackageNotFoundError(name) - - @classmethod - def discover(cls, **kwargs): - """Return an iterable of Distribution objects for all packages. - - Pass a ``context`` or pass keyword arguments for constructing - a context. - - :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. - """ - context = kwargs.pop('context', None) - if context and kwargs: - raise ValueError("cannot accept context and kwargs") - context = context or DistributionFinder.Context(**kwargs) - return itertools.chain.from_iterable( - resolver(context) for resolver in cls._discover_resolvers() - ) - - @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path - - :param path: a string or path-like object - :return: a concrete Distribution instance for the path - """ - return PathDistribution(pathlib.Path(path)) - - @staticmethod - def _discover_resolvers(): - """Search the meta_path for resolvers.""" - declared = ( - getattr(finder, 'find_distributions', None) for finder in sys.meta_path - ) - return filter(None, declared) - - @property - def metadata(self) -> _meta.PackageMetadata: - """Return the parsed metadata for this Distribution. - - The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. - """ - text = ( - self.read_text('METADATA') - or self.read_text('PKG-INFO') - # This last clause is here to support old egg-info files. Its - # effect is to just end up using the PathDistribution's self._path - # (which points to the egg-info file) attribute unchanged. - or self.read_text('') - ) - return _adapters.Message(email.message_from_string(text)) - - @property - def name(self): - """Return the 'Name' metadata for the distribution package.""" - return self.metadata['Name'] - - @property - def _normalized_name(self): - """Return a normalized version of the name.""" - return Prepared.normalize(self.name) - - @property - def version(self): - """Return the 'Version' metadata for the distribution package.""" - return self.metadata['Version'] - - @property - def entry_points(self): - return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) - - @property - def files(self): - """Files in this distribution. - - :return: List of PackagePath for this distribution or None - - Result is `None` if the metadata file that enumerates files - (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is - missing. - Result may be empty if the metadata exists but is empty. - """ - - def make_file(name, hash=None, size_str=None): - result = PackagePath(name) - result.hash = FileHash(hash) if hash else None - result.size = int(size_str) if size_str else None - result.dist = self - return result - - @pass_none - def make_files(lines): - return list(starmap(make_file, csv.reader(lines))) - - return make_files(self._read_files_distinfo() or self._read_files_egginfo()) - - def _read_files_distinfo(self): - """ - Read the lines of RECORD - """ - text = self.read_text('RECORD') - return text and text.splitlines() - - def _read_files_egginfo(self): - """ - SOURCES.txt might contain literal commas, so wrap each line - in quotes. - """ - text = self.read_text('SOURCES.txt') - return text and map('"{}"'.format, text.splitlines()) - - @property - def requires(self): - """Generated requirements specified for this Distribution""" - reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() - return reqs and list(reqs) - - def _read_dist_info_reqs(self): - return self.metadata.get_all('Requires-Dist') - - def _read_egg_info_reqs(self): - source = self.read_text('requires.txt') - return pass_none(self._deps_from_requires_text)(source) - - @classmethod - def _deps_from_requires_text(cls, source): - return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) - - @staticmethod - def _convert_egg_info_reqs_to_simple_reqs(sections): - """ - Historically, setuptools would solicit and store 'extra' - requirements, including those with environment markers, - in separate sections. More modern tools expect each - dependency to be defined separately, with any relevant - extras and environment markers attached directly to that - requirement. This method converts the former to the - latter. See _test_deps_from_requires_text for an example. - """ - - def make_condition(name): - return name and f'extra == "{name}"' - - def quoted_marker(section): - section = section or '' - extra, sep, markers = section.partition(':') - if extra and markers: - markers = f'({markers})' - conditions = list(filter(None, [markers, make_condition(extra)])) - return '; ' + ' and '.join(conditions) if conditions else '' - - def url_req_space(req): - """ - PEP 508 requires a space between the url_spec and the quoted_marker. - Ref python/importlib_metadata#357. - """ - # '@' is uniquely indicative of a url_req. - return ' ' * ('@' in req) - - for section in sections: - space = url_req_space(section.value) - yield section.value + space + quoted_marker(section.name) - - -class DistributionFinder(MetaPathFinder): - """ - A MetaPathFinder capable of discovering installed distributions. - """ - - class Context: - """ - Keyword arguments presented by the caller to - ``distributions()`` or ``Distribution.discover()`` - to narrow the scope of a search for distributions - in all DistributionFinders. - - Each DistributionFinder may expect any parameters - and should attempt to honor the canonical - parameters defined below when appropriate. - """ - - name = None - """ - Specific name for which a distribution finder should match. - A name of ``None`` matches all distributions. - """ - - def __init__(self, **kwargs): - vars(self).update(kwargs) - - @property - def path(self): - """ - The sequence of directory path that a distribution finder - should search. - - Typically refers to Python installed package paths such as - "site-packages" directories and defaults to ``sys.path``. - """ - return vars(self).get('path', sys.path) - - @abc.abstractmethod - def find_distributions(self, context=Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching the ``context``, - a DistributionFinder.Context instance. - """ - - -class FastPath: - """ - Micro-optimized class for searching a path for - children. - - >>> FastPath('').children() - ['...'] - """ - - @functools.lru_cache() # type: ignore - def __new__(cls, root): - return super().__new__(cls) - - def __init__(self, root): - self.root = root - - def joinpath(self, child): - return pathlib.Path(self.root, child) - - def children(self): - with suppress(Exception): - return os.listdir(self.root or '.') - with suppress(Exception): - return self.zip_children() - return [] - - def zip_children(self): - zip_path = zipp.Path(self.root) - names = zip_path.root.namelist() - self.joinpath = zip_path.joinpath - - return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - - def search(self, name): - return self.lookup(self.mtime).search(name) - - @property - def mtime(self): - with suppress(OSError): - return os.stat(self.root).st_mtime - self.lookup.cache_clear() - - @method_cache - def lookup(self, mtime): - return Lookup(self) - - -class Lookup: - def __init__(self, path: FastPath): - base = os.path.basename(path.root).lower() - base_is_egg = base.endswith(".egg") - self.infos = FreezableDefaultDict(list) - self.eggs = FreezableDefaultDict(list) - - for child in path.children(): - low = child.lower() - if low.endswith((".dist-info", ".egg-info")): - # rpartition is faster than splitext and suitable for this purpose. - name = low.rpartition(".")[0].partition("-")[0] - normalized = Prepared.normalize(name) - self.infos[normalized].append(path.joinpath(child)) - elif base_is_egg and low == "egg-info": - name = base.rpartition(".")[0].partition("-")[0] - legacy_normalized = Prepared.legacy_normalize(name) - self.eggs[legacy_normalized].append(path.joinpath(child)) - - self.infos.freeze() - self.eggs.freeze() - - def search(self, prepared): - infos = ( - self.infos[prepared.normalized] - if prepared - else itertools.chain.from_iterable(self.infos.values()) - ) - eggs = ( - self.eggs[prepared.legacy_normalized] - if prepared - else itertools.chain.from_iterable(self.eggs.values()) - ) - return itertools.chain(infos, eggs) - - -class Prepared: - """ - A prepared search for metadata on a possibly-named package. - """ - - normalized = None - legacy_normalized = None - - def __init__(self, name): - self.name = name - if name is None: - return - self.normalized = self.normalize(name) - self.legacy_normalized = self.legacy_normalize(name) - - @staticmethod - def normalize(name): - """ - PEP 503 normalization plus dashes as underscores. - """ - return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') - - @staticmethod - def legacy_normalize(name): - """ - Normalize the package name as found in the convention in - older packaging tools versions and specs. - """ - return name.lower().replace('-', '_') - - def __bool__(self): - return bool(self.name) - - -@install -class MetadataPathFinder(NullFinder, DistributionFinder): - """A degenerate finder for distribution packages on the file system. - - This finder supplies only a find_distributions() method for versions - of Python that do not have a PathFinder find_distributions(). - """ - - def find_distributions(self, context=DistributionFinder.Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching ``context.name`` - (or all names if ``None`` indicated) along the paths in the list - of directories ``context.path``. - """ - found = self._search_paths(context.name, context.path) - return map(PathDistribution, found) - - @classmethod - def _search_paths(cls, name, paths): - """Find metadata directories in paths heuristically.""" - prepared = Prepared(name) - return itertools.chain.from_iterable( - path.search(prepared) for path in map(FastPath, paths) - ) - - def invalidate_caches(cls): - FastPath.__new__.cache_clear() - - -class PathDistribution(Distribution): - def __init__(self, path: SimplePath): - """Construct a distribution. - - :param path: SimplePath indicating the metadata directory. - """ - self._path = path - - def read_text(self, filename): - with suppress( - FileNotFoundError, - IsADirectoryError, - KeyError, - NotADirectoryError, - PermissionError, - ): - return self._path.joinpath(filename).read_text(encoding='utf-8') - - read_text.__doc__ = Distribution.read_text.__doc__ - - def locate_file(self, path): - return self._path.parent / path - - @property - def _normalized_name(self): - """ - Performance optimization: where possible, resolve the - normalized name from the file system path. - """ - stem = os.path.basename(str(self._path)) - return ( - pass_none(Prepared.normalize)(self._name_from_stem(stem)) - or super()._normalized_name - ) - - @staticmethod - def _name_from_stem(stem): - """ - >>> PathDistribution._name_from_stem('foo-3.0.egg-info') - 'foo' - >>> PathDistribution._name_from_stem('CherryPy-3.0.dist-info') - 'CherryPy' - >>> PathDistribution._name_from_stem('face.egg-info') - 'face' - >>> PathDistribution._name_from_stem('foo.bar') - """ - filename, ext = os.path.splitext(stem) - if ext not in ('.dist-info', '.egg-info'): - return - name, sep, rest = filename.partition('-') - return name - - -def distribution(distribution_name): - """Get the ``Distribution`` instance for the named package. - - :param distribution_name: The name of the distribution package as a string. - :return: A ``Distribution`` instance (or subclass thereof). - """ - return Distribution.from_name(distribution_name) - - -def distributions(**kwargs): - """Get all ``Distribution`` instances in the current environment. - - :return: An iterable of ``Distribution`` instances. - """ - return Distribution.discover(**kwargs) - - -def metadata(distribution_name) -> _meta.PackageMetadata: - """Get the metadata for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: A PackageMetadata containing the parsed metadata. - """ - return Distribution.from_name(distribution_name).metadata - - -def version(distribution_name): - """Get the version string for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: The version string for the package as defined in the package's - "Version" metadata key. - """ - return distribution(distribution_name).version - - -_unique = functools.partial( - unique_everseen, - key=_py39compat.normalized_name, -) -""" -Wrapper for ``distributions`` to return unique distributions by name. -""" - - -def entry_points(**params) -> EntryPoints: - """Return EntryPoint objects for all installed packages. - - Pass selection parameters (group or name) to filter the - result to entry points matching those properties (see - EntryPoints.select()). - - :return: EntryPoints for all installed packages. - """ - eps = itertools.chain.from_iterable( - dist.entry_points for dist in _unique(distributions()) - ) - return EntryPoints(eps).select(**params) - - -def files(distribution_name): - """Return a list of files for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: List of files composing the distribution. - """ - return distribution(distribution_name).files - - -def requires(distribution_name): - """ - Return a list of requirements for the named package. - - :return: An iterator of requirements, suitable for - packaging.requirement.Requirement. - """ - return distribution(distribution_name).requires - - -def packages_distributions() -> Mapping[str, List[str]]: - """ - Return a mapping of top-level packages to their - distributions. - - >>> import collections.abc - >>> pkgs = packages_distributions() - >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) - True - """ - pkg_to_dist = collections.defaultdict(list) - for dist in distributions(): - for pkg in _top_level_declared(dist) or _top_level_inferred(dist): - pkg_to_dist[pkg].append(dist.metadata['Name']) - return dict(pkg_to_dist) - - -def _top_level_declared(dist): - return (dist.read_text('top_level.txt') or '').split() - - -def _top_level_inferred(dist): - return { - f.parts[0] if len(f.parts) > 1 else f.with_suffix('').name - for f in always_iterable(dist.files) - if f.suffix == ".py" - } diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py deleted file mode 100644 index e33cba5e44d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py +++ /dev/null @@ -1,90 +0,0 @@ -import functools -import warnings -import re -import textwrap -import email.message - -from ._text import FoldedCase -from ._compat import pypy_partial - - -# Do not remove prior to 2024-01-01 or Python 3.14 -_warn = functools.partial( - warnings.warn, - "Implicit None on return values is deprecated and will raise KeyErrors.", - DeprecationWarning, - stacklevel=pypy_partial(2), -) - - -class Message(email.message.Message): - multiple_use_keys = set( - map( - FoldedCase, - [ - 'Classifier', - 'Obsoletes-Dist', - 'Platform', - 'Project-URL', - 'Provides-Dist', - 'Provides-Extra', - 'Requires-Dist', - 'Requires-External', - 'Supported-Platform', - 'Dynamic', - ], - ) - ) - """ - Keys that may be indicated multiple times per PEP 566. - """ - - def __new__(cls, orig: email.message.Message): - res = super().__new__(cls) - vars(res).update(vars(orig)) - return res - - def __init__(self, *args, **kwargs): - self._headers = self._repair_headers() - - # suppress spurious error from mypy - def __iter__(self): - return super().__iter__() - - def __getitem__(self, item): - """ - Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. - """ - res = super().__getitem__(item) - if res is None: - _warn() - return res - - def _repair_headers(self): - def redent(value): - "Correct for RFC822 indentation" - if not value or '\n' not in value: - return value - return textwrap.dedent(' ' * 8 + value) - - headers = [(key, redent(value)) for key, value in vars(self)['_headers']] - if self._payload: - headers.append(('Description', self.get_payload())) - return headers - - @property - def json(self): - """ - Convert PackageMetadata to a JSON-compatible format - per PEP 0566. - """ - - def transform(key): - value = self.get_all(key) if key in self.multiple_use_keys else self[key] - if key == 'Keywords': - value = re.split(r'\s+', value) - tk = key.lower().replace('-', '_') - return tk, value - - return dict(map(transform, map(FoldedCase, self))) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py deleted file mode 100644 index cf0954e1a30..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py +++ /dev/null @@ -1,30 +0,0 @@ -import collections - - -# from jaraco.collections 3.3 -class FreezableDefaultDict(collections.defaultdict): - """ - Often it is desirable to prevent the mutation of - a default dict after its initial construction, such - as to prevent mutation during iteration. - - >>> dd = FreezableDefaultDict(list) - >>> dd[0].append('1') - >>> dd.freeze() - >>> dd[1] - [] - >>> len(dd) - 1 - """ - - def __missing__(self, key): - return getattr(self, '_frozen', super().__missing__)(key) - - def freeze(self): - self._frozen = lambda key: self.default_factory() - - -class Pair(collections.namedtuple('Pair', 'name value')): - @classmethod - def parse(cls, text): - return cls(*map(str.strip, text.split("=", 1))) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py deleted file mode 100644 index 84f9eea4f3c..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py +++ /dev/null @@ -1,72 +0,0 @@ -import sys -import platform - - -__all__ = ['install', 'NullFinder', 'Protocol'] - - -try: - from typing import Protocol -except ImportError: # pragma: no cover - # Python 3.7 compatibility - from ..typing_extensions import Protocol # type: ignore - - -def install(cls): - """ - Class decorator for installation on sys.meta_path. - - Adds the backport DistributionFinder to sys.meta_path and - attempts to disable the finder functionality of the stdlib - DistributionFinder. - """ - sys.meta_path.append(cls()) - disable_stdlib_finder() - return cls - - -def disable_stdlib_finder(): - """ - Give the backport primacy for discovering path-based distributions - by monkey-patching the stdlib O_O. - - See #91 for more background for rationale on this sketchy - behavior. - """ - - def matches(finder): - return getattr( - finder, '__module__', None - ) == '_frozen_importlib_external' and hasattr(finder, 'find_distributions') - - for finder in filter(matches, sys.meta_path): # pragma: nocover - del finder.find_distributions - - -class NullFinder: - """ - A "Finder" (aka "MetaClassFinder") that never finds any modules, - but may find distributions. - """ - - @staticmethod - def find_spec(*args, **kwargs): - return None - - # In Python 2, the import system requires finders - # to have a find_module() method, but this usage - # is deprecated in Python 3 in favor of find_spec(). - # For the purposes of this finder (i.e. being present - # on sys.meta_path but having no other import - # system functionality), the two methods are identical. - find_module = find_spec - - -def pypy_partial(val): - """ - Adjust for variable stacklevel on partial under PyPy. - - Workaround for #327. - """ - is_pypy = platform.python_implementation() == 'PyPy' - return val + is_pypy diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py deleted file mode 100644 index 71f66bd03cb..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py +++ /dev/null @@ -1,104 +0,0 @@ -import types -import functools - - -# from jaraco.functools 3.3 -def method_cache(method, cache_wrapper=None): - """ - Wrap lru_cache to support storing the cache data in the object instances. - - Abstracts the common paradigm where the method explicitly saves an - underscore-prefixed protected property on first call and returns that - subsequently. - - >>> class MyClass: - ... calls = 0 - ... - ... @method_cache - ... def method(self, value): - ... self.calls += 1 - ... return value - - >>> a = MyClass() - >>> a.method(3) - 3 - >>> for x in range(75): - ... res = a.method(x) - >>> a.calls - 75 - - Note that the apparent behavior will be exactly like that of lru_cache - except that the cache is stored on each instance, so values in one - instance will not flush values from another, and when an instance is - deleted, so are the cached values for that instance. - - >>> b = MyClass() - >>> for x in range(35): - ... res = b.method(x) - >>> b.calls - 35 - >>> a.method(0) - 0 - >>> a.calls - 75 - - Note that if method had been decorated with ``functools.lru_cache()``, - a.calls would have been 76 (due to the cached value of 0 having been - flushed by the 'b' instance). - - Clear the cache with ``.cache_clear()`` - - >>> a.method.cache_clear() - - Same for a method that hasn't yet been called. - - >>> c = MyClass() - >>> c.method.cache_clear() - - Another cache wrapper may be supplied: - - >>> cache = functools.lru_cache(maxsize=2) - >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) - >>> a = MyClass() - >>> a.method2() - 3 - - Caution - do not subsequently wrap the method with another decorator, such - as ``@property``, which changes the semantics of the function. - - See also - http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ - for another implementation and additional justification. - """ - cache_wrapper = cache_wrapper or functools.lru_cache() - - def wrapper(self, *args, **kwargs): - # it's the first call, replace the method with a cached, bound method - bound_method = types.MethodType(method, self) - cached_method = cache_wrapper(bound_method) - setattr(self, method.__name__, cached_method) - return cached_method(*args, **kwargs) - - # Support cache clear even before cache has been created. - wrapper.cache_clear = lambda: None - - return wrapper - - -# From jaraco.functools 3.3 -def pass_none(func): - """ - Wrap func so it's not called if its first param is None - - >>> print_text = pass_none(print) - >>> print_text('text') - text - >>> print_text(None) - """ - - @functools.wraps(func) - def wrapper(param, *args, **kwargs): - if param is not None: - return func(param, *args, **kwargs) - - return wrapper diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py deleted file mode 100644 index d4ca9b9140e..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py +++ /dev/null @@ -1,73 +0,0 @@ -from itertools import filterfalse - - -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - -# copied from more_itertools 8.8 -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py deleted file mode 100644 index 259b15ba194..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py +++ /dev/null @@ -1,49 +0,0 @@ -from ._compat import Protocol -from typing import Any, Dict, Iterator, List, TypeVar, Union - - -_T = TypeVar("_T") - - -class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover - - def __contains__(self, item: str) -> bool: - ... # pragma: no cover - - def __getitem__(self, key: str) -> str: - ... # pragma: no cover - - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover - - def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: - """ - Return all values associated with a possibly multi-valued key. - """ - - @property - def json(self) -> Dict[str, Union[str, List[str]]]: - """ - A JSON-compatible form of the metadata. - """ - - -class SimplePath(Protocol[_T]): - """ - A minimal subset of pathlib.Path required by PathDistribution. - """ - - def joinpath(self) -> _T: - ... # pragma: no cover - - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover - - @property - def parent(self) -> _T: - ... # pragma: no cover - - def read_text(self) -> str: - ... # pragma: no cover diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py deleted file mode 100644 index cde4558fbbe..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Compatibility layer with Python 3.8/3.9 -""" -from typing import TYPE_CHECKING, Any, Optional - -if TYPE_CHECKING: # pragma: no cover - # Prevent circular imports on runtime. - from . import Distribution, EntryPoint -else: - Distribution = EntryPoint = Any - - -def normalized_name(dist: Distribution) -> Optional[str]: - """ - Honor name normalization for distributions that don't provide ``_normalized_name``. - """ - try: - return dist._normalized_name - except AttributeError: - from . import Prepared # -> delay to prevent circular imports. - - return Prepared.normalize(getattr(dist, "name", None) or dist.metadata['Name']) - - -def ep_matches(ep: EntryPoint, **params) -> bool: - """ - Workaround for ``EntryPoint`` objects without the ``matches`` method. - """ - try: - return ep.matches(**params) - except AttributeError: - from . import EntryPoint # -> delay to prevent circular imports. - - # Reconstruct the EntryPoint object to make sure it is compatible. - return EntryPoint(ep.name, ep.value, ep.group).matches(**params) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py deleted file mode 100644 index c88cfbb2349..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py +++ /dev/null @@ -1,99 +0,0 @@ -import re - -from ._functools import method_cache - - -# from jaraco.text 3.5 -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use in_: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super().lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super().lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py deleted file mode 100644 index 34e3a9950cc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Read resources contained within a package.""" - -from ._common import ( - as_file, - files, - Package, -) - -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - -from .abc import ResourceReader - - -__all__ = [ - 'Package', - 'Resource', - 'ResourceReader', - 'as_file', - 'contents', - 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', -] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py deleted file mode 100644 index ea363d86a56..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py +++ /dev/null @@ -1,170 +0,0 @@ -from contextlib import suppress -from io import TextIOWrapper - -from . import abc - - -class SpecLoaderAdapter: - """ - Adapt a package spec to adapt the underlying loader. - """ - - def __init__(self, spec, adapter=lambda spec: spec.loader): - self.spec = spec - self.loader = adapter(spec) - - def __getattr__(self, name): - return getattr(self.spec, name) - - -class TraversableResourcesLoader: - """ - Adapt a loader to provide TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - def get_resource_reader(self, name): - return CompatibilityFiles(self.spec)._native() - - -def _io_wrapper(file, mode='r', *args, **kwargs): - if mode == 'r': - return TextIOWrapper(file, *args, **kwargs) - elif mode == 'rb': - return file - raise ValueError( - "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode) - ) - - -class CompatibilityFiles: - """ - Adapter for an existing or non-existent resource reader - to provide a compatibility .files(). - """ - - class SpecPath(abc.Traversable): - """ - Path tied to a module spec. - Can be read and exposes the resource reader children. - """ - - def __init__(self, spec, reader): - self._spec = spec - self._reader = reader - - def iterdir(self): - if not self._reader: - return iter(()) - return iter( - CompatibilityFiles.ChildPath(self._reader, path) - for path in self._reader.contents() - ) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - if not self._reader: - return CompatibilityFiles.OrphanPath(other) - return CompatibilityFiles.ChildPath(self._reader, other) - - @property - def name(self): - return self._spec.name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) - - class ChildPath(abc.Traversable): - """ - Path tied to a resource reader child. - Can be read but doesn't expose any meaningful children. - """ - - def __init__(self, reader, name): - self._reader = reader - self._name = name - - def iterdir(self): - return iter(()) - - def is_file(self): - return self._reader.is_resource(self.name) - - def is_dir(self): - return not self.is_file() - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(self.name, other) - - @property - def name(self): - return self._name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper( - self._reader.open_resource(self.name), mode, *args, **kwargs - ) - - class OrphanPath(abc.Traversable): - """ - Orphan path, not tied to a module spec or resource reader. - Can't be read and doesn't expose any meaningful children. - """ - - def __init__(self, *path_parts): - if len(path_parts) < 1: - raise ValueError('Need at least one path part to construct a path') - self._path = path_parts - - def iterdir(self): - return iter(()) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(*self._path, other) - - @property - def name(self): - return self._path[-1] - - def open(self, mode='r', *args, **kwargs): - raise FileNotFoundError("Can't open orphan path") - - def __init__(self, spec): - self.spec = spec - - @property - def _reader(self): - with suppress(AttributeError): - return self.spec.loader.get_resource_reader(self.spec.name) - - def _native(self): - """ - Return the native reader if it supports files(). - """ - reader = self._reader - return reader if hasattr(reader, 'files') else self - - def __getattr__(self, attr): - return getattr(self._reader, attr) - - def files(self): - return CompatibilityFiles.SpecPath(self.spec, self._reader) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - """ - return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py deleted file mode 100644 index 3c6de1cfb2e..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -import pathlib -import tempfile -import functools -import contextlib -import types -import importlib -import inspect -import warnings -import itertools - -from typing import Union, Optional, cast -from .abc import ResourceReader, Traversable - -from ._compat import wrap_spec - -Package = Union[types.ModuleType, str] -Anchor = Package - - -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor -def files(anchor: Optional[Anchor] = None) -> Traversable: - """ - Get a Traversable resource for an anchor. - """ - return from_package(resolve(anchor)) - - -def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: - """ - Return the package's loader if it's a ResourceReader. - """ - # We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore - if reader is None: - return None - return reader(spec.name) # type: ignore - - -def resolve(cand: Optional[Anchor]) -> types.ModuleType: - return cast(types.ModuleType, cand) - - -def _(cand: str) -> types.ModuleType: - return importlib.import_module(cand) - - -def _(cand: None) -> types.ModuleType: - return resolve(_infer_caller().f_globals['__name__']) - - -def _infer_caller(): - """ - Walk the stack and find the frame of the first caller not in this module. - """ - - def is_this_file(frame_info): - return frame_info.filename == __file__ - - def is_wrapper(frame_info): - return frame_info.function == 'wrapper' - - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) - # also exclude 'wrapper' due to singledispatch in the call stack - callers = itertools.filterfalse(is_wrapper, not_this_file) - return next(callers).frame - - -def from_package(package: types.ModuleType): - """ - Return a Traversable object for the given package. - - """ - spec = wrap_spec(package) - reader = spec.loader.get_resource_reader(spec.name) - return reader.files() - - -def _tempfile( - reader, - suffix='', - # gh-93353: Keep a reference to call os.remove() in late Python - # finalization. - *, - _os_remove=os.remove, -): - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on Windows - # properly. - fd, raw_path = tempfile.mkstemp(suffix=suffix) - try: - try: - os.write(fd, reader()) - finally: - os.close(fd) - del reader - yield pathlib.Path(raw_path) - finally: - try: - _os_remove(raw_path) - except FileNotFoundError: - pass - - -def _temp_file(path): - return _tempfile(path.read_bytes, suffix=path.name) - - -def _is_present_dir(path: Traversable) -> bool: - """ - Some Traversables implement ``is_dir()`` to raise an - exception (i.e. ``FileNotFoundError``) when the - directory doesn't exist. This function wraps that call - to always return a boolean and only return True - if there's a dir and it exists. - """ - with contextlib.suppress(FileNotFoundError): - return path.is_dir() - return False - - -def as_file(path): - """ - Given a Traversable object, return that object as a - path on the local file system in a context manager. - """ - return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) - - -@as_file.register(pathlib.Path) -def _(path): - """ - Degenerate behavior for pathlib.Path objects. - """ - yield path - - -def _temp_path(dir: tempfile.TemporaryDirectory): - """ - Wrap tempfile.TemporyDirectory to return a pathlib object. - """ - with dir as result: - yield pathlib.Path(result) - - -def _temp_dir(path): - """ - Given a traversable dir, recursively replicate the whole tree - to the file system in a context manager. - """ - assert path.is_dir() - with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: - yield _write_contents(temp_dir, path) - - -def _write_contents(target, source): - child = target.joinpath(source.name) - if source.is_dir(): - child.mkdir() - for item in source.iterdir(): - _write_contents(child, item) - else: - child.write_bytes(source.read_bytes()) - return child diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py deleted file mode 100644 index 8b5b1d280f3..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py +++ /dev/null @@ -1,108 +0,0 @@ -# flake8: noqa - -import abc -import os -import sys -import pathlib -from contextlib import suppress -from typing import Union - - -if sys.version_info >= (3, 10): - from zipfile import Path as ZipPath # type: ignore -else: - from ..zipp import Path as ZipPath # type: ignore - - -try: - from typing import runtime_checkable # type: ignore -except ImportError: - - def runtime_checkable(cls): # type: ignore - return cls - - -try: - from typing import Protocol # type: ignore -except ImportError: - Protocol = abc.ABC # type: ignore - - -class TraversableResourcesLoader: - """ - Adapt loaders to provide TraversableResources and other - compatibility. - - Used primarily for Python 3.9 and earlier where the native - loaders do not yet implement TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - @property - def path(self): - return self.spec.origin - - def get_resource_reader(self, name): - from . import readers, _adapters - - def _zip_reader(spec): - with suppress(AttributeError): - return readers.ZipReader(spec.loader, spec.name) - - def _namespace_reader(spec): - with suppress(AttributeError, ValueError): - return readers.NamespaceReader(spec.submodule_search_locations) - - def _available_reader(spec): - with suppress(AttributeError): - return spec.loader.get_resource_reader(spec.name) - - def _native_reader(spec): - reader = _available_reader(spec) - return reader if hasattr(reader, 'files') else None - - def _file_reader(spec): - try: - path = pathlib.Path(self.path) - except TypeError: - return None - if path.exists(): - return readers.FileReader(self) - - return ( - # native reader if it supplies 'files' - _native_reader(self.spec) - or - # local ZipReader if a zip module - _zip_reader(self.spec) - or - # local NamespaceReader if a namespace module - _namespace_reader(self.spec) - or - # local FileReader - _file_reader(self.spec) - # fallback - adapt the spec ResourceReader to TraversableReader - or _adapters.CompatibilityFiles(self.spec) - ) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - - Supersedes _adapters.wrap_spec to use TraversableResourcesLoader - from above for older Python compatibility (<3.10). - """ - from . import _adapters - - return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py deleted file mode 100644 index cce05582ffc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py +++ /dev/null @@ -1,35 +0,0 @@ -from itertools import filterfalse - -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py deleted file mode 100644 index 23b6aeafe4f..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py +++ /dev/null @@ -1,170 +0,0 @@ -import abc -import io -import itertools -import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional - -from ._compat import runtime_checkable, Protocol, StrPath - - -__all__ = ["ResourceReader", "Traversable", "TraversableResources"] - - -class ResourceReader(metaclass=abc.ABCMeta): - """Abstract base class for loaders to provide resource reading support.""" - - @abc.abstractmethod - def open_resource(self, resource: Text) -> BinaryIO: - """Return an opened, file-like object for binary reading. - - The 'resource' argument is expected to represent only a file name. - If the resource cannot be found, FileNotFoundError is raised. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def resource_path(self, resource: Text) -> Text: - """Return the file system path to the specified resource. - - The 'resource' argument is expected to represent only a file name. - If the resource does not exist on the file system, raise - FileNotFoundError. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def is_resource(self, path: Text) -> bool: - """Return True if the named 'path' is a resource. - - Files are resources, directories are not. - """ - raise FileNotFoundError - - @abc.abstractmethod - def contents(self) -> Iterable[str]: - """Return an iterable of entries in `package`.""" - raise FileNotFoundError - - -class TraversalError(Exception): - pass - - -@runtime_checkable -class Traversable(Protocol): - """ - An object with a subset of pathlib.Path methods suitable for - traversing directories and opening files. - - Any exceptions that occur when accessing the backing resource - may propagate unaltered. - """ - - @abc.abstractmethod - def iterdir(self) -> Iterator["Traversable"]: - """ - Yield Traversable objects in self - """ - - def read_bytes(self) -> bytes: - """ - Read contents of self as bytes - """ - with self.open('rb') as strm: - return strm.read() - - def read_text(self, encoding: Optional[str] = None) -> str: - """ - Read contents of self as text - """ - with self.open(encoding=encoding) as strm: - return strm.read() - - @abc.abstractmethod - def is_dir(self) -> bool: - """ - Return True if self is a directory - """ - - @abc.abstractmethod - def is_file(self) -> bool: - """ - Return True if self is a file - """ - - def joinpath(self, *descendants: StrPath) -> "Traversable": - """ - Return Traversable resolved with any descendants applied. - - Each descendant should be a path segment relative to self - and each may contain multiple levels separated by - ``posixpath.sep`` (``/``). - """ - if not descendants: - return self - names = itertools.chain.from_iterable( - path.parts for path in map(pathlib.PurePosixPath, descendants) - ) - target = next(names) - matches = ( - traversable for traversable in self.iterdir() if traversable.name == target - ) - try: - match = next(matches) - except StopIteration: - raise TraversalError( - "Target not found during traversal.", target, list(names) - ) - return match.joinpath(*names) - - def __truediv__(self, child: StrPath) -> "Traversable": - """ - Return Traversable child in self - """ - return self.joinpath(child) - - @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): - """ - mode may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as pathlib.Path.open). - - When opening as text, accepts encoding parameters such as those - accepted by io.TextIOWrapper. - """ - - @property - @abc.abstractmethod - def name(self) -> str: - """ - The base name of this object without any parent references. - """ - - -class TraversableResources(ResourceReader): - """ - The required interface for providing traversable - resources. - """ - - @abc.abstractmethod - def files(self) -> "Traversable": - """Return a Traversable object for the loaded package.""" - - def open_resource(self, resource: StrPath) -> io.BufferedReader: - return self.files().joinpath(resource).open('rb') - - def resource_path(self, resource: Any) -> NoReturn: - raise FileNotFoundError(resource) - - def is_resource(self, path: StrPath) -> bool: - return self.files().joinpath(path).is_file() - - def contents(self) -> Iterator[str]: - return (item.name for item in self.files().iterdir()) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py deleted file mode 100644 index ab34db74091..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py +++ /dev/null @@ -1,120 +0,0 @@ -import collections -import pathlib -import operator - -from . import abc - -from ._itertools import unique_everseen -from ._compat import ZipPath - - -def remove_duplicates(items): - return iter(collections.OrderedDict.fromkeys(items)) - - -class FileReader(abc.TraversableResources): - def __init__(self, loader): - self.path = pathlib.Path(loader.path).parent - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path - - -class ZipReader(abc.TraversableResources): - def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' - self.archive = loader.archive - - def open_resource(self, resource): - try: - return super().open_resource(resource) - except KeyError as exc: - raise FileNotFoundError(exc.args[0]) - - def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. - target = self.files().joinpath(path) - return target.is_file() and target.exists() - - def files(self): - return ZipPath(self.archive, self.prefix) - - -class MultiplexedPath(abc.Traversable): - """ - Given a series of Traversable objects, implement a merged - version of the interface across all objects. Useful for - namespace packages which may be multihomed at a single - name. - """ - - def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) - if not self._paths: - message = 'MultiplexedPath must contain at least one path' - raise FileNotFoundError(message) - if not all(path.is_dir() for path in self._paths): - raise NotADirectoryError('MultiplexedPath only supports directories') - - def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) - - def read_bytes(self): - raise FileNotFoundError(f'{self} is not a file') - - def read_text(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - def is_dir(self): - return True - - def is_file(self): - return False - - def joinpath(self, *descendants): - try: - return super().joinpath(*descendants) - except abc.TraversalError: - # One of the paths did not resolve (a directory does not exist). - # Just return something that will not exist. - return self._paths[0].joinpath(*descendants) - - def open(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - @property - def name(self): - return self._paths[0].name - - def __repr__(self): - paths = ', '.join(f"'{path}'" for path in self._paths) - return f'MultiplexedPath({paths})' - - -class NamespaceReader(abc.TraversableResources): - def __init__(self, namespace_path): - if 'NamespacePath' not in str(namespace_path): - raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py deleted file mode 100644 index 7770c922c84..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Interface adapters for low-level readers. -""" - -import abc -import io -import itertools -from typing import BinaryIO, List - -from .abc import Traversable, TraversableResources - - -class SimpleReader(abc.ABC): - """ - The minimum, low-level interface required from a resource - provider. - """ - - @property - @abc.abstractmethod - def package(self) -> str: - """ - The name of the package for which this reader loads resources. - """ - - @abc.abstractmethod - def children(self) -> List['SimpleReader']: - """ - Obtain an iterable of SimpleReader for available - child containers (e.g. directories). - """ - - @abc.abstractmethod - def resources(self) -> List[str]: - """ - Obtain available named resources for this virtual package. - """ - - @abc.abstractmethod - def open_binary(self, resource: str) -> BinaryIO: - """ - Obtain a File-like for a named resource. - """ - - @property - def name(self): - return self.package.split('.')[-1] - - -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader: SimpleReader): - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - -class ResourceHandle(Traversable): - """ - Handle to a named resource in a ResourceReader. - """ - - def __init__(self, parent: ResourceContainer, name: str): - self.parent = parent - self.name = name # type: ignore - - def is_file(self): - return True - - def is_dir(self): - return False - - def open(self, mode='r', *args, **kwargs): - stream = self.parent.reader.open_binary(self.name) - if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) - return stream - - def joinpath(self, name): - raise RuntimeError("Cannot traverse into a resource") - - -class TraversableReader(TraversableResources, SimpleReader): - """ - A TraversableResources based on SimpleReader. Resource providers - may derive from this class to provide the TraversableResources - interface by supplying the SimpleReader interface. - """ - - def files(self): - return ResourceContainer(self) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/context.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/context.py deleted file mode 100644 index 0322c45d4ab..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/context.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import annotations - -import contextlib -import functools -import operator -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.request -import warnings -from typing import Iterator - - -if sys.version_info < (3, 12): - from setuptools.extern.backports import tarfile -else: - import tarfile - - -def pushd(dir: str | os.PathLike) -> Iterator[str | os.PathLike]: - """ - >>> tmp_path = getfixture('tmp_path') - >>> with pushd(tmp_path): - ... assert os.getcwd() == os.fspath(tmp_path) - >>> assert os.getcwd() != os.fspath(tmp_path) - """ - - orig = os.getcwd() - os.chdir(dir) - try: - yield dir - finally: - os.chdir(orig) - - -def tarball( - url, target_dir: str | os.PathLike | None = None -) -> Iterator[str | os.PathLike]: - """ - Get a tarball, extract it, yield, then clean up. - - >>> import urllib.request - >>> url = getfixture('tarfile_served') - >>> target = getfixture('tmp_path') / 'out' - >>> tb = tarball(url, target_dir=target) - >>> import pathlib - >>> with tb as extracted: - ... contents = pathlib.Path(extracted, 'contents.txt').read_text(encoding='utf-8') - >>> assert not os.path.exists(extracted) - """ - if target_dir is None: - target_dir = os.path.basename(url).replace('.tar.gz', '').replace('.tgz', '') - # In the tar command, use --strip-components=1 to strip the first path and - # then - # use -C to cause the files to be extracted to {target_dir}. This ensures - # that we always know where the files were extracted. - os.mkdir(target_dir) - try: - req = urllib.request.urlopen(url) - with tarfile.open(fileobj=req, mode='r|*') as tf: - tf.extractall(path=target_dir, filter=strip_first_component) - yield target_dir - finally: - shutil.rmtree(target_dir) - - -def strip_first_component( - member: tarfile.TarInfo, - path, -) -> tarfile.TarInfo: - _, member.name = member.name.split('/', 1) - return member - - -def _compose(*cmgrs): - """ - Compose any number of dependent context managers into a single one. - - The last, innermost context manager may take arbitrary arguments, but - each successive context manager should accept the result from the - previous as a single parameter. - - Like :func:`jaraco.functools.compose`, behavior works from right to - left, so the context manager should be indicated from outermost to - innermost. - - Example, to create a context manager to change to a temporary - directory: - - >>> temp_dir_as_cwd = _compose(pushd, temp_dir) - >>> with temp_dir_as_cwd() as dir: - ... assert os.path.samefile(os.getcwd(), dir) - """ - - def compose_two(inner, outer): - def composed(*args, **kwargs): - with inner(*args, **kwargs) as saved, outer(saved) as res: - yield res - - return contextlib.contextmanager(composed) - - return functools.reduce(compose_two, reversed(cmgrs)) - - -tarball_cwd = _compose(pushd, tarball) - - -def tarball_context(*args, **kwargs): - warnings.warn( - "tarball_context is deprecated. Use tarball or tarball_cwd instead.", - DeprecationWarning, - stacklevel=2, - ) - pushd_ctx = kwargs.pop('pushd', pushd) - with tarball(*args, **kwargs) as tball, pushd_ctx(tball) as dir: - yield dir - - -def infer_compression(url): - """ - Given a URL or filename, infer the compression code for tar. - - >>> infer_compression('http://foo/bar.tar.gz') - 'z' - >>> infer_compression('http://foo/bar.tgz') - 'z' - >>> infer_compression('file.bz') - 'j' - >>> infer_compression('file.xz') - 'J' - """ - warnings.warn( - "infer_compression is deprecated with no replacement", - DeprecationWarning, - stacklevel=2, - ) - # cheat and just assume it's the last two characters - compression_indicator = url[-2:] - mapping = dict(gz='z', bz='j', xz='J') - # Assume 'z' (gzip) if no match - return mapping.get(compression_indicator, 'z') - - -def temp_dir(remover=shutil.rmtree): - """ - Create a temporary directory context. Pass a custom remover - to override the removal behavior. - - >>> import pathlib - >>> with temp_dir() as the_dir: - ... assert os.path.isdir(the_dir) - ... _ = pathlib.Path(the_dir).joinpath('somefile').write_text('contents', encoding='utf-8') - >>> assert not os.path.exists(the_dir) - """ - temp_dir = tempfile.mkdtemp() - try: - yield temp_dir - finally: - remover(temp_dir) - - -def repo_context(url, branch=None, quiet=True, dest_ctx=temp_dir): - """ - Check out the repo indicated by url. - - If dest_ctx is supplied, it should be a context manager - to yield the target directory for the check out. - """ - exe = 'git' if 'git' in url else 'hg' - with dest_ctx() as repo_dir: - cmd = [exe, 'clone', url, repo_dir] - if branch: - cmd.extend(['--branch', branch]) - devnull = open(os.path.devnull, 'w') - stdout = devnull if quiet else None - subprocess.check_call(cmd, stdout=stdout) - yield repo_dir - - -def null(): - """ - A null context suitable to stand in for a meaningful context. - - >>> with null() as value: - ... assert value is None - - This context is most useful when dealing with two or more code - branches but only some need a context. Wrap the others in a null - context to provide symmetry across all options. - """ - warnings.warn( - "null is deprecated. Use contextlib.nullcontext", - DeprecationWarning, - stacklevel=2, - ) - return contextlib.nullcontext() - - -class ExceptionTrap: - """ - A context manager that will catch certain exceptions and provide an - indication they occurred. - - >>> with ExceptionTrap() as trap: - ... raise Exception() - >>> bool(trap) - True - - >>> with ExceptionTrap() as trap: - ... pass - >>> bool(trap) - False - - >>> with ExceptionTrap(ValueError) as trap: - ... raise ValueError("1 + 1 is not 3") - >>> bool(trap) - True - >>> trap.value - ValueError('1 + 1 is not 3') - >>> trap.tb - <traceback object at ...> - - >>> with ExceptionTrap(ValueError) as trap: - ... raise Exception() - Traceback (most recent call last): - ... - Exception - - >>> bool(trap) - False - """ - - exc_info = None, None, None - - def __init__(self, exceptions=(Exception,)): - self.exceptions = exceptions - - def __enter__(self): - return self - - @property - def type(self): - return self.exc_info[0] - - @property - def value(self): - return self.exc_info[1] - - @property - def tb(self): - return self.exc_info[2] - - def __exit__(self, *exc_info): - type = exc_info[0] - matches = type and issubclass(type, self.exceptions) - if matches: - self.exc_info = exc_info - return matches - - def __bool__(self): - return bool(self.type) - - def raises(self, func, *, _test=bool): - """ - Wrap func and replace the result with the truth - value of the trap (True if an exception occurred). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> raises = ExceptionTrap(ValueError).raises - - Now decorate a function that always fails. - - >>> @raises - ... def fail(): - ... raise ValueError('failed') - >>> fail() - True - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - with ExceptionTrap(self.exceptions) as trap: - func(*args, **kwargs) - return _test(trap) - - return wrapper - - def passes(self, func): - """ - Wrap func and replace the result with the truth - value of the trap (True if no exception). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> passes = ExceptionTrap(ValueError).passes - - Now decorate a function that always fails. - - >>> @passes - ... def fail(): - ... raise ValueError('failed') - - >>> fail() - False - """ - return self.raises(func, _test=operator.not_) - - -class suppress(contextlib.suppress, contextlib.ContextDecorator): - """ - A version of contextlib.suppress with decorator support. - - >>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ - - -class on_interrupt(contextlib.ContextDecorator): - """ - Replace a KeyboardInterrupt with SystemExit(1) - - >>> def do_interrupt(): - ... raise KeyboardInterrupt() - >>> on_interrupt('error')(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 1 - >>> on_interrupt('error', code=255)(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 255 - >>> on_interrupt('suppress')(do_interrupt)() - >>> with __import__('pytest').raises(KeyboardInterrupt): - ... on_interrupt('ignore')(do_interrupt)() - """ - - def __init__(self, action='error', /, code=1): - self.action = action - self.code = code - - def __enter__(self): - return self - - def __exit__(self, exctype, excinst, exctb): - if exctype is not KeyboardInterrupt or self.action == 'ignore': - return - elif self.action == 'error': - raise SystemExit(self.code) from excinst - return self.action == 'suppress' diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.py deleted file mode 100644 index 130b87a4859..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.py +++ /dev/null @@ -1,633 +0,0 @@ -import collections.abc -import functools -import inspect -import itertools -import operator -import time -import types -import warnings - -import setuptools.extern.more_itertools - - -def compose(*funcs): - """ - Compose any number of unary functions into a single unary function. - - >>> import textwrap - >>> expected = str.strip(textwrap.dedent(compose.__doc__)) - >>> strip_and_dedent = compose(str.strip, textwrap.dedent) - >>> strip_and_dedent(compose.__doc__) == expected - True - - Compose also allows the innermost function to take arbitrary arguments. - - >>> round_three = lambda x: round(x, ndigits=3) - >>> f = compose(round_three, int.__truediv__) - >>> [f(3*x, x+1) for x in range(1,10)] - [1.5, 2.0, 2.25, 2.4, 2.5, 2.571, 2.625, 2.667, 2.7] - """ - - def compose_two(f1, f2): - return lambda *args, **kwargs: f1(f2(*args, **kwargs)) - - return functools.reduce(compose_two, funcs) - - -def once(func): - """ - Decorate func so it's only ever called the first time. - - This decorator can ensure that an expensive or non-idempotent function - will not be expensive on subsequent calls and is idempotent. - - >>> add_three = once(lambda a: a+3) - >>> add_three(3) - 6 - >>> add_three(9) - 6 - >>> add_three('12') - 6 - - To reset the stored value, simply clear the property ``saved_result``. - - >>> del add_three.saved_result - >>> add_three(9) - 12 - >>> add_three(8) - 12 - - Or invoke 'reset()' on it. - - >>> add_three.reset() - >>> add_three(-3) - 0 - >>> add_three(0) - 0 - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if not hasattr(wrapper, 'saved_result'): - wrapper.saved_result = func(*args, **kwargs) - return wrapper.saved_result - - wrapper.reset = lambda: vars(wrapper).__delitem__('saved_result') - return wrapper - - -def method_cache(method, cache_wrapper=functools.lru_cache()): - """ - Wrap lru_cache to support storing the cache data in the object instances. - - Abstracts the common paradigm where the method explicitly saves an - underscore-prefixed protected property on first call and returns that - subsequently. - - >>> class MyClass: - ... calls = 0 - ... - ... @method_cache - ... def method(self, value): - ... self.calls += 1 - ... return value - - >>> a = MyClass() - >>> a.method(3) - 3 - >>> for x in range(75): - ... res = a.method(x) - >>> a.calls - 75 - - Note that the apparent behavior will be exactly like that of lru_cache - except that the cache is stored on each instance, so values in one - instance will not flush values from another, and when an instance is - deleted, so are the cached values for that instance. - - >>> b = MyClass() - >>> for x in range(35): - ... res = b.method(x) - >>> b.calls - 35 - >>> a.method(0) - 0 - >>> a.calls - 75 - - Note that if method had been decorated with ``functools.lru_cache()``, - a.calls would have been 76 (due to the cached value of 0 having been - flushed by the 'b' instance). - - Clear the cache with ``.cache_clear()`` - - >>> a.method.cache_clear() - - Same for a method that hasn't yet been called. - - >>> c = MyClass() - >>> c.method.cache_clear() - - Another cache wrapper may be supplied: - - >>> cache = functools.lru_cache(maxsize=2) - >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) - >>> a = MyClass() - >>> a.method2() - 3 - - Caution - do not subsequently wrap the method with another decorator, such - as ``@property``, which changes the semantics of the function. - - See also - http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ - for another implementation and additional justification. - """ - - def wrapper(self, *args, **kwargs): - # it's the first call, replace the method with a cached, bound method - bound_method = types.MethodType(method, self) - cached_method = cache_wrapper(bound_method) - setattr(self, method.__name__, cached_method) - return cached_method(*args, **kwargs) - - # Support cache clear even before cache has been created. - wrapper.cache_clear = lambda: None - - return _special_method_cache(method, cache_wrapper) or wrapper - - -def _special_method_cache(method, cache_wrapper): - """ - Because Python treats special methods differently, it's not - possible to use instance attributes to implement the cached - methods. - - Instead, install the wrapper method under a different name - and return a simple proxy to that wrapper. - - https://github.com/jaraco/jaraco.functools/issues/5 - """ - name = method.__name__ - special_names = '__getattr__', '__getitem__' - - if name not in special_names: - return None - - wrapper_name = '__cached' + name - - def proxy(self, /, *args, **kwargs): - if wrapper_name not in vars(self): - bound = types.MethodType(method, self) - cache = cache_wrapper(bound) - setattr(self, wrapper_name, cache) - else: - cache = getattr(self, wrapper_name) - return cache(*args, **kwargs) - - return proxy - - -def apply(transform): - """ - Decorate a function with a transform function that is - invoked on results returned from the decorated function. - - >>> @apply(reversed) - ... def get_numbers(start): - ... "doc for get_numbers" - ... return range(start, start+3) - >>> list(get_numbers(4)) - [6, 5, 4] - >>> get_numbers.__doc__ - 'doc for get_numbers' - """ - - def wrap(func): - return functools.wraps(func)(compose(transform, func)) - - return wrap - - -def result_invoke(action): - r""" - Decorate a function with an action function that is - invoked on the results returned from the decorated - function (for its side effect), then return the original - result. - - >>> @result_invoke(print) - ... def add_two(a, b): - ... return a + b - >>> x = add_two(2, 3) - 5 - >>> x - 5 - """ - - def wrap(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - action(result) - return result - - return wrapper - - return wrap - - -def invoke(f, /, *args, **kwargs): - """ - Call a function for its side effect after initialization. - - The benefit of using the decorator instead of simply invoking a function - after defining it is that it makes explicit the author's intent for the - function to be called immediately. Whereas if one simply calls the - function immediately, it's less obvious if that was intentional or - incidental. It also avoids repeating the name - the two actions, defining - the function and calling it immediately are modeled separately, but linked - by the decorator construct. - - The benefit of having a function construct (opposed to just invoking some - behavior inline) is to serve as a scope in which the behavior occurs. It - avoids polluting the global namespace with local variables, provides an - anchor on which to attach documentation (docstring), keeps the behavior - logically separated (instead of conceptually separated or not separated at - all), and provides potential to re-use the behavior for testing or other - purposes. - - This function is named as a pithy way to communicate, "call this function - primarily for its side effect", or "while defining this function, also - take it aside and call it". It exists because there's no Python construct - for "define and call" (nor should there be, as decorators serve this need - just fine). The behavior happens immediately and synchronously. - - >>> @invoke - ... def func(): print("called") - called - >>> func() - called - - Use functools.partial to pass parameters to the initial call - - >>> @functools.partial(invoke, name='bingo') - ... def func(name): print('called with', name) - called with bingo - """ - f(*args, **kwargs) - return f - - -class Throttler: - """Rate-limit a function (or other callable).""" - - def __init__(self, func, max_rate=float('Inf')): - if isinstance(func, Throttler): - func = func.func - self.func = func - self.max_rate = max_rate - self.reset() - - def reset(self): - self.last_called = 0 - - def __call__(self, *args, **kwargs): - self._wait() - return self.func(*args, **kwargs) - - def _wait(self): - """Ensure at least 1/max_rate seconds from last call.""" - elapsed = time.time() - self.last_called - must_wait = 1 / self.max_rate - elapsed - time.sleep(max(0, must_wait)) - self.last_called = time.time() - - def __get__(self, obj, owner=None): - return first_invoke(self._wait, functools.partial(self.func, obj)) - - -def first_invoke(func1, func2): - """ - Return a function that when invoked will invoke func1 without - any parameters (for its side effect) and then invoke func2 - with whatever parameters were passed, returning its result. - """ - - def wrapper(*args, **kwargs): - func1() - return func2(*args, **kwargs) - - return wrapper - - -method_caller = first_invoke( - lambda: warnings.warn( - '`jaraco.functools.method_caller` is deprecated, ' - 'use `operator.methodcaller` instead', - DeprecationWarning, - stacklevel=3, - ), - operator.methodcaller, -) - - -def retry_call(func, cleanup=lambda: None, retries=0, trap=()): - """ - Given a callable func, trap the indicated exceptions - for up to 'retries' times, invoking cleanup on the - exception. On the final attempt, allow any exceptions - to propagate. - """ - attempts = itertools.count() if retries == float('inf') else range(retries) - for _ in attempts: - try: - return func() - except trap: - cleanup() - - return func() - - -def retry(*r_args, **r_kwargs): - """ - Decorator wrapper for retry_call. Accepts arguments to retry_call - except func and then returns a decorator for the decorated function. - - Ex: - - >>> @retry(retries=3) - ... def my_func(a, b): - ... "this is my funk" - ... print(a, b) - >>> my_func.__doc__ - 'this is my funk' - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*f_args, **f_kwargs): - bound = functools.partial(func, *f_args, **f_kwargs) - return retry_call(bound, *r_args, **r_kwargs) - - return wrapper - - return decorate - - -def print_yielded(func): - """ - Convert a generator into a function that prints all yielded elements. - - >>> @print_yielded - ... def x(): - ... yield 3; yield None - >>> x() - 3 - None - """ - print_all = functools.partial(map, print) - print_results = compose(more_itertools.consume, print_all, func) - return functools.wraps(func)(print_results) - - -def pass_none(func): - """ - Wrap func so it's not called if its first param is None. - - >>> print_text = pass_none(print) - >>> print_text('text') - text - >>> print_text(None) - """ - - @functools.wraps(func) - def wrapper(param, /, *args, **kwargs): - if param is not None: - return func(param, *args, **kwargs) - return None - - return wrapper - - -def assign_params(func, namespace): - """ - Assign parameters from namespace where func solicits. - - >>> def func(x, y=3): - ... print(x, y) - >>> assigned = assign_params(func, dict(x=2, z=4)) - >>> assigned() - 2 3 - - The usual errors are raised if a function doesn't receive - its required parameters: - - >>> assigned = assign_params(func, dict(y=3, z=4)) - >>> assigned() - Traceback (most recent call last): - TypeError: func() ...argument... - - It even works on methods: - - >>> class Handler: - ... def meth(self, arg): - ... print(arg) - >>> assign_params(Handler().meth, dict(arg='crystal', foo='clear'))() - crystal - """ - sig = inspect.signature(func) - params = sig.parameters.keys() - call_ns = {k: namespace[k] for k in params if k in namespace} - return functools.partial(func, **call_ns) - - -def save_method_args(method): - """ - Wrap a method such that when it is called, the args and kwargs are - saved on the method. - - >>> class MyClass: - ... @save_method_args - ... def method(self, a, b): - ... print(a, b) - >>> my_ob = MyClass() - >>> my_ob.method(1, 2) - 1 2 - >>> my_ob._saved_method.args - (1, 2) - >>> my_ob._saved_method.kwargs - {} - >>> my_ob.method(a=3, b='foo') - 3 foo - >>> my_ob._saved_method.args - () - >>> my_ob._saved_method.kwargs == dict(a=3, b='foo') - True - - The arguments are stored on the instance, allowing for - different instance to save different args. - - >>> your_ob = MyClass() - >>> your_ob.method({str('x'): 3}, b=[4]) - {'x': 3} [4] - >>> your_ob._saved_method.args - ({'x': 3},) - >>> my_ob._saved_method.args - () - """ - args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') - - @functools.wraps(method) - def wrapper(self, /, *args, **kwargs): - attr_name = '_saved_' + method.__name__ - attr = args_and_kwargs(args, kwargs) - setattr(self, attr_name, attr) - return method(self, *args, **kwargs) - - return wrapper - - -def except_(*exceptions, replace=None, use=None): - """ - Replace the indicated exceptions, if raised, with the indicated - literal replacement or evaluated expression (if present). - - >>> safe_int = except_(ValueError)(int) - >>> safe_int('five') - >>> safe_int('5') - 5 - - Specify a literal replacement with ``replace``. - - >>> safe_int_r = except_(ValueError, replace=0)(int) - >>> safe_int_r('five') - 0 - - Provide an expression to ``use`` to pass through particular parameters. - - >>> safe_int_pt = except_(ValueError, use='args[0]')(int) - >>> safe_int_pt('five') - 'five' - - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except exceptions: - try: - return eval(use) - except TypeError: - return replace - - return wrapper - - return decorate - - -def identity(x): - """ - Return the argument. - - >>> o = object() - >>> identity(o) is o - True - """ - return x - - -def bypass_when(check, *, _op=identity): - """ - Decorate a function to return its parameter when ``check``. - - >>> bypassed = [] # False - - >>> @bypass_when(bypassed) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> bypassed[:] = [object()] # True - >>> double(2) - 2 - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(param, /): - return param if _op(check) else func(param) - - return wrapper - - return decorate - - -def bypass_unless(check): - """ - Decorate a function to return its parameter unless ``check``. - - >>> enabled = [object()] # True - - >>> @bypass_unless(enabled) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> del enabled[:] # False - >>> double(2) - 2 - """ - return bypass_when(check, _op=operator.not_) - - -def _splat_inner(args, func): - """Splat args to func.""" - return func(*args) - - -@_splat_inner.register -def _(args: collections.abc.Mapping, func): - """Splat kargs to func as kwargs.""" - return func(**args) - - -def splat(func): - """ - Wrap func to expect its parameters to be passed positionally in a tuple. - - Has a similar effect to that of ``itertools.starmap`` over - simple ``map``. - - >>> pairs = [(-1, 1), (0, 2)] - >>> setuptools.extern.more_itertools.consume(itertools.starmap(print, pairs)) - -1 1 - 0 2 - >>> setuptools.extern.more_itertools.consume(map(splat(print), pairs)) - -1 1 - 0 2 - - The approach generalizes to other iterators that don't have a "star" - equivalent, such as a "starfilter". - - >>> list(filter(splat(operator.add), pairs)) - [(0, 2)] - - Splat also accepts a mapping argument. - - >>> def is_nice(msg, code): - ... return "smile" in msg or code == 0 - >>> msgs = [ - ... dict(msg='smile!', code=20), - ... dict(msg='error :(', code=1), - ... dict(msg='unknown', code=0), - ... ] - >>> for msg in filter(splat(is_nice), msgs): - ... print(msg) - {'msg': 'smile!', 'code': 20} - {'msg': 'unknown', 'code': 0} - """ - return functools.wraps(func)(functools.partial(_splat_inner, func=func)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.pyi deleted file mode 100644 index c2b9ab1757e..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.pyi +++ /dev/null @@ -1,128 +0,0 @@ -from collections.abc import Callable, Hashable, Iterator -from functools import partial -from operator import methodcaller -import sys -from typing import ( - Any, - Generic, - Protocol, - TypeVar, - overload, -) - -if sys.version_info >= (3, 10): - from typing import Concatenate, ParamSpec -else: - from typing_extensions import Concatenate, ParamSpec - -_P = ParamSpec('_P') -_R = TypeVar('_R') -_T = TypeVar('_T') -_R1 = TypeVar('_R1') -_R2 = TypeVar('_R2') -_V = TypeVar('_V') -_S = TypeVar('_S') -_R_co = TypeVar('_R_co', covariant=True) - -class _OnceCallable(Protocol[_P, _R]): - saved_result: _R - reset: Callable[[], None] - def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... - -class _ProxyMethodCacheWrapper(Protocol[_R_co]): - cache_clear: Callable[[], None] - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -class _MethodCacheWrapper(Protocol[_R_co]): - def cache_clear(self) -> None: ... - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -# `compose()` overloads below will cover most use cases. - -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[_P, _R], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R1], _R], - __func3: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R2], _R], - __func3: Callable[[_R1], _R2], - __func4: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -def once(func: Callable[_P, _R]) -> _OnceCallable[_P, _R]: ... -def method_cache( - method: Callable[..., _R], - cache_wrapper: Callable[[Callable[..., _R]], _MethodCacheWrapper[_R]] = ..., -) -> _MethodCacheWrapper[_R] | _ProxyMethodCacheWrapper[_R]: ... -def apply( - transform: Callable[[_R], _T] -) -> Callable[[Callable[_P, _R]], Callable[_P, _T]]: ... -def result_invoke( - action: Callable[[_R], Any] -) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: ... -def invoke( - f: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... -def call_aside( - f: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... - -class Throttler(Generic[_R]): - last_called: float - func: Callable[..., _R] - max_rate: float - def __init__( - self, func: Callable[..., _R] | Throttler[_R], max_rate: float = ... - ) -> None: ... - def reset(self) -> None: ... - def __call__(self, *args: Any, **kwargs: Any) -> _R: ... - def __get__(self, obj: Any, owner: type[Any] | None = ...) -> Callable[..., _R]: ... - -def first_invoke( - func1: Callable[..., Any], func2: Callable[_P, _R] -) -> Callable[_P, _R]: ... - -method_caller: Callable[..., methodcaller] - -def retry_call( - func: Callable[..., _R], - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> _R: ... -def retry( - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> Callable[[Callable[..., _R]], Callable[..., _R]]: ... -def print_yielded(func: Callable[_P, Iterator[Any]]) -> Callable[_P, None]: ... -def pass_none( - func: Callable[Concatenate[_T, _P], _R] -) -> Callable[Concatenate[_T, _P], _R]: ... -def assign_params( - func: Callable[..., _R], namespace: dict[str, Any] -) -> partial[_R]: ... -def save_method_args( - method: Callable[Concatenate[_S, _P], _R] -) -> Callable[Concatenate[_S, _P], _R]: ... -def except_( - *exceptions: type[BaseException], replace: Any = ..., use: Any = ... -) -> Callable[[Callable[_P, Any]], Callable[_P, Any]]: ... -def identity(x: _T) -> _T: ... -def bypass_when( - check: _V, *, _op: Callable[[_V], Any] = ... -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... -def bypass_unless( - check: Any, -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py deleted file mode 100644 index a0306d5ff5c..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py +++ /dev/null @@ -1,599 +0,0 @@ -import re -import itertools -import textwrap -import functools - -try: - from importlib.resources import files # type: ignore -except ImportError: # pragma: nocover - from setuptools.extern.importlib_resources import files # type: ignore - -from setuptools.extern.jaraco.functools import compose, method_cache -from setuptools.extern.jaraco.context import ExceptionTrap - - -def substitution(old, new): - """ - Return a function that will perform a substitution on a string - """ - return lambda s: s.replace(old, new) - - -def multi_substitution(*substitutions): - """ - Take a sequence of pairs specifying substitutions, and create - a function that performs those substitutions. - - >>> multi_substitution(('foo', 'bar'), ('bar', 'baz'))('foo') - 'baz' - """ - substitutions = itertools.starmap(substitution, substitutions) - # compose function applies last function first, so reverse the - # substitutions to get the expected order. - substitutions = reversed(tuple(substitutions)) - return compose(*substitutions) - - -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use ``in_``: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super().lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super().lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) - - -# Python 3.8 compatibility -_unicode_trap = ExceptionTrap(UnicodeDecodeError) - - -@_unicode_trap.passes -def is_decodable(value): - r""" - Return True if the supplied value is decodable (using the default - encoding). - - >>> is_decodable(b'\xff') - False - >>> is_decodable(b'\x32') - True - """ - value.decode() - - -def is_binary(value): - r""" - Return True if the value appears to be binary (that is, it's a byte - string and isn't decodable). - - >>> is_binary(b'\xff') - True - >>> is_binary('\xff') - False - """ - return isinstance(value, bytes) and not is_decodable(value) - - -def trim(s): - r""" - Trim something like a docstring to remove the whitespace that - is common due to indentation and formatting. - - >>> trim("\n\tfoo = bar\n\t\tbar = baz\n") - 'foo = bar\n\tbar = baz' - """ - return textwrap.dedent(s).strip() - - -def wrap(s): - """ - Wrap lines of text, retaining existing newlines as - paragraph markers. - - >>> print(wrap(lorem_ipsum)) - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do - eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad - minim veniam, quis nostrud exercitation ullamco laboris nisi ut - aliquip ex ea commodo consequat. Duis aute irure dolor in - reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla - pariatur. Excepteur sint occaecat cupidatat non proident, sunt in - culpa qui officia deserunt mollit anim id est laborum. - <BLANKLINE> - Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam - varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus - magna felis sollicitudin mauris. Integer in mauris eu nibh euismod - gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis - risus a elit. Etiam tempor. Ut ullamcorper, ligula eu tempor congue, - eros est euismod turpis, id tincidunt sapien risus a quam. Maecenas - fermentum consequat mi. Donec fermentum. Pellentesque malesuada nulla - a mi. Duis sapien sem, aliquet nec, commodo eget, consequat quis, - neque. Aliquam faucibus, elit ut dictum aliquet, felis nisl adipiscing - sapien, sed malesuada diam lacus eget erat. Cras mollis scelerisque - nunc. Nullam arcu. Aliquam consequat. Curabitur augue lorem, dapibus - quis, laoreet et, pretium ac, nisi. Aenean magna nisl, mollis quis, - molestie eu, feugiat in, orci. In hac habitasse platea dictumst. - """ - paragraphs = s.splitlines() - wrapped = ('\n'.join(textwrap.wrap(para)) for para in paragraphs) - return '\n\n'.join(wrapped) - - -def unwrap(s): - r""" - Given a multi-line string, return an unwrapped version. - - >>> wrapped = wrap(lorem_ipsum) - >>> wrapped.count('\n') - 20 - >>> unwrapped = unwrap(wrapped) - >>> unwrapped.count('\n') - 1 - >>> print(unwrapped) - Lorem ipsum dolor sit amet, consectetur adipiscing ... - Curabitur pretium tincidunt lacus. Nulla gravida orci ... - - """ - paragraphs = re.split(r'\n\n+', s) - cleaned = (para.replace('\n', ' ') for para in paragraphs) - return '\n'.join(cleaned) - - - - -class Splitter(object): - """object that will split a string with the given arguments for each call - - >>> s = Splitter(',') - >>> s('hello, world, this is your, master calling') - ['hello', ' world', ' this is your', ' master calling'] - """ - - def __init__(self, *args): - self.args = args - - def __call__(self, s): - return s.split(*self.args) - - -def indent(string, prefix=' ' * 4): - """ - >>> indent('foo') - ' foo' - """ - return prefix + string - - -class WordSet(tuple): - """ - Given an identifier, return the words that identifier represents, - whether in camel case, underscore-separated, etc. - - >>> WordSet.parse("camelCase") - ('camel', 'Case') - - >>> WordSet.parse("under_sep") - ('under', 'sep') - - Acronyms should be retained - - >>> WordSet.parse("firstSNL") - ('first', 'SNL') - - >>> WordSet.parse("you_and_I") - ('you', 'and', 'I') - - >>> WordSet.parse("A simple test") - ('A', 'simple', 'test') - - Multiple caps should not interfere with the first cap of another word. - - >>> WordSet.parse("myABCClass") - ('my', 'ABC', 'Class') - - The result is a WordSet, so you can get the form you need. - - >>> WordSet.parse("myABCClass").underscore_separated() - 'my_ABC_Class' - - >>> WordSet.parse('a-command').camel_case() - 'ACommand' - - >>> WordSet.parse('someIdentifier').lowered().space_separated() - 'some identifier' - - Slices of the result should return another WordSet. - - >>> WordSet.parse('taken-out-of-context')[1:].underscore_separated() - 'out_of_context' - - >>> WordSet.from_class_name(WordSet()).lowered().space_separated() - 'word set' - - >>> example = WordSet.parse('figured it out') - >>> example.headless_camel_case() - 'figuredItOut' - >>> example.dash_separated() - 'figured-it-out' - - """ - - _pattern = re.compile('([A-Z]?[a-z]+)|([A-Z]+(?![a-z]))') - - def capitalized(self): - return WordSet(word.capitalize() for word in self) - - def lowered(self): - return WordSet(word.lower() for word in self) - - def camel_case(self): - return ''.join(self.capitalized()) - - def headless_camel_case(self): - words = iter(self) - first = next(words).lower() - new_words = itertools.chain((first,), WordSet(words).camel_case()) - return ''.join(new_words) - - def underscore_separated(self): - return '_'.join(self) - - def dash_separated(self): - return '-'.join(self) - - def space_separated(self): - return ' '.join(self) - - def trim_right(self, item): - """ - Remove the item from the end of the set. - - >>> WordSet.parse('foo bar').trim_right('foo') - ('foo', 'bar') - >>> WordSet.parse('foo bar').trim_right('bar') - ('foo',) - >>> WordSet.parse('').trim_right('bar') - () - """ - return self[:-1] if self and self[-1] == item else self - - def trim_left(self, item): - """ - Remove the item from the beginning of the set. - - >>> WordSet.parse('foo bar').trim_left('foo') - ('bar',) - >>> WordSet.parse('foo bar').trim_left('bar') - ('foo', 'bar') - >>> WordSet.parse('').trim_left('bar') - () - """ - return self[1:] if self and self[0] == item else self - - def trim(self, item): - """ - >>> WordSet.parse('foo bar').trim('foo') - ('bar',) - """ - return self.trim_left(item).trim_right(item) - - def __getitem__(self, item): - result = super(WordSet, self).__getitem__(item) - if isinstance(item, slice): - result = WordSet(result) - return result - - @classmethod - def parse(cls, identifier): - matches = cls._pattern.finditer(identifier) - return WordSet(match.group(0) for match in matches) - - @classmethod - def from_class_name(cls, subject): - return cls.parse(subject.__class__.__name__) - - -# for backward compatibility -words = WordSet.parse - - -def simple_html_strip(s): - r""" - Remove HTML from the string `s`. - - >>> str(simple_html_strip('')) - '' - - >>> print(simple_html_strip('A <bold>stormy</bold> day in paradise')) - A stormy day in paradise - - >>> print(simple_html_strip('Somebody <!-- do not --> tell the truth.')) - Somebody tell the truth. - - >>> print(simple_html_strip('What about<br/>\nmultiple lines?')) - What about - multiple lines? - """ - html_stripper = re.compile('(<!--.*?-->)|(<[^>]*>)|([^<]+)', re.DOTALL) - texts = (match.group(3) or '' for match in html_stripper.finditer(s)) - return ''.join(texts) - - -class SeparatedValues(str): - """ - A string separated by a separator. Overrides __iter__ for getting - the values. - - >>> list(SeparatedValues('a,b,c')) - ['a', 'b', 'c'] - - Whitespace is stripped and empty values are discarded. - - >>> list(SeparatedValues(' a, b , c, ')) - ['a', 'b', 'c'] - """ - - separator = ',' - - def __iter__(self): - parts = self.split(self.separator) - return filter(None, (part.strip() for part in parts)) - - -class Stripper: - r""" - Given a series of lines, find the common prefix and strip it from them. - - >>> lines = [ - ... 'abcdefg\n', - ... 'abc\n', - ... 'abcde\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix - 'abc' - >>> list(res.lines) - ['defg\n', '\n', 'de\n'] - - If no prefix is common, nothing should be stripped. - - >>> lines = [ - ... 'abcd\n', - ... '1234\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix = '' - >>> list(res.lines) - ['abcd\n', '1234\n'] - """ - - def __init__(self, prefix, lines): - self.prefix = prefix - self.lines = map(self, lines) - - @classmethod - def strip_prefix(cls, lines): - prefix_lines, lines = itertools.tee(lines) - prefix = functools.reduce(cls.common_prefix, prefix_lines) - return cls(prefix, lines) - - def __call__(self, line): - if not self.prefix: - return line - null, prefix, rest = line.partition(self.prefix) - return rest - - @staticmethod - def common_prefix(s1, s2): - """ - Return the common prefix of two lines. - """ - index = min(len(s1), len(s2)) - while s1[:index] != s2[:index]: - index -= 1 - return s1[:index] - - -def remove_prefix(text, prefix): - """ - Remove the prefix from the text if it exists. - - >>> remove_prefix('underwhelming performance', 'underwhelming ') - 'performance' - - >>> remove_prefix('something special', 'sample') - 'something special' - """ - null, prefix, rest = text.rpartition(prefix) - return rest - - -def remove_suffix(text, suffix): - """ - Remove the suffix from the text if it exists. - - >>> remove_suffix('name.git', '.git') - 'name' - - >>> remove_suffix('something special', 'sample') - 'something special' - """ - rest, suffix, null = text.partition(suffix) - return rest - - -def normalize_newlines(text): - r""" - Replace alternate newlines with the canonical newline. - - >>> normalize_newlines('Lorem Ipsum\u2029') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\r\n') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\x85') - 'Lorem Ipsum\n' - """ - newlines = ['\r\n', '\r', '\n', '\u0085', '\u2028', '\u2029'] - pattern = '|'.join(newlines) - return re.sub(pattern, '\n', text) - - -def _nonblank(str): - return str and not str.startswith('#') - - -def yield_lines(iterable): - r""" - Yield valid lines of a string or iterable. - - >>> list(yield_lines('')) - [] - >>> list(yield_lines(['foo', 'bar'])) - ['foo', 'bar'] - >>> list(yield_lines('foo\nbar')) - ['foo', 'bar'] - >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) - ['foo', 'baz #comment'] - >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) - ['foo', 'bar', 'baz', 'bing'] - """ - return itertools.chain.from_iterable(map(yield_lines, iterable)) - - -@yield_lines.register(str) -def _(text): - return filter(_nonblank, map(str.strip, text.splitlines())) - - -def drop_comment(line): - """ - Drop comments. - - >>> drop_comment('foo # bar') - 'foo' - - A hash without a space may be in a URL. - - >>> drop_comment('http://example.com/foo#bar') - 'http://example.com/foo#bar' - """ - return line.partition(' #')[0] - - -def join_continuation(lines): - r""" - Join lines continued by a trailing backslash. - - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar \\', 'baz'])) - ['foobarbaz'] - - Not sure why, but... - The character preceeding the backslash is also elided. - - >>> list(join_continuation(['goo\\', 'dly'])) - ['godly'] - - A terrible idea, but... - If no line is available to continue, suppress the lines. - - >>> list(join_continuation(['foo', 'bar\\', 'baz\\'])) - ['foo'] - """ - lines = iter(lines) - for item in lines: - while item.endswith('\\'): - try: - item = item[:-2].strip() + next(lines) - except StopIteration: - return - yield item diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py deleted file mode 100644 index 19a169fc301..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .more import * # noqa -from .recipes import * # noqa - -__version__ = '8.8.0' diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi deleted file mode 100644 index 96f6e36c7f4..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi +++ /dev/null @@ -1,2 +0,0 @@ -from .more import * -from .recipes import * diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py deleted file mode 100644 index e6fca4d47f6..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py +++ /dev/null @@ -1,3824 +0,0 @@ -import warnings - -from collections import Counter, defaultdict, deque, abc -from collections.abc import Sequence -from functools import partial, reduce, wraps -from heapq import merge, heapify, heapreplace, heappop -from itertools import ( - chain, - compress, - count, - cycle, - dropwhile, - groupby, - islice, - repeat, - starmap, - takewhile, - tee, - zip_longest, -) -from math import exp, factorial, floor, log -from queue import Empty, Queue -from random import random, randrange, uniform -from operator import itemgetter, mul, sub, gt, lt -from sys import hexversion, maxsize -from time import monotonic - -from .recipes import ( - consume, - flatten, - pairwise, - powerset, - take, - unique_everseen, -) - -__all__ = [ - 'AbortThread', - 'adjacent', - 'always_iterable', - 'always_reversible', - 'bucket', - 'callback_iter', - 'chunked', - 'circular_shifts', - 'collapse', - 'collate', - 'consecutive_groups', - 'consumer', - 'countable', - 'count_cycle', - 'mark_ends', - 'difference', - 'distinct_combinations', - 'distinct_permutations', - 'distribute', - 'divide', - 'exactly_n', - 'filter_except', - 'first', - 'groupby_transform', - 'ilen', - 'interleave_longest', - 'interleave', - 'intersperse', - 'islice_extended', - 'iterate', - 'ichunked', - 'is_sorted', - 'last', - 'locate', - 'lstrip', - 'make_decorator', - 'map_except', - 'map_reduce', - 'nth_or_last', - 'nth_permutation', - 'nth_product', - 'numeric_range', - 'one', - 'only', - 'padded', - 'partitions', - 'set_partitions', - 'peekable', - 'repeat_last', - 'replace', - 'rlocate', - 'rstrip', - 'run_length', - 'sample', - 'seekable', - 'SequenceView', - 'side_effect', - 'sliced', - 'sort_together', - 'split_at', - 'split_after', - 'split_before', - 'split_when', - 'split_into', - 'spy', - 'stagger', - 'strip', - 'substrings', - 'substrings_indexes', - 'time_limited', - 'unique_to_each', - 'unzip', - 'windowed', - 'with_iter', - 'UnequalIterablesError', - 'zip_equal', - 'zip_offset', - 'windowed_complete', - 'all_unique', - 'value_chain', - 'product_index', - 'combination_index', - 'permutation_index', -] - -_marker = object() - - -def chunked(iterable, n, strict=False): - """Break *iterable* into lists of length *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) - [[1, 2, 3], [4, 5, 6]] - - By the default, the last yielded list will have fewer than *n* elements - if the length of *iterable* is not divisible by *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) - [[1, 2, 3], [4, 5, 6], [7, 8]] - - To use a fill-in value instead, see the :func:`grouper` recipe. - - If the length of *iterable* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - list is yielded. - - """ - iterator = iter(partial(take, n, iter(iterable)), []) - if strict: - - def ret(): - for chunk in iterator: - if len(chunk) != n: - raise ValueError('iterable is not divisible by n.') - yield chunk - - return iter(ret()) - else: - return iterator - - -def first(iterable, default=_marker): - """Return the first item of *iterable*, or *default* if *iterable* is - empty. - - >>> first([0, 1, 2, 3]) - 0 - >>> first([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - - :func:`first` is useful when you have a generator of expensive-to-retrieve - values and want any arbitrary one. It is marginally shorter than - ``next(iter(iterable), default)``. - - """ - try: - return next(iter(iterable)) - except StopIteration as e: - if default is _marker: - raise ValueError( - 'first() was called on an empty iterable, and no ' - 'default value was provided.' - ) from e - return default - - -def last(iterable, default=_marker): - """Return the last item of *iterable*, or *default* if *iterable* is - empty. - - >>> last([0, 1, 2, 3]) - 3 - >>> last([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - try: - if isinstance(iterable, Sequence): - return iterable[-1] - # Work around https://bugs.python.org/issue38525 - elif hasattr(iterable, '__reversed__') and (hexversion != 0x030800F0): - return next(reversed(iterable)) - else: - return deque(iterable, maxlen=1)[-1] - except (IndexError, TypeError, StopIteration): - if default is _marker: - raise ValueError( - 'last() was called on an empty iterable, and no default was ' - 'provided.' - ) - return default - - -def nth_or_last(iterable, n, default=_marker): - """Return the nth or the last item of *iterable*, - or *default* if *iterable* is empty. - - >>> nth_or_last([0, 1, 2, 3], 2) - 2 - >>> nth_or_last([0, 1], 2) - 1 - >>> nth_or_last([], 0, 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - return last(islice(iterable, n + 1), default=default) - - -class peekable: - """Wrap an iterator to allow lookahead and prepending elements. - - Call :meth:`peek` on the result to get the value that will be returned - by :func:`next`. This won't advance the iterator: - - >>> p = peekable(['a', 'b']) - >>> p.peek() - 'a' - >>> next(p) - 'a' - - Pass :meth:`peek` a default value to return that instead of raising - ``StopIteration`` when the iterator is exhausted. - - >>> p = peekable([]) - >>> p.peek('hi') - 'hi' - - peekables also offer a :meth:`prepend` method, which "inserts" items - at the head of the iterable: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> p.peek() - 11 - >>> list(p) - [11, 12, 1, 2, 3] - - peekables can be indexed. Index 0 is the item that will be returned by - :func:`next`, index 1 is the item after that, and so on: - The values up to the given index will be cached. - - >>> p = peekable(['a', 'b', 'c', 'd']) - >>> p[0] - 'a' - >>> p[1] - 'b' - >>> next(p) - 'a' - - Negative indexes are supported, but be aware that they will cache the - remaining items in the source iterator, which may require significant - storage. - - To check whether a peekable is exhausted, check its truth value: - - >>> p = peekable(['a', 'b']) - >>> if p: # peekable has items - ... list(p) - ['a', 'b'] - >>> if not p: # peekable is exhausted - ... list(p) - [] - - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self._cache = deque() - - def __iter__(self): - return self - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - """Return the item that will be next returned from ``next()``. - - Return ``default`` if there are no items left. If ``default`` is not - provided, raise ``StopIteration``. - - """ - if not self._cache: - try: - self._cache.append(next(self._it)) - except StopIteration: - if default is _marker: - raise - return default - return self._cache[0] - - def prepend(self, *items): - """Stack up items to be the next ones returned from ``next()`` or - ``self.peek()``. The items will be returned in - first in, first out order:: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> list(p) - [11, 12, 1, 2, 3] - - It is possible, by prepending items, to "resurrect" a peekable that - previously raised ``StopIteration``. - - >>> p = peekable([]) - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - >>> p.prepend(1) - >>> next(p) - 1 - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - - """ - self._cache.extendleft(reversed(items)) - - def __next__(self): - if self._cache: - return self._cache.popleft() - - return next(self._it) - - def _get_slice(self, index): - # Normalize the slice's arguments - step = 1 if (index.step is None) else index.step - if step > 0: - start = 0 if (index.start is None) else index.start - stop = maxsize if (index.stop is None) else index.stop - elif step < 0: - start = -1 if (index.start is None) else index.start - stop = (-maxsize - 1) if (index.stop is None) else index.stop - else: - raise ValueError('slice step cannot be zero') - - # If either the start or stop index is negative, we'll need to cache - # the rest of the iterable in order to slice from the right side. - if (start < 0) or (stop < 0): - self._cache.extend(self._it) - # Otherwise we'll need to find the rightmost index and cache to that - # point. - else: - n = min(max(start, stop) + 1, maxsize) - cache_len = len(self._cache) - if n >= cache_len: - self._cache.extend(islice(self._it, n - cache_len)) - - return list(self._cache)[index] - - def __getitem__(self, index): - if isinstance(index, slice): - return self._get_slice(index) - - cache_len = len(self._cache) - if index < 0: - self._cache.extend(self._it) - elif index >= cache_len: - self._cache.extend(islice(self._it, index + 1 - cache_len)) - - return self._cache[index] - - -def collate(*iterables, **kwargs): - """Return a sorted merge of the items from each of several already-sorted - *iterables*. - - >>> list(collate('ACDZ', 'AZ', 'JKL')) - ['A', 'A', 'C', 'D', 'J', 'K', 'L', 'Z', 'Z'] - - Works lazily, keeping only the next value from each iterable in memory. Use - :func:`collate` to, for example, perform a n-way mergesort of items that - don't fit in memory. - - If a *key* function is specified, the iterables will be sorted according - to its result: - - >>> key = lambda s: int(s) # Sort by numeric value, not by string - >>> list(collate(['1', '10'], ['2', '11'], key=key)) - ['1', '2', '10', '11'] - - - If the *iterables* are sorted in descending order, set *reverse* to - ``True``: - - >>> list(collate([5, 3, 1], [4, 2, 0], reverse=True)) - [5, 4, 3, 2, 1, 0] - - If the elements of the passed-in iterables are out of order, you might get - unexpected results. - - On Python 3.5+, this function is an alias for :func:`heapq.merge`. - - """ - warnings.warn( - "collate is no longer part of more_itertools, use heapq.merge", - DeprecationWarning, - ) - return merge(*iterables, **kwargs) - - -def consumer(func): - """Decorator that automatically advances a PEP-342-style "reverse iterator" - to its first yield point so you don't have to call ``next()`` on it - manually. - - >>> @consumer - ... def tally(): - ... i = 0 - ... while True: - ... print('Thing number %s is %s.' % (i, (yield))) - ... i += 1 - ... - >>> t = tally() - >>> t.send('red') - Thing number 0 is red. - >>> t.send('fish') - Thing number 1 is fish. - - Without the decorator, you would have to call ``next(t)`` before - ``t.send()`` could be used. - - """ - - @wraps(func) - def wrapper(*args, **kwargs): - gen = func(*args, **kwargs) - next(gen) - return gen - - return wrapper - - -def ilen(iterable): - """Return the number of items in *iterable*. - - >>> ilen(x for x in range(1000000) if x % 3 == 0) - 333334 - - This consumes the iterable, so handle with care. - - """ - # This approach was selected because benchmarks showed it's likely the - # fastest of the known implementations at the time of writing. - # See GitHub tracker: #236, #230. - counter = count() - deque(zip(iterable, counter), maxlen=0) - return next(counter) - - -def iterate(func, start): - """Return ``start``, ``func(start)``, ``func(func(start))``, ... - - >>> from itertools import islice - >>> list(islice(iterate(lambda x: 2*x, 1), 10)) - [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - - """ - while True: - yield start - start = func(start) - - -def with_iter(context_manager): - """Wrap an iterable in a ``with`` statement, so it closes once exhausted. - - For example, this will close the file when the iterator is exhausted:: - - upper_lines = (line.upper() for line in with_iter(open('foo'))) - - Any context manager which returns an iterable is a candidate for - ``with_iter``. - - """ - with context_manager as iterable: - yield from iterable - - -def one(iterable, too_short=None, too_long=None): - """Return the first item from *iterable*, which is expected to contain only - that item. Raise an exception if *iterable* is empty or has more than one - item. - - :func:`one` is useful for ensuring that an iterable contains only one item. - For example, it can be used to retrieve the result of a database query - that is expected to return a single row. - - If *iterable* is empty, ``ValueError`` will be raised. You may specify a - different exception with the *too_short* keyword: - - >>> it = [] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (expected 1)' - >>> too_short = IndexError('too few items') - >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - IndexError: too few items - - Similarly, if *iterable* contains more than one item, ``ValueError`` will - be raised. You may specify a different exception with the *too_long* - keyword: - - >>> it = ['too', 'many'] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 'too', - 'many', and perhaps more. - >>> too_long = RuntimeError - >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - Note that :func:`one` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check iterable - contents less destructively. - - """ - it = iter(iterable) - - try: - first_value = next(it) - except StopIteration as e: - raise ( - too_short or ValueError('too few items in iterable (expected 1)') - ) from e - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def distinct_permutations(iterable, r=None): - """Yield successive distinct permutations of the elements in *iterable*. - - >>> sorted(distinct_permutations([1, 0, 1])) - [(0, 1, 1), (1, 0, 1), (1, 1, 0)] - - Equivalent to ``set(permutations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - Duplicate permutations arise when there are duplicated elements in the - input iterable. The number of items returned is - `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of - items input, and each `x_i` is the count of a distinct item in the input - sequence. - - If *r* is given, only the *r*-length permutations are yielded. - - >>> sorted(distinct_permutations([1, 0, 1], r=2)) - [(0, 1), (1, 0), (1, 1)] - >>> sorted(distinct_permutations(range(3), r=2)) - [(0, 1), (0, 2), (1, 0), (1, 2), (2, 0), (2, 1)] - - """ - # Algorithm: https://w.wiki/Qai - def _full(A): - while True: - # Yield the permutation we have - yield tuple(A) - - # Find the largest index i such that A[i] < A[i + 1] - for i in range(size - 2, -1, -1): - if A[i] < A[i + 1]: - break - # If no such index exists, this permutation is the last one - else: - return - - # Find the largest index j greater than j such that A[i] < A[j] - for j in range(size - 1, i, -1): - if A[i] < A[j]: - break - - # Swap the value of A[i] with that of A[j], then reverse the - # sequence from A[i + 1] to form the new permutation - A[i], A[j] = A[j], A[i] - A[i + 1 :] = A[: i - size : -1] # A[i + 1:][::-1] - - # Algorithm: modified from the above - def _partial(A, r): - # Split A into the first r items and the last r items - head, tail = A[:r], A[r:] - right_head_indexes = range(r - 1, -1, -1) - left_tail_indexes = range(len(tail)) - - while True: - # Yield the permutation we have - yield tuple(head) - - # Starting from the right, find the first index of the head with - # value smaller than the maximum value of the tail - call it i. - pivot = tail[-1] - for i in right_head_indexes: - if head[i] < pivot: - break - pivot = head[i] - else: - return - - # Starting from the left, find the first value of the tail - # with a value greater than head[i] and swap. - for j in left_tail_indexes: - if tail[j] > head[i]: - head[i], tail[j] = tail[j], head[i] - break - # If we didn't find one, start from the right and find the first - # index of the head with a value greater than head[i] and swap. - else: - for j in right_head_indexes: - if head[j] > head[i]: - head[i], head[j] = head[j], head[i] - break - - # Reverse head[i + 1:] and swap it with tail[:r - (i + 1)] - tail += head[: i - r : -1] # head[i + 1:][::-1] - i += 1 - head[i:], tail[:] = tail[: r - i], tail[r - i :] - - items = sorted(iterable) - - size = len(items) - if r is None: - r = size - - if 0 < r <= size: - return _full(items) if (r == size) else _partial(items, r) - - return iter(() if r else ((),)) - - -def intersperse(e, iterable, n=1): - """Intersperse filler element *e* among the items in *iterable*, leaving - *n* items between each filler element. - - >>> list(intersperse('!', [1, 2, 3, 4, 5])) - [1, '!', 2, '!', 3, '!', 4, '!', 5] - - >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) - [1, 2, None, 3, 4, None, 5] - - """ - if n == 0: - raise ValueError('n must be > 0') - elif n == 1: - # interleave(repeat(e), iterable) -> e, x_0, e, e, x_1, e, x_2... - # islice(..., 1, None) -> x_0, e, e, x_1, e, x_2... - return islice(interleave(repeat(e), iterable), 1, None) - else: - # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... - # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... - # flatten(...) -> x_0, x_1, e, x_2, x_3... - filler = repeat([e]) - chunks = chunked(iterable, n) - return flatten(islice(interleave(filler, chunks), 1, None)) - - -def unique_to_each(*iterables): - """Return the elements from each of the input iterables that aren't in the - other input iterables. - - For example, suppose you have a set of packages, each with a set of - dependencies:: - - {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} - - If you remove one package, which dependencies can also be removed? - - If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not - associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for - ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: - - >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) - [['A'], ['C'], ['D']] - - If there are duplicates in one input iterable that aren't in the others - they will be duplicated in the output. Input order is preserved:: - - >>> unique_to_each("mississippi", "missouri") - [['p', 'p'], ['o', 'u', 'r']] - - It is assumed that the elements of each iterable are hashable. - - """ - pool = [list(it) for it in iterables] - counts = Counter(chain.from_iterable(map(set, pool))) - uniques = {element for element in counts if counts[element] == 1} - return [list(filter(uniques.__contains__, it)) for it in pool] - - -def windowed(seq, n, fillvalue=None, step=1): - """Return a sliding window of width *n* over the given iterable. - - >>> all_windows = windowed([1, 2, 3, 4, 5], 3) - >>> list(all_windows) - [(1, 2, 3), (2, 3, 4), (3, 4, 5)] - - When the window is larger than the iterable, *fillvalue* is used in place - of missing values: - - >>> list(windowed([1, 2, 3], 4)) - [(1, 2, 3, None)] - - Each window will advance in increments of *step*: - - >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) - [(1, 2, 3), (3, 4, 5), (5, 6, '!')] - - To slide into the iterable's items, use :func:`chain` to add filler items - to the left: - - >>> iterable = [1, 2, 3, 4] - >>> n = 3 - >>> padding = [None] * (n - 1) - >>> list(windowed(chain(padding, iterable), 3)) - [(None, None, 1), (None, 1, 2), (1, 2, 3), (2, 3, 4)] - """ - if n < 0: - raise ValueError('n must be >= 0') - if n == 0: - yield tuple() - return - if step < 1: - raise ValueError('step must be >= 1') - - window = deque(maxlen=n) - i = n - for _ in map(window.append, seq): - i -= 1 - if not i: - i = step - yield tuple(window) - - size = len(window) - if size < n: - yield tuple(chain(window, repeat(fillvalue, n - size))) - elif 0 < i < min(step, n): - window += (fillvalue,) * i - yield tuple(window) - - -def substrings(iterable): - """Yield all of the substrings of *iterable*. - - >>> [''.join(s) for s in substrings('more')] - ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] - - Note that non-string iterables can also be subdivided. - - >>> list(substrings([0, 1, 2])) - [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] - - """ - # The length-1 substrings - seq = [] - for item in iter(iterable): - seq.append(item) - yield (item,) - seq = tuple(seq) - item_count = len(seq) - - # And the rest - for n in range(2, item_count + 1): - for i in range(item_count - n + 1): - yield seq[i : i + n] - - -def substrings_indexes(seq, reverse=False): - """Yield all substrings and their positions in *seq* - - The items yielded will be a tuple of the form ``(substr, i, j)``, where - ``substr == seq[i:j]``. - - This function only works for iterables that support slicing, such as - ``str`` objects. - - >>> for item in substrings_indexes('more'): - ... print(item) - ('m', 0, 1) - ('o', 1, 2) - ('r', 2, 3) - ('e', 3, 4) - ('mo', 0, 2) - ('or', 1, 3) - ('re', 2, 4) - ('mor', 0, 3) - ('ore', 1, 4) - ('more', 0, 4) - - Set *reverse* to ``True`` to yield the same items in the opposite order. - - - """ - r = range(1, len(seq) + 1) - if reverse: - r = reversed(r) - return ( - (seq[i : i + L], i, i + L) for L in r for i in range(len(seq) - L + 1) - ) - - -class bucket: - """Wrap *iterable* and return an object that buckets it iterable into - child iterables based on a *key* function. - - >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] - >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character - >>> sorted(list(s)) # Get the keys - ['a', 'b', 'c'] - >>> a_iterable = s['a'] - >>> next(a_iterable) - 'a1' - >>> next(a_iterable) - 'a2' - >>> list(s['b']) - ['b1', 'b2', 'b3'] - - The original iterable will be advanced and its items will be cached until - they are used by the child iterables. This may require significant storage. - - By default, attempting to select a bucket to which no items belong will - exhaust the iterable and cache all values. - If you specify a *validator* function, selected buckets will instead be - checked against it. - - >>> from itertools import count - >>> it = count(1, 2) # Infinite sequence of odd numbers - >>> key = lambda x: x % 10 # Bucket by last digit - >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only - >>> s = bucket(it, key=key, validator=validator) - >>> 2 in s - False - >>> list(s[2]) - [] - - """ - - def __init__(self, iterable, key, validator=None): - self._it = iter(iterable) - self._key = key - self._cache = defaultdict(deque) - self._validator = validator or (lambda x: True) - - def __contains__(self, value): - if not self._validator(value): - return False - - try: - item = next(self[value]) - except StopIteration: - return False - else: - self._cache[value].appendleft(item) - - return True - - def _get_values(self, value): - """ - Helper to yield items from the parent iterator that match *value*. - Items that don't match are stored in the local cache as they - are encountered. - """ - while True: - # If we've cached some items that match the target value, emit - # the first one and evict it from the cache. - if self._cache[value]: - yield self._cache[value].popleft() - # Otherwise we need to advance the parent iterator to search for - # a matching item, caching the rest. - else: - while True: - try: - item = next(self._it) - except StopIteration: - return - item_value = self._key(item) - if item_value == value: - yield item - break - elif self._validator(item_value): - self._cache[item_value].append(item) - - def __iter__(self): - for item in self._it: - item_value = self._key(item) - if self._validator(item_value): - self._cache[item_value].append(item) - - yield from self._cache.keys() - - def __getitem__(self, value): - if not self._validator(value): - return iter(()) - - return self._get_values(value) - - -def spy(iterable, n=1): - """Return a 2-tuple with a list containing the first *n* elements of - *iterable*, and an iterator with the same items as *iterable*. - This allows you to "look ahead" at the items in the iterable without - advancing it. - - There is one item in the list by default: - - >>> iterable = 'abcdefg' - >>> head, iterable = spy(iterable) - >>> head - ['a'] - >>> list(iterable) - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - - You may use unpacking to retrieve items instead of lists: - - >>> (head,), iterable = spy('abcdefg') - >>> head - 'a' - >>> (first, second), iterable = spy('abcdefg', 2) - >>> first - 'a' - >>> second - 'b' - - The number of items requested can be larger than the number of items in - the iterable: - - >>> iterable = [1, 2, 3, 4, 5] - >>> head, iterable = spy(iterable, 10) - >>> head - [1, 2, 3, 4, 5] - >>> list(iterable) - [1, 2, 3, 4, 5] - - """ - it = iter(iterable) - head = take(n, it) - - return head.copy(), chain(head, it) - - -def interleave(*iterables): - """Return a new iterable yielding from each iterable in turn, - until the shortest is exhausted. - - >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7] - - For a version that doesn't terminate after the shortest iterable is - exhausted, see :func:`interleave_longest`. - - """ - return chain.from_iterable(zip(*iterables)) - - -def interleave_longest(*iterables): - """Return a new iterable yielding from each iterable in turn, - skipping any that are exhausted. - - >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7, 3, 8] - - This function produces the same output as :func:`roundrobin`, but may - perform better for some inputs (in particular when the number of iterables - is large). - - """ - i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) - return (x for x in i if x is not _marker) - - -def collapse(iterable, base_type=None, levels=None): - """Flatten an iterable with multiple levels of nesting (e.g., a list of - lists of tuples) into non-iterable types. - - >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] - >>> list(collapse(iterable)) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and - will not be collapsed. - - To avoid collapsing other types, specify *base_type*: - - >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] - >>> list(collapse(iterable, base_type=tuple)) - ['ab', ('cd', 'ef'), 'gh', 'ij'] - - Specify *levels* to stop flattening after a certain level: - - >>> iterable = [('a', ['b']), ('c', ['d'])] - >>> list(collapse(iterable)) # Fully flattened - ['a', 'b', 'c', 'd'] - >>> list(collapse(iterable, levels=1)) # Only one level flattened - ['a', ['b'], 'c', ['d']] - - """ - - def walk(node, level): - if ( - ((levels is not None) and (level > levels)) - or isinstance(node, (str, bytes)) - or ((base_type is not None) and isinstance(node, base_type)) - ): - yield node - return - - try: - tree = iter(node) - except TypeError: - yield node - return - else: - for child in tree: - yield from walk(child, level + 1) - - yield from walk(iterable, 0) - - -def side_effect(func, iterable, chunk_size=None, before=None, after=None): - """Invoke *func* on each item in *iterable* (or on each *chunk_size* group - of items) before yielding the item. - - `func` must be a function that takes a single argument. Its return value - will be discarded. - - *before* and *after* are optional functions that take no arguments. They - will be executed before iteration starts and after it ends, respectively. - - `side_effect` can be used for logging, updating progress bars, or anything - that is not functionally "pure." - - Emitting a status message: - - >>> from more_itertools import consume - >>> func = lambda item: print('Received {}'.format(item)) - >>> consume(side_effect(func, range(2))) - Received 0 - Received 1 - - Operating on chunks of items: - - >>> pair_sums = [] - >>> func = lambda chunk: pair_sums.append(sum(chunk)) - >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) - [0, 1, 2, 3, 4, 5] - >>> list(pair_sums) - [1, 5, 9] - - Writing to a file-like object: - - >>> from io import StringIO - >>> from more_itertools import consume - >>> f = StringIO() - >>> func = lambda x: print(x, file=f) - >>> before = lambda: print(u'HEADER', file=f) - >>> after = f.close - >>> it = [u'a', u'b', u'c'] - >>> consume(side_effect(func, it, before=before, after=after)) - >>> f.closed - True - - """ - try: - if before is not None: - before() - - if chunk_size is None: - for item in iterable: - func(item) - yield item - else: - for chunk in chunked(iterable, chunk_size): - func(chunk) - yield from chunk - finally: - if after is not None: - after() - - -def sliced(seq, n, strict=False): - """Yield slices of length *n* from the sequence *seq*. - - >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) - [(1, 2, 3), (4, 5, 6)] - - By the default, the last yielded slice will have fewer than *n* elements - if the length of *seq* is not divisible by *n*: - - >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) - [(1, 2, 3), (4, 5, 6), (7, 8)] - - If the length of *seq* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - slice is yielded. - - This function will only work for iterables that support slicing. - For non-sliceable iterables, see :func:`chunked`. - - """ - iterator = takewhile(len, (seq[i : i + n] for i in count(0, n))) - if strict: - - def ret(): - for _slice in iterator: - if len(_slice) != n: - raise ValueError("seq is not divisible by n.") - yield _slice - - return iter(ret()) - else: - return iterator - - -def split_at(iterable, pred, maxsplit=-1, keep_separator=False): - """Yield lists of items from *iterable*, where each list is delimited by - an item where callable *pred* returns ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b')) - [['a'], ['c', 'd', 'c'], ['a']] - - >>> list(split_at(range(10), lambda n: n % 2 == 1)) - [[0], [2], [4], [6], [8], []] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_at(range(10), lambda n: n % 2 == 1, maxsplit=2)) - [[0], [2], [4, 5, 6, 7, 8, 9]] - - By default, the delimiting items are not included in the output. - The include them, set *keep_separator* to ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b', keep_separator=True)) - [['a'], ['b'], ['c', 'd', 'c'], ['b'], ['a']] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item): - yield buf - if keep_separator: - yield [item] - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - else: - buf.append(item) - yield buf - - -def split_before(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends just before - an item for which callable *pred* returns ``True``: - - >>> list(split_before('OneTwo', lambda s: s.isupper())) - [['O', 'n', 'e'], ['T', 'w', 'o']] - - >>> list(split_before(range(10), lambda n: n % 3 == 0)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_before(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]] - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield [item] + list(it) - return - buf = [] - maxsplit -= 1 - buf.append(item) - if buf: - yield buf - - -def split_after(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends with an - item where callable *pred* returns ``True``: - - >>> list(split_after('one1two2', lambda s: s.isdigit())) - [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] - - >>> list(split_after(range(10), lambda n: n % 3 == 0)) - [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_after(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0], [1, 2, 3], [4, 5, 6, 7, 8, 9]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - buf.append(item) - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - if buf: - yield buf - - -def split_when(iterable, pred, maxsplit=-1): - """Split *iterable* into pieces based on the output of *pred*. - *pred* should be a function that takes successive pairs of items and - returns ``True`` if the iterable should be split in between them. - - For example, to find runs of increasing numbers, split the iterable when - element ``i`` is larger than element ``i + 1``: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], lambda x, y: x > y)) - [[1, 2, 3, 3], [2, 5], [2, 4], [2]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], - ... lambda x, y: x > y, maxsplit=2)) - [[1, 2, 3, 3], [2, 5], [2, 4, 2]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - it = iter(iterable) - try: - cur_item = next(it) - except StopIteration: - return - - buf = [cur_item] - for next_item in it: - if pred(cur_item, next_item): - yield buf - if maxsplit == 1: - yield [next_item] + list(it) - return - buf = [] - maxsplit -= 1 - - buf.append(next_item) - cur_item = next_item - - yield buf - - -def split_into(iterable, sizes): - """Yield a list of sequential items from *iterable* of length 'n' for each - integer 'n' in *sizes*. - - >>> list(split_into([1,2,3,4,5,6], [1,2,3])) - [[1], [2, 3], [4, 5, 6]] - - If the sum of *sizes* is smaller than the length of *iterable*, then the - remaining items of *iterable* will not be returned. - - >>> list(split_into([1,2,3,4,5,6], [2,3])) - [[1, 2], [3, 4, 5]] - - If the sum of *sizes* is larger than the length of *iterable*, fewer items - will be returned in the iteration that overruns *iterable* and further - lists will be empty: - - >>> list(split_into([1,2,3,4], [1,2,3,4])) - [[1], [2, 3], [4], []] - - When a ``None`` object is encountered in *sizes*, the returned list will - contain items up to the end of *iterable* the same way that itertools.slice - does: - - >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) - [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] - - :func:`split_into` can be useful for grouping a series of items where the - sizes of the groups are not uniform. An example would be where in a row - from a table, multiple columns represent elements of the same feature - (e.g. a point represented by x,y,z) but, the format is not the same for - all columns. - """ - # convert the iterable argument into an iterator so its contents can - # be consumed by islice in case it is a generator - it = iter(iterable) - - for size in sizes: - if size is None: - yield list(it) - return - else: - yield list(islice(it, size)) - - -def padded(iterable, fillvalue=None, n=None, next_multiple=False): - """Yield the elements from *iterable*, followed by *fillvalue*, such that - at least *n* items are emitted. - - >>> list(padded([1, 2, 3], '?', 5)) - [1, 2, 3, '?', '?'] - - If *next_multiple* is ``True``, *fillvalue* will be emitted until the - number of items emitted is a multiple of *n*:: - - >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) - [1, 2, 3, 4, None, None] - - If *n* is ``None``, *fillvalue* will be emitted indefinitely. - - """ - it = iter(iterable) - if n is None: - yield from chain(it, repeat(fillvalue)) - elif n < 1: - raise ValueError('n must be at least 1') - else: - item_count = 0 - for item in it: - yield item - item_count += 1 - - remaining = (n - item_count) % n if next_multiple else n - item_count - for _ in range(remaining): - yield fillvalue - - -def repeat_last(iterable, default=None): - """After the *iterable* is exhausted, keep yielding its last element. - - >>> list(islice(repeat_last(range(3)), 5)) - [0, 1, 2, 2, 2] - - If the iterable is empty, yield *default* forever:: - - >>> list(islice(repeat_last(range(0), 42), 5)) - [42, 42, 42, 42, 42] - - """ - item = _marker - for item in iterable: - yield item - final = default if item is _marker else item - yield from repeat(final) - - -def distribute(n, iterable): - """Distribute the items from *iterable* among *n* smaller iterables. - - >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 3, 5] - >>> list(group_2) - [2, 4, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 4, 7], [2, 5], [3, 6]] - - If the length of *iterable* is smaller than *n*, then the last returned - iterables will be empty: - - >>> children = distribute(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function uses :func:`itertools.tee` and may require significant - storage. If you need the order items in the smaller iterables to match the - original iterable, see :func:`divide`. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - children = tee(iterable, n) - return [islice(it, index, None, n) for index, it in enumerate(children)] - - -def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): - """Yield tuples whose elements are offset from *iterable*. - The amount by which the `i`-th item in each tuple is offset is given by - the `i`-th item in *offsets*. - - >>> list(stagger([0, 1, 2, 3])) - [(None, 0, 1), (0, 1, 2), (1, 2, 3)] - >>> list(stagger(range(8), offsets=(0, 2, 4))) - [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] - - By default, the sequence will end when the final element of a tuple is the - last item in the iterable. To continue until the first element of a tuple - is the last item in the iterable, set *longest* to ``True``:: - - >>> list(stagger([0, 1, 2, 3], longest=True)) - [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - children = tee(iterable, len(offsets)) - - return zip_offset( - *children, offsets=offsets, longest=longest, fillvalue=fillvalue - ) - - -class UnequalIterablesError(ValueError): - def __init__(self, details=None): - msg = 'Iterables have different lengths' - if details is not None: - msg += (': index 0 has length {}; index {} has length {}').format( - *details - ) - - super().__init__(msg) - - -def _zip_equal_generator(iterables): - for combo in zip_longest(*iterables, fillvalue=_marker): - for val in combo: - if val is _marker: - raise UnequalIterablesError() - yield combo - - -def zip_equal(*iterables): - """``zip`` the input *iterables* together, but raise - ``UnequalIterablesError`` if they aren't all the same length. - - >>> it_1 = range(3) - >>> it_2 = iter('abc') - >>> list(zip_equal(it_1, it_2)) - [(0, 'a'), (1, 'b'), (2, 'c')] - - >>> it_1 = range(3) - >>> it_2 = iter('abcd') - >>> list(zip_equal(it_1, it_2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - more_itertools.more.UnequalIterablesError: Iterables have different - lengths - - """ - if hexversion >= 0x30A00A6: - warnings.warn( - ( - 'zip_equal will be removed in a future version of ' - 'more-itertools. Use the builtin zip function with ' - 'strict=True instead.' - ), - DeprecationWarning, - ) - # Check whether the iterables are all the same size. - try: - first_size = len(iterables[0]) - for i, it in enumerate(iterables[1:], 1): - size = len(it) - if size != first_size: - break - else: - # If we didn't break out, we can use the built-in zip. - return zip(*iterables) - - # If we did break out, there was a mismatch. - raise UnequalIterablesError(details=(first_size, i, size)) - # If any one of the iterables didn't have a length, start reading - # them until one runs out. - except TypeError: - return _zip_equal_generator(iterables) - - -def zip_offset(*iterables, offsets, longest=False, fillvalue=None): - """``zip`` the input *iterables* together, but offset the `i`-th iterable - by the `i`-th item in *offsets*. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] - - This can be used as a lightweight alternative to SciPy or pandas to analyze - data sets in which some series have a lead or lag relationship. - - By default, the sequence will end when the shortest iterable is exhausted. - To continue until the longest iterable is exhausted, set *longest* to - ``True``. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - if len(iterables) != len(offsets): - raise ValueError("Number of iterables and offsets didn't match") - - staggered = [] - for it, n in zip(iterables, offsets): - if n < 0: - staggered.append(chain(repeat(fillvalue, -n), it)) - elif n > 0: - staggered.append(islice(it, n, None)) - else: - staggered.append(it) - - if longest: - return zip_longest(*staggered, fillvalue=fillvalue) - - return zip(*staggered) - - -def sort_together(iterables, key_list=(0,), key=None, reverse=False): - """Return the input iterables sorted together, with *key_list* as the - priority for sorting. All iterables are trimmed to the length of the - shortest one. - - This can be used like the sorting function in a spreadsheet. If each - iterable represents a column of data, the key list determines which - columns are used for sorting. - - By default, all iterables are sorted using the ``0``-th iterable:: - - >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] - >>> sort_together(iterables) - [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] - - Set a different key list to sort according to another iterable. - Specifying multiple keys dictates how ties are broken:: - - >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] - >>> sort_together(iterables, key_list=(1, 2)) - [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] - - To sort by a function of the elements of the iterable, pass a *key* - function. Its arguments are the elements of the iterables corresponding to - the key list:: - - >>> names = ('a', 'b', 'c') - >>> lengths = (1, 2, 3) - >>> widths = (5, 2, 1) - >>> def area(length, width): - ... return length * width - >>> sort_together([names, lengths, widths], key_list=(1, 2), key=area) - [('c', 'b', 'a'), (3, 2, 1), (1, 2, 5)] - - Set *reverse* to ``True`` to sort in descending order. - - >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) - [(3, 2, 1), ('a', 'b', 'c')] - - """ - if key is None: - # if there is no key function, the key argument to sorted is an - # itemgetter - key_argument = itemgetter(*key_list) - else: - # if there is a key function, call it with the items at the offsets - # specified by the key function as arguments - key_list = list(key_list) - if len(key_list) == 1: - # if key_list contains a single item, pass the item at that offset - # as the only argument to the key function - key_offset = key_list[0] - key_argument = lambda zipped_items: key(zipped_items[key_offset]) - else: - # if key_list contains multiple items, use itemgetter to return a - # tuple of items, which we pass as *args to the key function - get_key_items = itemgetter(*key_list) - key_argument = lambda zipped_items: key( - *get_key_items(zipped_items) - ) - - return list( - zip(*sorted(zip(*iterables), key=key_argument, reverse=reverse)) - ) - - -def unzip(iterable): - """The inverse of :func:`zip`, this function disaggregates the elements - of the zipped *iterable*. - - The ``i``-th iterable contains the ``i``-th element from each element - of the zipped iterable. The first element is used to to determine the - length of the remaining elements. - - >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> letters, numbers = unzip(iterable) - >>> list(letters) - ['a', 'b', 'c', 'd'] - >>> list(numbers) - [1, 2, 3, 4] - - This is similar to using ``zip(*iterable)``, but it avoids reading - *iterable* into memory. Note, however, that this function uses - :func:`itertools.tee` and thus may require significant storage. - - """ - head, iterable = spy(iter(iterable)) - if not head: - # empty iterable, e.g. zip([], [], []) - return () - # spy returns a one-length iterable as head - head = head[0] - iterables = tee(iterable, len(head)) - - def itemgetter(i): - def getter(obj): - try: - return obj[i] - except IndexError: - # basically if we have an iterable like - # iter([(1, 2, 3), (4, 5), (6,)]) - # the second unzipped iterable would fail at the third tuple - # since it would try to access tup[1] - # same with the third unzipped iterable and the second tuple - # to support these "improperly zipped" iterables, - # we create a custom itemgetter - # which just stops the unzipped iterables - # at first length mismatch - raise StopIteration - - return getter - - return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) - - -def divide(n, iterable): - """Divide the elements from *iterable* into *n* parts, maintaining - order. - - >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 2, 3] - >>> list(group_2) - [4, 5, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 2, 3], [4, 5], [6, 7]] - - If the length of the iterable is smaller than n, then the last returned - iterables will be empty: - - >>> children = divide(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function will exhaust the iterable before returning and may require - significant storage. If order is not important, see :func:`distribute`, - which does not first pull the iterable into memory. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - try: - iterable[:0] - except TypeError: - seq = tuple(iterable) - else: - seq = iterable - - q, r = divmod(len(seq), n) - - ret = [] - stop = 0 - for i in range(1, n + 1): - start = stop - stop += q + 1 if i <= r else q - ret.append(iter(seq[start:stop])) - - return ret - - -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) - - -def adjacent(predicate, iterable, distance=1): - """Return an iterable over `(bool, item)` tuples where the `item` is - drawn from *iterable* and the `bool` indicates whether - that item satisfies the *predicate* or is adjacent to an item that does. - - For example, to find whether items are adjacent to a ``3``:: - - >>> list(adjacent(lambda x: x == 3, range(6))) - [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] - - Set *distance* to change what counts as adjacent. For example, to find - whether items are two places away from a ``3``: - - >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) - [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] - - This is useful for contextualizing the results of a search function. - For example, a code comparison tool might want to identify lines that - have changed, but also surrounding lines to give the viewer of the diff - context. - - The predicate function will only be called once for each item in the - iterable. - - See also :func:`groupby_transform`, which can be used with this function - to group ranges of items with the same `bool` value. - - """ - # Allow distance=0 mainly for testing that it reproduces results with map() - if distance < 0: - raise ValueError('distance must be at least 0') - - i1, i2 = tee(iterable) - padding = [False] * distance - selected = chain(padding, map(predicate, i1), padding) - adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) - return zip(adjacent_to_selected, i2) - - -def groupby_transform(iterable, keyfunc=None, valuefunc=None, reducefunc=None): - """An extension of :func:`itertools.groupby` that can apply transformations - to the grouped data. - - * *keyfunc* is a function computing a key value for each item in *iterable* - * *valuefunc* is a function that transforms the individual items from - *iterable* after grouping - * *reducefunc* is a function that transforms each group of items - - >>> iterable = 'aAAbBBcCC' - >>> keyfunc = lambda k: k.upper() - >>> valuefunc = lambda v: v.lower() - >>> reducefunc = lambda g: ''.join(g) - >>> list(groupby_transform(iterable, keyfunc, valuefunc, reducefunc)) - [('A', 'aaa'), ('B', 'bbb'), ('C', 'ccc')] - - Each optional argument defaults to an identity function if not specified. - - :func:`groupby_transform` is useful when grouping elements of an iterable - using a separate iterable as the key. To do this, :func:`zip` the iterables - and pass a *keyfunc* that extracts the first element and a *valuefunc* - that extracts the second element:: - - >>> from operator import itemgetter - >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] - >>> values = 'abcdefghi' - >>> iterable = zip(keys, values) - >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) - >>> [(k, ''.join(g)) for k, g in grouper] - [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] - - Note that the order of items in the iterable is significant. - Only adjacent items are grouped together, so if you don't want any - duplicate groups, you should sort the iterable by the key function. - - """ - ret = groupby(iterable, keyfunc) - if valuefunc: - ret = ((k, map(valuefunc, g)) for k, g in ret) - if reducefunc: - ret = ((k, reducefunc(g)) for k, g in ret) - - return ret - - -class numeric_range(abc.Sequence, abc.Hashable): - """An extension of the built-in ``range()`` function whose arguments can - be any orderable numeric type. - - With only *stop* specified, *start* defaults to ``0`` and *step* - defaults to ``1``. The output items will match the type of *stop*: - - >>> list(numeric_range(3.5)) - [0.0, 1.0, 2.0, 3.0] - - With only *start* and *stop* specified, *step* defaults to ``1``. The - output items will match the type of *start*: - - >>> from decimal import Decimal - >>> start = Decimal('2.1') - >>> stop = Decimal('5.1') - >>> list(numeric_range(start, stop)) - [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] - - With *start*, *stop*, and *step* specified the output items will match - the type of ``start + step``: - - >>> from fractions import Fraction - >>> start = Fraction(1, 2) # Start at 1/2 - >>> stop = Fraction(5, 2) # End at 5/2 - >>> step = Fraction(1, 2) # Count by 1/2 - >>> list(numeric_range(start, stop, step)) - [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] - - If *step* is zero, ``ValueError`` is raised. Negative steps are supported: - - >>> list(numeric_range(3, -1, -1.0)) - [3.0, 2.0, 1.0, 0.0] - - Be aware of the limitations of floating point numbers; the representation - of the yielded numbers may be surprising. - - ``datetime.datetime`` objects can be used for *start* and *stop*, if *step* - is a ``datetime.timedelta`` object: - - >>> import datetime - >>> start = datetime.datetime(2019, 1, 1) - >>> stop = datetime.datetime(2019, 1, 3) - >>> step = datetime.timedelta(days=1) - >>> items = iter(numeric_range(start, stop, step)) - >>> next(items) - datetime.datetime(2019, 1, 1, 0, 0) - >>> next(items) - datetime.datetime(2019, 1, 2, 0, 0) - - """ - - _EMPTY_HASH = hash(range(0, 0)) - - def __init__(self, *args): - argc = len(args) - if argc == 1: - (self._stop,) = args - self._start = type(self._stop)(0) - self._step = type(self._stop - self._start)(1) - elif argc == 2: - self._start, self._stop = args - self._step = type(self._stop - self._start)(1) - elif argc == 3: - self._start, self._stop, self._step = args - elif argc == 0: - raise TypeError( - 'numeric_range expected at least ' - '1 argument, got {}'.format(argc) - ) - else: - raise TypeError( - 'numeric_range expected at most ' - '3 arguments, got {}'.format(argc) - ) - - self._zero = type(self._step)(0) - if self._step == self._zero: - raise ValueError('numeric_range() arg 3 must not be zero') - self._growing = self._step > self._zero - self._init_len() - - def __bool__(self): - if self._growing: - return self._start < self._stop - else: - return self._start > self._stop - - def __contains__(self, elem): - if self._growing: - if self._start <= elem < self._stop: - return (elem - self._start) % self._step == self._zero - else: - if self._start >= elem > self._stop: - return (self._start - elem) % (-self._step) == self._zero - - return False - - def __eq__(self, other): - if isinstance(other, numeric_range): - empty_self = not bool(self) - empty_other = not bool(other) - if empty_self or empty_other: - return empty_self and empty_other # True if both empty - else: - return ( - self._start == other._start - and self._step == other._step - and self._get_by_index(-1) == other._get_by_index(-1) - ) - else: - return False - - def __getitem__(self, key): - if isinstance(key, int): - return self._get_by_index(key) - elif isinstance(key, slice): - step = self._step if key.step is None else key.step * self._step - - if key.start is None or key.start <= -self._len: - start = self._start - elif key.start >= self._len: - start = self._stop - else: # -self._len < key.start < self._len - start = self._get_by_index(key.start) - - if key.stop is None or key.stop >= self._len: - stop = self._stop - elif key.stop <= -self._len: - stop = self._start - else: # -self._len < key.stop < self._len - stop = self._get_by_index(key.stop) - - return numeric_range(start, stop, step) - else: - raise TypeError( - 'numeric range indices must be ' - 'integers or slices, not {}'.format(type(key).__name__) - ) - - def __hash__(self): - if self: - return hash((self._start, self._get_by_index(-1), self._step)) - else: - return self._EMPTY_HASH - - def __iter__(self): - values = (self._start + (n * self._step) for n in count()) - if self._growing: - return takewhile(partial(gt, self._stop), values) - else: - return takewhile(partial(lt, self._stop), values) - - def __len__(self): - return self._len - - def _init_len(self): - if self._growing: - start = self._start - stop = self._stop - step = self._step - else: - start = self._stop - stop = self._start - step = -self._step - distance = stop - start - if distance <= self._zero: - self._len = 0 - else: # distance > 0 and step > 0: regular euclidean division - q, r = divmod(distance, step) - self._len = int(q) + int(r != self._zero) - - def __reduce__(self): - return numeric_range, (self._start, self._stop, self._step) - - def __repr__(self): - if self._step == 1: - return "numeric_range({}, {})".format( - repr(self._start), repr(self._stop) - ) - else: - return "numeric_range({}, {}, {})".format( - repr(self._start), repr(self._stop), repr(self._step) - ) - - def __reversed__(self): - return iter( - numeric_range( - self._get_by_index(-1), self._start - self._step, -self._step - ) - ) - - def count(self, value): - return int(value in self) - - def index(self, value): - if self._growing: - if self._start <= value < self._stop: - q, r = divmod(value - self._start, self._step) - if r == self._zero: - return int(q) - else: - if self._start >= value > self._stop: - q, r = divmod(self._start - value, -self._step) - if r == self._zero: - return int(q) - - raise ValueError("{} is not in numeric range".format(value)) - - def _get_by_index(self, i): - if i < 0: - i += self._len - if i < 0 or i >= self._len: - raise IndexError("numeric range object index out of range") - return self._start + i * self._step - - -def count_cycle(iterable, n=None): - """Cycle through the items from *iterable* up to *n* times, yielding - the number of completed cycles along with each item. If *n* is omitted the - process repeats indefinitely. - - >>> list(count_cycle('AB', 3)) - [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] - - """ - iterable = tuple(iterable) - if not iterable: - return iter(()) - counter = count() if n is None else range(n) - return ((i, item) for i in counter for item in iterable) - - -def mark_ends(iterable): - """Yield 3-tuples of the form ``(is_first, is_last, item)``. - - >>> list(mark_ends('ABC')) - [(True, False, 'A'), (False, False, 'B'), (False, True, 'C')] - - Use this when looping over an iterable to take special action on its first - and/or last items: - - >>> iterable = ['Header', 100, 200, 'Footer'] - >>> total = 0 - >>> for is_first, is_last, item in mark_ends(iterable): - ... if is_first: - ... continue # Skip the header - ... if is_last: - ... continue # Skip the footer - ... total += item - >>> print(total) - 300 - """ - it = iter(iterable) - - try: - b = next(it) - except StopIteration: - return - - try: - for i in count(): - a = b - b = next(it) - yield i == 0, False, a - - except StopIteration: - yield i == 0, True, a - - -def locate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(locate([0, 1, 1, 0, 1, 0, 0])) - [1, 2, 4] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item. - - >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) - [1, 3] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(locate(iterable, pred=pred, window_size=3)) - [1, 5, 9] - - Use with :func:`seekable` to find indexes and then retrieve the associated - items: - - >>> from itertools import count - >>> from more_itertools import seekable - >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) - >>> it = seekable(source) - >>> pred = lambda x: x > 100 - >>> indexes = locate(it, pred=pred) - >>> i = next(indexes) - >>> it.seek(i) - >>> next(it) - 106 - - """ - if window_size is None: - return compress(count(), map(pred, iterable)) - - if window_size < 1: - raise ValueError('window size must be at least 1') - - it = windowed(iterable, window_size, fillvalue=_marker) - return compress(count(), starmap(pred, it)) - - -def lstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the beginning - for which *pred* returns ``True``. - - For example, to remove a set of items from the start of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(lstrip(iterable, pred)) - [1, 2, None, 3, False, None] - - This function is analogous to to :func:`str.lstrip`, and is essentially - an wrapper for :func:`itertools.dropwhile`. - - """ - return dropwhile(pred, iterable) - - -def rstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the end - for which *pred* returns ``True``. - - For example, to remove a set of items from the end of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(rstrip(iterable, pred)) - [None, False, None, 1, 2, None, 3] - - This function is analogous to :func:`str.rstrip`. - - """ - cache = [] - cache_append = cache.append - cache_clear = cache.clear - for x in iterable: - if pred(x): - cache_append(x) - else: - yield from cache - cache_clear() - yield x - - -def strip(iterable, pred): - """Yield the items from *iterable*, but strip any from the - beginning and end for which *pred* returns ``True``. - - For example, to remove a set of items from both ends of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(strip(iterable, pred)) - [1, 2, None, 3] - - This function is analogous to :func:`str.strip`. - - """ - return rstrip(lstrip(iterable, pred), pred) - - -class islice_extended: - """An extension of :func:`itertools.islice` that supports negative values - for *stop*, *start*, and *step*. - - >>> iterable = iter('abcdefgh') - >>> list(islice_extended(iterable, -4, -1)) - ['e', 'f', 'g'] - - Slices with negative values require some caching of *iterable*, but this - function takes care to minimize the amount of memory required. - - For example, you can use a negative step with an infinite iterator: - - >>> from itertools import count - >>> list(islice_extended(count(), 110, 99, -2)) - [110, 108, 106, 104, 102, 100] - - You can also use slice notation directly: - - >>> iterable = map(str, count()) - >>> it = islice_extended(iterable)[10:20:2] - >>> list(it) - ['10', '12', '14', '16', '18'] - - """ - - def __init__(self, iterable, *args): - it = iter(iterable) - if args: - self._iterable = _islice_helper(it, slice(*args)) - else: - self._iterable = it - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterable) - - def __getitem__(self, key): - if isinstance(key, slice): - return islice_extended(_islice_helper(self._iterable, key)) - - raise TypeError('islice_extended.__getitem__ argument must be a slice') - - -def _islice_helper(it, s): - start = s.start - stop = s.stop - if s.step == 0: - raise ValueError('step argument must be a non-zero integer or None.') - step = s.step or 1 - - if step > 0: - start = 0 if (start is None) else start - - if start < 0: - # Consume all but the last -start items - cache = deque(enumerate(it, 1), maxlen=-start) - len_iter = cache[-1][0] if cache else 0 - - # Adjust start to be positive - i = max(len_iter + start, 0) - - # Adjust stop to be positive - if stop is None: - j = len_iter - elif stop >= 0: - j = min(stop, len_iter) - else: - j = max(len_iter + stop, 0) - - # Slice the cache - n = j - i - if n <= 0: - return - - for index, item in islice(cache, 0, n, step): - yield item - elif (stop is not None) and (stop < 0): - # Advance to the start position - next(islice(it, start, start), None) - - # When stop is negative, we have to carry -stop items while - # iterating - cache = deque(islice(it, -stop), maxlen=-stop) - - for index, item in enumerate(it): - cached_item = cache.popleft() - if index % step == 0: - yield cached_item - cache.append(item) - else: - # When both start and stop are positive we have the normal case - yield from islice(it, start, stop, step) - else: - start = -1 if (start is None) else start - - if (stop is not None) and (stop < 0): - # Consume all but the last items - n = -stop - 1 - cache = deque(enumerate(it, 1), maxlen=n) - len_iter = cache[-1][0] if cache else 0 - - # If start and stop are both negative they are comparable and - # we can just slice. Otherwise we can adjust start to be negative - # and then slice. - if start < 0: - i, j = start, stop - else: - i, j = min(start - len_iter, -1), None - - for index, item in list(cache)[i:j:step]: - yield item - else: - # Advance to the stop position - if stop is not None: - m = stop + 1 - next(islice(it, m, m), None) - - # stop is positive, so if start is negative they are not comparable - # and we need the rest of the items. - if start < 0: - i = start - n = None - # stop is None and start is positive, so we just need items up to - # the start index. - elif stop is None: - i = None - n = start + 1 - # Both stop and start are positive, so they are comparable. - else: - i = None - n = start - stop - if n <= 0: - return - - cache = list(islice(it, n)) - - yield from cache[i::step] - - -def always_reversible(iterable): - """An extension of :func:`reversed` that supports all iterables, not - just those which implement the ``Reversible`` or ``Sequence`` protocols. - - >>> print(*always_reversible(x for x in range(3))) - 2 1 0 - - If the iterable is already reversible, this function returns the - result of :func:`reversed()`. If the iterable is not reversible, - this function will cache the remaining items in the iterable and - yield them in reverse order, which may require significant storage. - """ - try: - return reversed(iterable) - except TypeError: - return reversed(list(iterable)) - - -def consecutive_groups(iterable, ordering=lambda x: x): - """Yield groups of consecutive items using :func:`itertools.groupby`. - The *ordering* function determines whether two items are adjacent by - returning their position. - - By default, the ordering function is the identity function. This is - suitable for finding runs of numbers: - - >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] - >>> for group in consecutive_groups(iterable): - ... print(list(group)) - [1] - [10, 11, 12] - [20] - [30, 31, 32, 33] - [40] - - For finding runs of adjacent letters, try using the :meth:`index` method - of a string of letters: - - >>> from string import ascii_lowercase - >>> iterable = 'abcdfgilmnop' - >>> ordering = ascii_lowercase.index - >>> for group in consecutive_groups(iterable, ordering): - ... print(list(group)) - ['a', 'b', 'c', 'd'] - ['f', 'g'] - ['i'] - ['l', 'm', 'n', 'o', 'p'] - - Each group of consecutive items is an iterator that shares it source with - *iterable*. When an an output group is advanced, the previous group is - no longer available unless its elements are copied (e.g., into a ``list``). - - >>> iterable = [1, 2, 11, 12, 21, 22] - >>> saved_groups = [] - >>> for group in consecutive_groups(iterable): - ... saved_groups.append(list(group)) # Copy group elements - >>> saved_groups - [[1, 2], [11, 12], [21, 22]] - - """ - for k, g in groupby( - enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) - ): - yield map(itemgetter(1), g) - - -def difference(iterable, func=sub, *, initial=None): - """This function is the inverse of :func:`itertools.accumulate`. By default - it will compute the first difference of *iterable* using - :func:`operator.sub`: - - >>> from itertools import accumulate - >>> iterable = accumulate([0, 1, 2, 3, 4]) # produces 0, 1, 3, 6, 10 - >>> list(difference(iterable)) - [0, 1, 2, 3, 4] - - *func* defaults to :func:`operator.sub`, but other functions can be - specified. They will be applied as follows:: - - A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... - - For example, to do progressive division: - - >>> iterable = [1, 2, 6, 24, 120] - >>> func = lambda x, y: x // y - >>> list(difference(iterable, func)) - [1, 2, 3, 4, 5] - - If the *initial* keyword is set, the first element will be skipped when - computing successive differences. - - >>> it = [10, 11, 13, 16] # from accumulate([1, 2, 3], initial=10) - >>> list(difference(it, initial=10)) - [1, 2, 3] - - """ - a, b = tee(iterable) - try: - first = [next(b)] - except StopIteration: - return iter([]) - - if initial is not None: - first = [] - - return chain(first, starmap(func, zip(b, a))) - - -class SequenceView(Sequence): - """Return a read-only view of the sequence object *target*. - - :class:`SequenceView` objects are analogous to Python's built-in - "dictionary view" types. They provide a dynamic view of a sequence's items, - meaning that when the sequence updates, so does the view. - - >>> seq = ['0', '1', '2'] - >>> view = SequenceView(seq) - >>> view - SequenceView(['0', '1', '2']) - >>> seq.append('3') - >>> view - SequenceView(['0', '1', '2', '3']) - - Sequence views support indexing, slicing, and length queries. They act - like the underlying sequence, except they don't allow assignment: - - >>> view[1] - '1' - >>> view[1:-1] - ['1', '2'] - >>> len(view) - 4 - - Sequence views are useful as an alternative to copying, as they don't - require (much) extra storage. - - """ - - def __init__(self, target): - if not isinstance(target, Sequence): - raise TypeError - self._target = target - - def __getitem__(self, index): - return self._target[index] - - def __len__(self): - return len(self._target) - - def __repr__(self): - return '{}({})'.format(self.__class__.__name__, repr(self._target)) - - -class seekable: - """Wrap an iterator to allow for seeking backward and forward. This - progressively caches the items in the source iterable so they can be - re-visited. - - Call :meth:`seek` with an index to seek to that position in the source - iterable. - - To "reset" an iterator, seek to ``0``: - - >>> from itertools import count - >>> it = seekable((str(n) for n in count())) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> it.seek(0) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> next(it) - '3' - - You can also seek forward: - - >>> it = seekable((str(n) for n in range(20))) - >>> it.seek(10) - >>> next(it) - '10' - >>> it.seek(20) # Seeking past the end of the source isn't a problem - >>> list(it) - [] - >>> it.seek(0) # Resetting works even after hitting the end - >>> next(it), next(it), next(it) - ('0', '1', '2') - - Call :meth:`peek` to look ahead one item without advancing the iterator: - - >>> it = seekable('1234') - >>> it.peek() - '1' - >>> list(it) - ['1', '2', '3', '4'] - >>> it.peek(default='empty') - 'empty' - - Before the iterator is at its end, calling :func:`bool` on it will return - ``True``. After it will return ``False``: - - >>> it = seekable('5678') - >>> bool(it) - True - >>> list(it) - ['5', '6', '7', '8'] - >>> bool(it) - False - - You may view the contents of the cache with the :meth:`elements` method. - That returns a :class:`SequenceView`, a view that updates automatically: - - >>> it = seekable((str(n) for n in range(10))) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> elements = it.elements() - >>> elements - SequenceView(['0', '1', '2']) - >>> next(it) - '3' - >>> elements - SequenceView(['0', '1', '2', '3']) - - By default, the cache grows as the source iterable progresses, so beware of - wrapping very large or infinite iterables. Supply *maxlen* to limit the - size of the cache (this of course limits how far back you can seek). - - >>> from itertools import count - >>> it = seekable((str(n) for n in count()), maxlen=2) - >>> next(it), next(it), next(it), next(it) - ('0', '1', '2', '3') - >>> list(it.elements()) - ['2', '3'] - >>> it.seek(0) - >>> next(it), next(it), next(it), next(it) - ('2', '3', '4', '5') - >>> next(it) - '6' - - """ - - def __init__(self, iterable, maxlen=None): - self._source = iter(iterable) - if maxlen is None: - self._cache = [] - else: - self._cache = deque([], maxlen) - self._index = None - - def __iter__(self): - return self - - def __next__(self): - if self._index is not None: - try: - item = self._cache[self._index] - except IndexError: - self._index = None - else: - self._index += 1 - return item - - item = next(self._source) - self._cache.append(item) - return item - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - try: - peeked = next(self) - except StopIteration: - if default is _marker: - raise - return default - if self._index is None: - self._index = len(self._cache) - self._index -= 1 - return peeked - - def elements(self): - return SequenceView(self._cache) - - def seek(self, index): - self._index = index - remainder = index - len(self._cache) - if remainder > 0: - consume(self, remainder) - - -class run_length: - """ - :func:`run_length.encode` compresses an iterable with run-length encoding. - It yields groups of repeated items with the count of how many times they - were repeated: - - >>> uncompressed = 'abbcccdddd' - >>> list(run_length.encode(uncompressed)) - [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - - :func:`run_length.decode` decompresses an iterable that was previously - compressed with run-length encoding. It yields the items of the - decompressed iterable: - - >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> list(run_length.decode(compressed)) - ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] - - """ - - @staticmethod - def encode(iterable): - return ((k, ilen(g)) for k, g in groupby(iterable)) - - @staticmethod - def decode(iterable): - return chain.from_iterable(repeat(k, n) for k, n in iterable) - - -def exactly_n(iterable, n, predicate=bool): - """Return ``True`` if exactly ``n`` items in the iterable are ``True`` - according to the *predicate* function. - - >>> exactly_n([True, True, False], 2) - True - >>> exactly_n([True, True, False], 1) - False - >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) - True - - The iterable will be advanced until ``n + 1`` truthy items are encountered, - so avoid calling it on infinite iterables. - - """ - return len(take(n + 1, filter(predicate, iterable))) == n - - -def circular_shifts(iterable): - """Return a list of circular shifts of *iterable*. - - >>> circular_shifts(range(4)) - [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] - """ - lst = list(iterable) - return take(len(lst), windowed(cycle(lst), len(lst))) - - -def make_decorator(wrapping_func, result_index=0): - """Return a decorator version of *wrapping_func*, which is a function that - modifies an iterable. *result_index* is the position in that function's - signature where the iterable goes. - - This lets you use itertools on the "production end," i.e. at function - definition. This can augment what the function returns without changing the - function's code. - - For example, to produce a decorator version of :func:`chunked`: - - >>> from more_itertools import chunked - >>> chunker = make_decorator(chunked, result_index=0) - >>> @chunker(3) - ... def iter_range(n): - ... return iter(range(n)) - ... - >>> list(iter_range(9)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8]] - - To only allow truthy items to be returned: - - >>> truth_serum = make_decorator(filter, result_index=1) - >>> @truth_serum(bool) - ... def boolean_test(): - ... return [0, 1, '', ' ', False, True] - ... - >>> list(boolean_test()) - [1, ' ', True] - - The :func:`peekable` and :func:`seekable` wrappers make for practical - decorators: - - >>> from more_itertools import peekable - >>> peekable_function = make_decorator(peekable) - >>> @peekable_function() - ... def str_range(*args): - ... return (str(x) for x in range(*args)) - ... - >>> it = str_range(1, 20, 2) - >>> next(it), next(it), next(it) - ('1', '3', '5') - >>> it.peek() - '7' - >>> next(it) - '7' - - """ - # See https://sites.google.com/site/bbayles/index/decorator_factory for - # notes on how this works. - def decorator(*wrapping_args, **wrapping_kwargs): - def outer_wrapper(f): - def inner_wrapper(*args, **kwargs): - result = f(*args, **kwargs) - wrapping_args_ = list(wrapping_args) - wrapping_args_.insert(result_index, result) - return wrapping_func(*wrapping_args_, **wrapping_kwargs) - - return inner_wrapper - - return outer_wrapper - - return decorator - - -def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): - """Return a dictionary that maps the items in *iterable* to categories - defined by *keyfunc*, transforms them with *valuefunc*, and - then summarizes them by category with *reducefunc*. - - *valuefunc* defaults to the identity function if it is unspecified. - If *reducefunc* is unspecified, no summarization takes place: - - >>> keyfunc = lambda x: x.upper() - >>> result = map_reduce('abbccc', keyfunc) - >>> sorted(result.items()) - [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] - - Specifying *valuefunc* transforms the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> result = map_reduce('abbccc', keyfunc, valuefunc) - >>> sorted(result.items()) - [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] - - Specifying *reducefunc* summarizes the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> reducefunc = sum - >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) - >>> sorted(result.items()) - [('A', 1), ('B', 2), ('C', 3)] - - You may want to filter the input iterable before applying the map/reduce - procedure: - - >>> all_items = range(30) - >>> items = [x for x in all_items if 10 <= x <= 20] # Filter - >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 - >>> categories = map_reduce(items, keyfunc=keyfunc) - >>> sorted(categories.items()) - [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] - >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) - >>> sorted(summaries.items()) - [(0, 90), (1, 75)] - - Note that all items in the iterable are gathered into a list before the - summarization step, which may require significant storage. - - The returned object is a :obj:`collections.defaultdict` with the - ``default_factory`` set to ``None``, such that it behaves like a normal - dictionary. - - """ - valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc - - ret = defaultdict(list) - for item in iterable: - key = keyfunc(item) - value = valuefunc(item) - ret[key].append(value) - - if reducefunc is not None: - for key, value_list in ret.items(): - ret[key] = reducefunc(value_list) - - ret.default_factory = None - return ret - - -def rlocate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``, starting from the right and moving left. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 - [4, 2, 1] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item: - - >>> iterable = iter('abcb') - >>> pred = lambda x: x == 'b' - >>> list(rlocate(iterable, pred)) - [3, 1] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(rlocate(iterable, pred=pred, window_size=3)) - [9, 5, 1] - - Beware, this function won't return anything for infinite iterables. - If *iterable* is reversible, ``rlocate`` will reverse it and search from - the right. Otherwise, it will search from the left and return the results - in reverse order. - - See :func:`locate` to for other example applications. - - """ - if window_size is None: - try: - len_iter = len(iterable) - return (len_iter - i - 1 for i in locate(reversed(iterable), pred)) - except TypeError: - pass - - return reversed(list(locate(iterable, pred, window_size))) - - -def replace(iterable, pred, substitutes, count=None, window_size=1): - """Yield the items from *iterable*, replacing the items for which *pred* - returns ``True`` with the items from the iterable *substitutes*. - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] - >>> pred = lambda x: x == 0 - >>> substitutes = (2, 3) - >>> list(replace(iterable, pred, substitutes)) - [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] - - If *count* is given, the number of replacements will be limited: - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] - >>> pred = lambda x: x == 0 - >>> substitutes = [None] - >>> list(replace(iterable, pred, substitutes, count=2)) - [1, 1, None, 1, 1, None, 1, 1, 0] - - Use *window_size* to control the number of items passed as arguments to - *pred*. This allows for locating and replacing subsequences. - - >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] - >>> window_size = 3 - >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred - >>> substitutes = [3, 4] # Splice in these items - >>> list(replace(iterable, pred, substitutes, window_size=window_size)) - [3, 4, 5, 3, 4, 5] - - """ - if window_size < 1: - raise ValueError('window_size must be at least 1') - - # Save the substitutes iterable, since it's used more than once - substitutes = tuple(substitutes) - - # Add padding such that the number of windows matches the length of the - # iterable - it = chain(iterable, [_marker] * (window_size - 1)) - windows = windowed(it, window_size) - - n = 0 - for w in windows: - # If the current window matches our predicate (and we haven't hit - # our maximum number of replacements), splice in the substitutes - # and then consume the following windows that overlap with this one. - # For example, if the iterable is (0, 1, 2, 3, 4...) - # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... - # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) - if pred(*w): - if (count is None) or (n < count): - n += 1 - yield from substitutes - consume(windows, window_size - 1) - continue - - # If there was no match (or we've reached the replacement limit), - # yield the first item from the window. - if w and (w[0] is not _marker): - yield w[0] - - -def partitions(iterable): - """Yield all possible order-preserving partitions of *iterable*. - - >>> iterable = 'abc' - >>> for part in partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['a', 'b', 'c'] - - This is unrelated to :func:`partition`. - - """ - sequence = list(iterable) - n = len(sequence) - for i in powerset(range(1, n)): - yield [sequence[i:j] for i, j in zip((0,) + i, i + (n,))] - - -def set_partitions(iterable, k=None): - """ - Yield the set partitions of *iterable* into *k* parts. Set partitions are - not order-preserving. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable, 2): - ... print([''.join(p) for p in part]) - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - - - If *k* is not given, every set partition is generated. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - ['a', 'b', 'c'] - - """ - L = list(iterable) - n = len(L) - if k is not None: - if k < 1: - raise ValueError( - "Can't partition in a negative or zero number of groups" - ) - elif k > n: - return - - def set_partitions_helper(L, k): - n = len(L) - if k == 1: - yield [L] - elif n == k: - yield [[s] for s in L] - else: - e, *M = L - for p in set_partitions_helper(M, k - 1): - yield [[e], *p] - for p in set_partitions_helper(M, k): - for i in range(len(p)): - yield p[:i] + [[e] + p[i]] + p[i + 1 :] - - if k is None: - for k in range(1, n + 1): - yield from set_partitions_helper(L, k) - else: - yield from set_partitions_helper(L, k) - - -class time_limited: - """ - Yield items from *iterable* until *limit_seconds* have passed. - If the time limit expires before all items have been yielded, the - ``timed_out`` parameter will be set to ``True``. - - >>> from time import sleep - >>> def generator(): - ... yield 1 - ... yield 2 - ... sleep(0.2) - ... yield 3 - >>> iterable = time_limited(0.1, generator()) - >>> list(iterable) - [1, 2] - >>> iterable.timed_out - True - - Note that the time is checked before each item is yielded, and iteration - stops if the time elapsed is greater than *limit_seconds*. If your time - limit is 1 second, but it takes 2 seconds to generate the first item from - the iterable, the function will run for 2 seconds and not yield anything. - - """ - - def __init__(self, limit_seconds, iterable): - if limit_seconds < 0: - raise ValueError('limit_seconds must be positive') - self.limit_seconds = limit_seconds - self._iterable = iter(iterable) - self._start_time = monotonic() - self.timed_out = False - - def __iter__(self): - return self - - def __next__(self): - item = next(self._iterable) - if monotonic() - self._start_time > self.limit_seconds: - self.timed_out = True - raise StopIteration - - return item - - -def only(iterable, default=None, too_long=None): - """If *iterable* has only one item, return it. - If it has zero items, return *default*. - If it has more than one item, raise the exception given by *too_long*, - which is ``ValueError`` by default. - - >>> only([], default='missing') - 'missing' - >>> only([1]) - 1 - >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 1, 2, - and perhaps more.' - >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - TypeError - - Note that :func:`only` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check - iterable contents less destructively. - """ - it = iter(iterable) - first_value = next(it, default) - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def ichunked(iterable, n): - """Break *iterable* into sub-iterables with *n* elements each. - :func:`ichunked` is like :func:`chunked`, but it yields iterables - instead of lists. - - If the sub-iterables are read in order, the elements of *iterable* - won't be stored in memory. - If they are read out of order, :func:`itertools.tee` is used to cache - elements as necessary. - - >>> from itertools import count - >>> all_chunks = ichunked(count(), 4) - >>> c_1, c_2, c_3 = next(all_chunks), next(all_chunks), next(all_chunks) - >>> list(c_2) # c_1's elements have been cached; c_3's haven't been - [4, 5, 6, 7] - >>> list(c_1) - [0, 1, 2, 3] - >>> list(c_3) - [8, 9, 10, 11] - - """ - source = iter(iterable) - - while True: - # Check to see whether we're at the end of the source iterable - item = next(source, _marker) - if item is _marker: - return - - # Clone the source and yield an n-length slice - source, it = tee(chain([item], source)) - yield islice(it, n) - - # Advance the source iterable - consume(source, n) - - -def distinct_combinations(iterable, r): - """Yield the distinct combinations of *r* items taken from *iterable*. - - >>> list(distinct_combinations([0, 0, 1], 2)) - [(0, 0), (0, 1)] - - Equivalent to ``set(combinations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - """ - if r < 0: - raise ValueError('r must be non-negative') - elif r == 0: - yield () - return - pool = tuple(iterable) - generators = [unique_everseen(enumerate(pool), key=itemgetter(1))] - current_combo = [None] * r - level = 0 - while generators: - try: - cur_idx, p = next(generators[-1]) - except StopIteration: - generators.pop() - level -= 1 - continue - current_combo[level] = p - if level + 1 == r: - yield tuple(current_combo) - else: - generators.append( - unique_everseen( - enumerate(pool[cur_idx + 1 :], cur_idx + 1), - key=itemgetter(1), - ) - ) - level += 1 - - -def filter_except(validator, iterable, *exceptions): - """Yield the items from *iterable* for which the *validator* function does - not raise one of the specified *exceptions*. - - *validator* is called for each item in *iterable*. - It should be a function that accepts one argument and raises an exception - if that item is not valid. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(filter_except(int, iterable, ValueError, TypeError)) - ['1', '2', '4'] - - If an exception other than one given by *exceptions* is raised by - *validator*, it is raised like normal. - """ - for item in iterable: - try: - validator(item) - except exceptions: - pass - else: - yield item - - -def map_except(function, iterable, *exceptions): - """Transform each item from *iterable* with *function* and yield the - result, unless *function* raises one of the specified *exceptions*. - - *function* is called to transform each item in *iterable*. - It should be a accept one argument. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(map_except(int, iterable, ValueError, TypeError)) - [1, 2, 4] - - If an exception other than one given by *exceptions* is raised by - *function*, it is raised like normal. - """ - for item in iterable: - try: - yield function(item) - except exceptions: - pass - - -def _sample_unweighted(iterable, k): - # Implementation of "Algorithm L" from the 1994 paper by Kim-Hung Li: - # "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". - - # Fill up the reservoir (collection of samples) with the first `k` samples - reservoir = take(k, iterable) - - # Generate random number that's the largest in a sample of k U(0,1) numbers - # Largest order statistic: https://en.wikipedia.org/wiki/Order_statistic - W = exp(log(random()) / k) - - # The number of elements to skip before changing the reservoir is a random - # number with a geometric distribution. Sample it using random() and logs. - next_index = k + floor(log(random()) / log(1 - W)) - - for index, element in enumerate(iterable, k): - - if index == next_index: - reservoir[randrange(k)] = element - # The new W is the largest in a sample of k U(0, `old_W`) numbers - W *= exp(log(random()) / k) - next_index += floor(log(random()) / log(1 - W)) + 1 - - return reservoir - - -def _sample_weighted(iterable, k, weights): - # Implementation of "A-ExpJ" from the 2006 paper by Efraimidis et al. : - # "Weighted random sampling with a reservoir". - - # Log-transform for numerical stability for weights that are small/large - weight_keys = (log(random()) / weight for weight in weights) - - # Fill up the reservoir (collection of samples) with the first `k` - # weight-keys and elements, then heapify the list. - reservoir = take(k, zip(weight_keys, iterable)) - heapify(reservoir) - - # The number of jumps before changing the reservoir is a random variable - # with an exponential distribution. Sample it using random() and logs. - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - - for weight, element in zip(weights, iterable): - if weight >= weights_to_skip: - # The notation here is consistent with the paper, but we store - # the weight-keys in log-space for better numerical stability. - smallest_weight_key, _ = reservoir[0] - t_w = exp(weight * smallest_weight_key) - r_2 = uniform(t_w, 1) # generate U(t_w, 1) - weight_key = log(r_2) / weight - heapreplace(reservoir, (weight_key, element)) - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - else: - weights_to_skip -= weight - - # Equivalent to [element for weight_key, element in sorted(reservoir)] - return [heappop(reservoir)[1] for _ in range(k)] - - -def sample(iterable, k, weights=None): - """Return a *k*-length list of elements chosen (without replacement) - from the *iterable*. Like :func:`random.sample`, but works on iterables - of unknown length. - - >>> iterable = range(100) - >>> sample(iterable, 5) # doctest: +SKIP - [81, 60, 96, 16, 4] - - An iterable with *weights* may also be given: - - >>> iterable = range(100) - >>> weights = (i * i + 1 for i in range(100)) - >>> sampled = sample(iterable, 5, weights=weights) # doctest: +SKIP - [79, 67, 74, 66, 78] - - The algorithm can also be used to generate weighted random permutations. - The relative weight of each item determines the probability that it - appears late in the permutation. - - >>> data = "abcdefgh" - >>> weights = range(1, len(data) + 1) - >>> sample(data, k=len(data), weights=weights) # doctest: +SKIP - ['c', 'a', 'b', 'e', 'g', 'd', 'h', 'f'] - """ - if k == 0: - return [] - - iterable = iter(iterable) - if weights is None: - return _sample_unweighted(iterable, k) - else: - weights = iter(weights) - return _sample_weighted(iterable, k, weights) - - -def is_sorted(iterable, key=None, reverse=False): - """Returns ``True`` if the items of iterable are in sorted order, and - ``False`` otherwise. *key* and *reverse* have the same meaning that they do - in the built-in :func:`sorted` function. - - >>> is_sorted(['1', '2', '3', '4', '5'], key=int) - True - >>> is_sorted([5, 4, 3, 1, 2], reverse=True) - False - - The function returns ``False`` after encountering the first out-of-order - item. If there are no out-of-order items, the iterable is exhausted. - """ - - compare = lt if reverse else gt - it = iterable if (key is None) else map(key, iterable) - return not any(starmap(compare, pairwise(it))) - - -class AbortThread(BaseException): - pass - - -class callback_iter: - """Convert a function that uses callbacks to an iterator. - - Let *func* be a function that takes a `callback` keyword argument. - For example: - - >>> def func(callback=None): - ... for i, c in [(1, 'a'), (2, 'b'), (3, 'c')]: - ... if callback: - ... callback(i, c) - ... return 4 - - - Use ``with callback_iter(func)`` to get an iterator over the parameters - that are delivered to the callback. - - >>> with callback_iter(func) as it: - ... for args, kwargs in it: - ... print(args) - (1, 'a') - (2, 'b') - (3, 'c') - - The function will be called in a background thread. The ``done`` property - indicates whether it has completed execution. - - >>> it.done - True - - If it completes successfully, its return value will be available - in the ``result`` property. - - >>> it.result - 4 - - Notes: - - * If the function uses some keyword argument besides ``callback``, supply - *callback_kwd*. - * If it finished executing, but raised an exception, accessing the - ``result`` property will raise the same exception. - * If it hasn't finished executing, accessing the ``result`` - property from within the ``with`` block will raise ``RuntimeError``. - * If it hasn't finished executing, accessing the ``result`` property from - outside the ``with`` block will raise a - ``more_itertools.AbortThread`` exception. - * Provide *wait_seconds* to adjust how frequently the it is polled for - output. - - """ - - def __init__(self, func, callback_kwd='callback', wait_seconds=0.1): - self._func = func - self._callback_kwd = callback_kwd - self._aborted = False - self._future = None - self._wait_seconds = wait_seconds - self._executor = __import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1) - self._iterator = self._reader() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._aborted = True - self._executor.shutdown() - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterator) - - @property - def done(self): - if self._future is None: - return False - return self._future.done() - - @property - def result(self): - if not self.done: - raise RuntimeError('Function has not yet completed') - - return self._future.result() - - def _reader(self): - q = Queue() - - def callback(*args, **kwargs): - if self._aborted: - raise AbortThread('canceled by user') - - q.put((args, kwargs)) - - self._future = self._executor.submit( - self._func, **{self._callback_kwd: callback} - ) - - while True: - try: - item = q.get(timeout=self._wait_seconds) - except Empty: - pass - else: - q.task_done() - yield item - - if self._future.done(): - break - - remaining = [] - while True: - try: - item = q.get_nowait() - except Empty: - break - else: - q.task_done() - remaining.append(item) - q.join() - yield from remaining - - -def windowed_complete(iterable, n): - """ - Yield ``(beginning, middle, end)`` tuples, where: - - * Each ``middle`` has *n* items from *iterable* - * Each ``beginning`` has the items before the ones in ``middle`` - * Each ``end`` has the items after the ones in ``middle`` - - >>> iterable = range(7) - >>> n = 3 - >>> for beginning, middle, end in windowed_complete(iterable, n): - ... print(beginning, middle, end) - () (0, 1, 2) (3, 4, 5, 6) - (0,) (1, 2, 3) (4, 5, 6) - (0, 1) (2, 3, 4) (5, 6) - (0, 1, 2) (3, 4, 5) (6,) - (0, 1, 2, 3) (4, 5, 6) () - - Note that *n* must be at least 0 and most equal to the length of - *iterable*. - - This function will exhaust the iterable and may require significant - storage. - """ - if n < 0: - raise ValueError('n must be >= 0') - - seq = tuple(iterable) - size = len(seq) - - if n > size: - raise ValueError('n must be <= len(seq)') - - for i in range(size - n + 1): - beginning = seq[:i] - middle = seq[i : i + n] - end = seq[i + n :] - yield beginning, middle, end - - -def all_unique(iterable, key=None): - """ - Returns ``True`` if all the elements of *iterable* are unique (no two - elements are equal). - - >>> all_unique('ABCB') - False - - If a *key* function is specified, it will be used to make comparisons. - - >>> all_unique('ABCb') - True - >>> all_unique('ABCb', str.lower) - False - - The function returns as soon as the first non-unique element is - encountered. Iterables with a mix of hashable and unhashable items can - be used, but the function will be slower for unhashable items. - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - for element in map(key, iterable) if key else iterable: - try: - if element in seenset: - return False - seenset_add(element) - except TypeError: - if element in seenlist: - return False - seenlist_add(element) - return True - - -def nth_product(index, *args): - """Equivalent to ``list(product(*args))[index]``. - - The products of *args* can be ordered lexicographically. - :func:`nth_product` computes the product at sort position *index* without - computing the previous products. - - >>> nth_product(8, range(2), range(2), range(2), range(2)) - (1, 0, 0, 0) - - ``IndexError`` will be raised if the given *index* is invalid. - """ - pools = list(map(tuple, reversed(args))) - ns = list(map(len, pools)) - - c = reduce(mul, ns) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - result = [] - for pool, n in zip(pools, ns): - result.append(pool[index % n]) - index //= n - - return tuple(reversed(result)) - - -def nth_permutation(iterable, r, index): - """Equivalent to ``list(permutations(iterable, r))[index]``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`nth_permutation` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences. - - >>> nth_permutation('ghijk', 2, 5) - ('h', 'i') - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = list(iterable) - n = len(pool) - - if r is None or r == n: - r, c = n, factorial(n) - elif not 0 <= r < n: - raise ValueError - else: - c = factorial(n) // factorial(n - r) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - if c == 0: - return tuple() - - result = [0] * r - q = index * factorial(n) // c if r < n else index - for d in range(1, n + 1): - q, i = divmod(q, d) - if 0 <= n - d < r: - result[n - d] = i - if q == 0: - break - - return tuple(map(pool.pop, result)) - - -def value_chain(*args): - """Yield all arguments passed to the function in the same order in which - they were passed. If an argument itself is iterable then iterate over its - values. - - >>> list(value_chain(1, 2, 3, [4, 5, 6])) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and are emitted - as-is: - - >>> list(value_chain('12', '34', ['56', '78'])) - ['12', '34', '56', '78'] - - - Multiple levels of nesting are not flattened. - - """ - for value in args: - if isinstance(value, (str, bytes)): - yield value - continue - try: - yield from value - except TypeError: - yield value - - -def product_index(element, *args): - """Equivalent to ``list(product(*args)).index(element)`` - - The products of *args* can be ordered lexicographically. - :func:`product_index` computes the first index of *element* without - computing the previous products. - - >>> product_index([8, 2], range(10), range(5)) - 42 - - ``ValueError`` will be raised if the given *element* isn't in the product - of *args*. - """ - index = 0 - - for x, pool in zip_longest(element, args, fillvalue=_marker): - if x is _marker or pool is _marker: - raise ValueError('element is not a product of args') - - pool = tuple(pool) - index = index * len(pool) + pool.index(x) - - return index - - -def combination_index(element, iterable): - """Equivalent to ``list(combinations(iterable, r)).index(element)`` - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`combination_index` computes the index of the - first *element*, without computing the previous combinations. - - >>> combination_index('adf', 'abcdefg') - 10 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations of *iterable*. - """ - element = enumerate(element) - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = enumerate(iterable) - for n, x in pool: - if x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - else: - raise ValueError('element is not a combination of iterable') - - n, _ = last(pool, default=(n, None)) - - # Python versiosn below 3.8 don't have math.comb - index = 1 - for i, j in enumerate(reversed(indexes), start=1): - j = n - j - if i <= j: - index += factorial(j) // (factorial(i) * factorial(j - i)) - - return factorial(n + 1) // (factorial(k + 1) * factorial(n - k)) - index - - -def permutation_index(element, iterable): - """Equivalent to ``list(permutations(iterable, r)).index(element)``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`permutation_index` - computes the index of the first *element* directly, without computing - the previous permutations. - - >>> permutation_index([1, 3, 2], range(5)) - 19 - - ``ValueError`` will be raised if the given *element* isn't one of the - permutations of *iterable*. - """ - index = 0 - pool = list(iterable) - for i, x in zip(range(len(pool), -1, -1), element): - r = pool.index(x) - index = index * i + r - del pool[r] - - return index - - -class countable: - """Wrap *iterable* and keep a count of how many items have been consumed. - - The ``items_seen`` attribute starts at ``0`` and increments as the iterable - is consumed: - - >>> iterable = map(str, range(10)) - >>> it = countable(iterable) - >>> it.items_seen - 0 - >>> next(it), next(it) - ('0', '1') - >>> list(it) - ['2', '3', '4', '5', '6', '7', '8', '9'] - >>> it.items_seen - 10 - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self.items_seen = 0 - - def __iter__(self): - return self - - def __next__(self): - item = next(self._it) - self.items_seen += 1 - - return item diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi deleted file mode 100644 index 2fba9cb300b..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi +++ /dev/null @@ -1,480 +0,0 @@ -"""Stubs for more_itertools.more""" - -from typing import ( - Any, - Callable, - Container, - Dict, - Generic, - Hashable, - Iterable, - Iterator, - List, - Optional, - Reversible, - Sequence, - Sized, - Tuple, - Union, - TypeVar, - type_check_only, -) -from types import TracebackType -from typing_extensions import ContextManager, Protocol, Type, overload - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') -_V = TypeVar('_V') -_W = TypeVar('_W') -_T_co = TypeVar('_T_co', covariant=True) -_GenFn = TypeVar('_GenFn', bound=Callable[..., Iterator[object]]) -_Raisable = Union[BaseException, 'Type[BaseException]'] - -@type_check_only -class _SizedIterable(Protocol[_T_co], Sized, Iterable[_T_co]): ... - -@type_check_only -class _SizedReversible(Protocol[_T_co], Sized, Reversible[_T_co]): ... - -def chunked( - iterable: Iterable[_T], n: int, strict: bool = ... -) -> Iterator[List[_T]]: ... -@overload -def first(iterable: Iterable[_T]) -> _T: ... -@overload -def first(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... -@overload -def last(iterable: Iterable[_T]) -> _T: ... -@overload -def last(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int) -> _T: ... -@overload -def nth_or_last( - iterable: Iterable[_T], n: int, default: _U -) -> Union[_T, _U]: ... - -class peekable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> peekable[_T]: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> Union[_T, _U]: ... - def prepend(self, *items: _T) -> None: ... - def __next__(self) -> _T: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> List[_T]: ... - -def collate(*iterables: Iterable[_T], **kwargs: Any) -> Iterable[_T]: ... -def consumer(func: _GenFn) -> _GenFn: ... -def ilen(iterable: Iterable[object]) -> int: ... -def iterate(func: Callable[[_T], _T], start: _T) -> Iterator[_T]: ... -def with_iter( - context_manager: ContextManager[Iterable[_T]], -) -> Iterator[_T]: ... -def one( - iterable: Iterable[_T], - too_short: Optional[_Raisable] = ..., - too_long: Optional[_Raisable] = ..., -) -> _T: ... -def distinct_permutations( - iterable: Iterable[_T], r: Optional[int] = ... -) -> Iterator[Tuple[_T, ...]]: ... -def intersperse( - e: _U, iterable: Iterable[_T], n: int = ... -) -> Iterator[Union[_T, _U]]: ... -def unique_to_each(*iterables: Iterable[_T]) -> List[List[_T]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, *, step: int = ... -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, fillvalue: _U, step: int = ... -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def substrings(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -def substrings_indexes( - seq: Sequence[_T], reverse: bool = ... -) -> Iterator[Tuple[Sequence[_T], int, int]]: ... - -class bucket(Generic[_T, _U], Container[_U]): - def __init__( - self, - iterable: Iterable[_T], - key: Callable[[_T], _U], - validator: Optional[Callable[[object], object]] = ..., - ) -> None: ... - def __contains__(self, value: object) -> bool: ... - def __iter__(self) -> Iterator[_U]: ... - def __getitem__(self, value: object) -> Iterator[_T]: ... - -def spy( - iterable: Iterable[_T], n: int = ... -) -> Tuple[List[_T], Iterator[_T]]: ... -def interleave(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_longest(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def collapse( - iterable: Iterable[Any], - base_type: Optional[type] = ..., - levels: Optional[int] = ..., -) -> Iterator[Any]: ... -@overload -def side_effect( - func: Callable[[_T], object], - iterable: Iterable[_T], - chunk_size: None = ..., - before: Optional[Callable[[], object]] = ..., - after: Optional[Callable[[], object]] = ..., -) -> Iterator[_T]: ... -@overload -def side_effect( - func: Callable[[List[_T]], object], - iterable: Iterable[_T], - chunk_size: int, - before: Optional[Callable[[], object]] = ..., - after: Optional[Callable[[], object]] = ..., -) -> Iterator[_T]: ... -def sliced( - seq: Sequence[_T], n: int, strict: bool = ... -) -> Iterator[Sequence[_T]]: ... -def split_at( - iterable: Iterable[_T], - pred: Callable[[_T], object], - maxsplit: int = ..., - keep_separator: bool = ..., -) -> Iterator[List[_T]]: ... -def split_before( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[List[_T]]: ... -def split_after( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[List[_T]]: ... -def split_when( - iterable: Iterable[_T], - pred: Callable[[_T, _T], object], - maxsplit: int = ..., -) -> Iterator[List[_T]]: ... -def split_into( - iterable: Iterable[_T], sizes: Iterable[Optional[int]] -) -> Iterator[List[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - *, - n: Optional[int] = ..., - next_multiple: bool = ... -) -> Iterator[Optional[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - fillvalue: _U, - n: Optional[int] = ..., - next_multiple: bool = ..., -) -> Iterator[Union[_T, _U]]: ... -@overload -def repeat_last(iterable: Iterable[_T]) -> Iterator[_T]: ... -@overload -def repeat_last( - iterable: Iterable[_T], default: _U -) -> Iterator[Union[_T, _U]]: ... -def distribute(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., - fillvalue: _U = ..., -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... - -class UnequalIterablesError(ValueError): - def __init__( - self, details: Optional[Tuple[int, int, int]] = ... - ) -> None: ... - -def zip_equal(*iterables: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -@overload -def zip_offset( - *iterables: Iterable[_T], offsets: _SizedIterable[int], longest: bool = ... -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def zip_offset( - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def sort_together( - iterables: Iterable[Iterable[_T]], - key_list: Iterable[int] = ..., - key: Optional[Callable[..., Any]] = ..., - reverse: bool = ..., -) -> List[Tuple[_T, ...]]: ... -def unzip(iterable: Iterable[Sequence[_T]]) -> Tuple[Iterator[_T], ...]: ... -def divide(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... -def always_iterable( - obj: object, - base_type: Union[ - type, Tuple[Union[type, Tuple[Any, ...]], ...], None - ] = ..., -) -> Iterator[Any]: ... -def adjacent( - predicate: Callable[[_T], bool], - iterable: Iterable[_T], - distance: int = ..., -) -> Iterator[Tuple[bool, _T]]: ... -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Optional[Callable[[_T], _U]] = ..., - valuefunc: Optional[Callable[[_T], _V]] = ..., - reducefunc: Optional[Callable[..., _W]] = ..., -) -> Iterator[Tuple[_T, _W]]: ... - -class numeric_range(Generic[_T, _U], Sequence[_T], Hashable, Reversible[_T]): - @overload - def __init__(self, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T, __step: _U) -> None: ... - def __bool__(self) -> bool: ... - def __contains__(self, elem: object) -> bool: ... - def __eq__(self, other: object) -> bool: ... - @overload - def __getitem__(self, key: int) -> _T: ... - @overload - def __getitem__(self, key: slice) -> numeric_range[_T, _U]: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[_T]: ... - def __len__(self) -> int: ... - def __reduce__( - self, - ) -> Tuple[Type[numeric_range[_T, _U]], Tuple[_T, _T, _U]]: ... - def __repr__(self) -> str: ... - def __reversed__(self) -> Iterator[_T]: ... - def count(self, value: _T) -> int: ... - def index(self, value: _T) -> int: ... # type: ignore - -def count_cycle( - iterable: Iterable[_T], n: Optional[int] = ... -) -> Iterable[Tuple[int, _T]]: ... -def mark_ends( - iterable: Iterable[_T], -) -> Iterable[Tuple[bool, bool, _T]]: ... -def locate( - iterable: Iterable[object], - pred: Callable[..., Any] = ..., - window_size: Optional[int] = ..., -) -> Iterator[int]: ... -def lstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def rstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def strip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... - -class islice_extended(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], *args: Optional[int] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - def __getitem__(self, index: slice) -> islice_extended[_T]: ... - -def always_reversible(iterable: Iterable[_T]) -> Iterator[_T]: ... -def consecutive_groups( - iterable: Iterable[_T], ordering: Callable[[_T], int] = ... -) -> Iterator[Iterator[_T]]: ... -@overload -def difference( - iterable: Iterable[_T], - func: Callable[[_T, _T], _U] = ..., - *, - initial: None = ... -) -> Iterator[Union[_T, _U]]: ... -@overload -def difference( - iterable: Iterable[_T], func: Callable[[_T, _T], _U] = ..., *, initial: _U -) -> Iterator[_U]: ... - -class SequenceView(Generic[_T], Sequence[_T]): - def __init__(self, target: Sequence[_T]) -> None: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> Sequence[_T]: ... - def __len__(self) -> int: ... - -class seekable(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], maxlen: Optional[int] = ... - ) -> None: ... - def __iter__(self) -> seekable[_T]: ... - def __next__(self) -> _T: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> Union[_T, _U]: ... - def elements(self) -> SequenceView[_T]: ... - def seek(self, index: int) -> None: ... - -class run_length: - @staticmethod - def encode(iterable: Iterable[_T]) -> Iterator[Tuple[_T, int]]: ... - @staticmethod - def decode(iterable: Iterable[Tuple[_T, int]]) -> Iterator[_T]: ... - -def exactly_n( - iterable: Iterable[_T], n: int, predicate: Callable[[_T], object] = ... -) -> bool: ... -def circular_shifts(iterable: Iterable[_T]) -> List[Tuple[_T, ...]]: ... -def make_decorator( - wrapping_func: Callable[..., _U], result_index: int = ... -) -> Callable[..., Callable[[Callable[..., Any]], Callable[..., _U]]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: None = ..., -) -> Dict[_U, List[_T]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None = ..., -) -> Dict[_U, List[_V]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: Callable[[List[_T]], _W] = ..., -) -> Dict[_U, _W]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[List[_V]], _W], -) -> Dict[_U, _W]: ... -def rlocate( - iterable: Iterable[_T], - pred: Callable[..., object] = ..., - window_size: Optional[int] = ..., -) -> Iterator[int]: ... -def replace( - iterable: Iterable[_T], - pred: Callable[..., object], - substitutes: Iterable[_U], - count: Optional[int] = ..., - window_size: int = ..., -) -> Iterator[Union[_T, _U]]: ... -def partitions(iterable: Iterable[_T]) -> Iterator[List[List[_T]]]: ... -def set_partitions( - iterable: Iterable[_T], k: Optional[int] = ... -) -> Iterator[List[List[_T]]]: ... - -class time_limited(Generic[_T], Iterator[_T]): - def __init__( - self, limit_seconds: float, iterable: Iterable[_T] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - -@overload -def only( - iterable: Iterable[_T], *, too_long: Optional[_Raisable] = ... -) -> Optional[_T]: ... -@overload -def only( - iterable: Iterable[_T], default: _U, too_long: Optional[_Raisable] = ... -) -> Union[_T, _U]: ... -def ichunked(iterable: Iterable[_T], n: int) -> Iterator[Iterator[_T]]: ... -def distinct_combinations( - iterable: Iterable[_T], r: int -) -> Iterator[Tuple[_T, ...]]: ... -def filter_except( - validator: Callable[[Any], object], - iterable: Iterable[_T], - *exceptions: Type[BaseException] -) -> Iterator[_T]: ... -def map_except( - function: Callable[[Any], _U], - iterable: Iterable[_T], - *exceptions: Type[BaseException] -) -> Iterator[_U]: ... -def sample( - iterable: Iterable[_T], - k: int, - weights: Optional[Iterable[float]] = ..., -) -> List[_T]: ... -def is_sorted( - iterable: Iterable[_T], - key: Optional[Callable[[_T], _U]] = ..., - reverse: bool = False, -) -> bool: ... - -class AbortThread(BaseException): - pass - -class callback_iter(Generic[_T], Iterator[_T]): - def __init__( - self, - func: Callable[..., Any], - callback_kwd: str = ..., - wait_seconds: float = ..., - ) -> None: ... - def __enter__(self) -> callback_iter[_T]: ... - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> Optional[bool]: ... - def __iter__(self) -> callback_iter[_T]: ... - def __next__(self) -> _T: ... - def _reader(self) -> Iterator[_T]: ... - @property - def done(self) -> bool: ... - @property - def result(self) -> Any: ... - -def windowed_complete( - iterable: Iterable[_T], n: int -) -> Iterator[Tuple[_T, ...]]: ... -def all_unique( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... -) -> bool: ... -def nth_product(index: int, *args: Iterable[_T]) -> Tuple[_T, ...]: ... -def nth_permutation( - iterable: Iterable[_T], r: int, index: int -) -> Tuple[_T, ...]: ... -def value_chain(*args: Union[_T, Iterable[_T]]) -> Iterable[_T]: ... -def product_index(element: Iterable[_T], *args: Iterable[_T]) -> int: ... -def combination_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def permutation_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... - -class countable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> countable[_T]: ... - def __next__(self) -> _T: ... diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py deleted file mode 100644 index 521abd7c2ca..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py +++ /dev/null @@ -1,620 +0,0 @@ -"""Imported from the recipes section of the itertools documentation. - -All functions taken from the recipes section of the itertools library docs -[1]_. -Some backward-compatible usability improvements have been made. - -.. [1] http://docs.python.org/library/itertools.html#recipes - -""" -import warnings -from collections import deque -from itertools import ( - chain, - combinations, - count, - cycle, - groupby, - islice, - repeat, - starmap, - tee, - zip_longest, -) -import operator -from random import randrange, sample, choice - -__all__ = [ - 'all_equal', - 'consume', - 'convolve', - 'dotproduct', - 'first_true', - 'flatten', - 'grouper', - 'iter_except', - 'ncycles', - 'nth', - 'nth_combination', - 'padnone', - 'pad_none', - 'pairwise', - 'partition', - 'powerset', - 'prepend', - 'quantify', - 'random_combination_with_replacement', - 'random_combination', - 'random_permutation', - 'random_product', - 'repeatfunc', - 'roundrobin', - 'tabulate', - 'tail', - 'take', - 'unique_everseen', - 'unique_justseen', -] - - -def take(n, iterable): - """Return first *n* items of the iterable as a list. - - >>> take(3, range(10)) - [0, 1, 2] - - If there are fewer than *n* items in the iterable, all of them are - returned. - - >>> take(10, range(3)) - [0, 1, 2] - - """ - return list(islice(iterable, n)) - - -def tabulate(function, start=0): - """Return an iterator over the results of ``func(start)``, - ``func(start + 1)``, ``func(start + 2)``... - - *func* should be a function that accepts one integer argument. - - If *start* is not specified it defaults to 0. It will be incremented each - time the iterator is advanced. - - >>> square = lambda x: x ** 2 - >>> iterator = tabulate(square, -3) - >>> take(4, iterator) - [9, 4, 1, 0] - - """ - return map(function, count(start)) - - -def tail(n, iterable): - """Return an iterator over the last *n* items of *iterable*. - - >>> t = tail(3, 'ABCDEFG') - >>> list(t) - ['E', 'F', 'G'] - - """ - return iter(deque(iterable, maxlen=n)) - - -def consume(iterator, n=None): - """Advance *iterable* by *n* steps. If *n* is ``None``, consume it - entirely. - - Efficiently exhausts an iterator without returning values. Defaults to - consuming the whole iterator, but an optional second argument may be - provided to limit consumption. - - >>> i = (x for x in range(10)) - >>> next(i) - 0 - >>> consume(i, 3) - >>> next(i) - 4 - >>> consume(i) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - If the iterator has fewer items remaining than the provided limit, the - whole iterator will be consumed. - - >>> i = (x for x in range(3)) - >>> consume(i, 5) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - """ - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - -def nth(iterable, n, default=None): - """Returns the nth item or a default value. - - >>> l = range(10) - >>> nth(l, 3) - 3 - >>> nth(l, 20, "zebra") - 'zebra' - - """ - return next(islice(iterable, n, None), default) - - -def all_equal(iterable): - """ - Returns ``True`` if all the elements are equal to each other. - - >>> all_equal('aaaa') - True - >>> all_equal('aaab') - False - - """ - g = groupby(iterable) - return next(g, True) and not next(g, False) - - -def quantify(iterable, pred=bool): - """Return the how many times the predicate is true. - - >>> quantify([True, False, True]) - 2 - - """ - return sum(map(pred, iterable)) - - -def pad_none(iterable): - """Returns the sequence of elements and then returns ``None`` indefinitely. - - >>> take(5, pad_none(range(3))) - [0, 1, 2, None, None] - - Useful for emulating the behavior of the built-in :func:`map` function. - - See also :func:`padded`. - - """ - return chain(iterable, repeat(None)) - - -padnone = pad_none - - -def ncycles(iterable, n): - """Returns the sequence elements *n* times - - >>> list(ncycles(["a", "b"], 3)) - ['a', 'b', 'a', 'b', 'a', 'b'] - - """ - return chain.from_iterable(repeat(tuple(iterable), n)) - - -def dotproduct(vec1, vec2): - """Returns the dot product of the two iterables. - - >>> dotproduct([10, 10], [20, 20]) - 400 - - """ - return sum(map(operator.mul, vec1, vec2)) - - -def flatten(listOfLists): - """Return an iterator flattening one level of nesting in a list of lists. - - >>> list(flatten([[0, 1], [2, 3]])) - [0, 1, 2, 3] - - See also :func:`collapse`, which can flatten multiple levels of nesting. - - """ - return chain.from_iterable(listOfLists) - - -def repeatfunc(func, times=None, *args): - """Call *func* with *args* repeatedly, returning an iterable over the - results. - - If *times* is specified, the iterable will terminate after that many - repetitions: - - >>> from operator import add - >>> times = 4 - >>> args = 3, 5 - >>> list(repeatfunc(add, times, *args)) - [8, 8, 8, 8] - - If *times* is ``None`` the iterable will not terminate: - - >>> from random import randrange - >>> times = None - >>> args = 1, 11 - >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP - [2, 4, 8, 1, 8, 4] - - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - -def _pairwise(iterable): - """Returns an iterator of paired items, overlapping, from the original - - >>> take(4, pairwise(count())) - [(0, 1), (1, 2), (2, 3), (3, 4)] - - On Python 3.10 and above, this is an alias for :func:`itertools.pairwise`. - - """ - a, b = tee(iterable) - next(b, None) - yield from zip(a, b) - - -try: - from itertools import pairwise as itertools_pairwise -except ImportError: - pairwise = _pairwise -else: - - def pairwise(iterable): - yield from itertools_pairwise(iterable) - - pairwise.__doc__ = _pairwise.__doc__ - - -def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks. - - >>> list(grouper('ABCDEFG', 3, 'x')) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] - - """ - if isinstance(iterable, int): - warnings.warn( - "grouper expects iterable as first parameter", DeprecationWarning - ) - n, iterable = iterable, n - args = [iter(iterable)] * n - return zip_longest(fillvalue=fillvalue, *args) - - -def roundrobin(*iterables): - """Yields an item from each iterable, alternating between them. - - >>> list(roundrobin('ABC', 'D', 'EF')) - ['A', 'D', 'E', 'B', 'F', 'C'] - - This function produces the same output as :func:`interleave_longest`, but - may perform better for some inputs (in particular when the number of - iterables is small). - - """ - # Recipe credited to George Sakkis - pending = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while pending: - try: - for next in nexts: - yield next() - except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) - - -def partition(pred, iterable): - """ - Returns a 2-tuple of iterables derived from the input iterable. - The first yields the items that have ``pred(item) == False``. - The second yields the items that have ``pred(item) == True``. - - >>> is_odd = lambda x: x % 2 != 0 - >>> iterable = range(10) - >>> even_items, odd_items = partition(is_odd, iterable) - >>> list(even_items), list(odd_items) - ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) - - If *pred* is None, :func:`bool` is used. - - >>> iterable = [0, 1, False, True, '', ' '] - >>> false_items, true_items = partition(None, iterable) - >>> list(false_items), list(true_items) - ([0, False, ''], [1, True, ' ']) - - """ - if pred is None: - pred = bool - - evaluations = ((pred(x), x) for x in iterable) - t1, t2 = tee(evaluations) - return ( - (x for (cond, x) in t1 if not cond), - (x for (cond, x) in t2 if cond), - ) - - -def powerset(iterable): - """Yields all possible subsets of the iterable. - - >>> list(powerset([1, 2, 3])) - [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - - :func:`powerset` will operate on iterables that aren't :class:`set` - instances, so repeated elements in the input will produce repeated elements - in the output. Use :func:`unique_everseen` on the input to avoid generating - duplicates: - - >>> seq = [1, 1, 0] - >>> list(powerset(seq)) - [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] - >>> from more_itertools import unique_everseen - >>> list(powerset(unique_everseen(seq))) - [(), (1,), (0,), (1, 0)] - - """ - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) - - -def unique_everseen(iterable, key=None): - """ - Yield unique elements, preserving order. - - >>> list(unique_everseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'D'] - - Sequences with a mix of hashable and unhashable items can be used. - The function will be slower (i.e., `O(n^2)`) for unhashable items. - - Remember that ``list`` objects are unhashable - you can use the *key* - parameter to transform the list to a tuple (which is hashable) to - avoid a slowdown. - - >>> iterable = ([1, 2], [2, 3], [1, 2]) - >>> list(unique_everseen(iterable)) # Slow - [[1, 2], [2, 3]] - >>> list(unique_everseen(iterable, key=tuple)) # Faster - [[1, 2], [2, 3]] - - Similary, you may want to convert unhashable ``set`` objects with - ``key=frozenset``. For ``dict`` objects, - ``key=lambda x: frozenset(x.items())`` can be used. - - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seenset: - seenset_add(k) - yield element - except TypeError: - if k not in seenlist: - seenlist_add(k) - yield element - - -def unique_justseen(iterable, key=None): - """Yields elements in order, ignoring serial duplicates - - >>> list(unique_justseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'A', 'D'] - - """ - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - -def iter_except(func, exception, first=None): - """Yields results from a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel - to end the loop. - - >>> l = [0, 1, 2] - >>> list(iter_except(l.pop, IndexError)) - [2, 1, 0] - - """ - try: - if first is not None: - yield first() - while 1: - yield func() - except exception: - pass - - -def first_true(iterable, default=None, pred=None): - """ - Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item for which - ``pred(item) == True`` . - - >>> first_true(range(10)) - 1 - >>> first_true(range(10), pred=lambda x: x > 5) - 6 - >>> first_true(range(10), default='missing', pred=lambda x: x > 9) - 'missing' - - """ - return next(filter(pred, iterable), default) - - -def random_product(*args, repeat=1): - """Draw an item at random from each of the input iterables. - - >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP - ('c', 3, 'Z') - - If *repeat* is provided as a keyword argument, that many items will be - drawn from each iterable. - - >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP - ('a', 2, 'd', 3) - - This equivalent to taking a random selection from - ``itertools.product(*args, **kwarg)``. - - """ - pools = [tuple(pool) for pool in args] * repeat - return tuple(choice(pool) for pool in pools) - - -def random_permutation(iterable, r=None): - """Return a random *r* length permutation of the elements in *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length of - *iterable*. - - >>> random_permutation(range(5)) # doctest:+SKIP - (3, 4, 0, 1, 2) - - This equivalent to taking a random selection from - ``itertools.permutations(iterable, r)``. - - """ - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(sample(pool, r)) - - -def random_combination(iterable, r): - """Return a random *r* length subsequence of the elements in *iterable*. - - >>> random_combination(range(5), 3) # doctest:+SKIP - (2, 3, 4) - - This equivalent to taking a random selection from - ``itertools.combinations(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(sample(range(n), r)) - return tuple(pool[i] for i in indices) - - -def random_combination_with_replacement(iterable, r): - """Return a random *r* length subsequence of elements in *iterable*, - allowing individual elements to be repeated. - - >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP - (0, 0, 1, 2, 2) - - This equivalent to taking a random selection from - ``itertools.combinations_with_replacement(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(randrange(n) for i in range(r)) - return tuple(pool[i] for i in indices) - - -def nth_combination(iterable, r, index): - """Equivalent to ``list(combinations(iterable, r))[index]``. - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`nth_combination` computes the subsequence at - sort position *index* directly, without computing the previous - subsequences. - - >>> nth_combination(range(5), 3, 5) - (0, 3, 4) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = 1 - k = min(r, n - r) - for i in range(1, k + 1): - c = c * (n - k + i) // i - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - while r: - c, n, r = c * r // n, n - 1, r - 1 - while index >= c: - index -= c - c, n = c * (n - r) // n, n - 1 - result.append(pool[-1 - n]) - - return tuple(result) - - -def prepend(value, iterator): - """Yield *value*, followed by the elements in *iterator*. - - >>> value = '0' - >>> iterator = ['1', '2', '3'] - >>> list(prepend(value, iterator)) - ['0', '1', '2', '3'] - - To prepend multiple values, see :func:`itertools.chain` - or :func:`value_chain`. - - """ - return chain([value], iterator) - - -def convolve(signal, kernel): - """Convolve the iterable *signal* with the iterable *kernel*. - - >>> signal = (1, 2, 3, 4, 5) - >>> kernel = [3, 2, 1] - >>> list(convolve(signal, kernel)) - [3, 8, 14, 20, 26, 14, 5] - - Note: the input arguments are not interchangeable, as the *kernel* - is immediately consumed and stored. - - """ - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n - 1)): - window.append(x) - yield sum(map(operator.mul, kernel, window)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi deleted file mode 100644 index 5e39d963907..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi +++ /dev/null @@ -1,103 +0,0 @@ -"""Stubs for more_itertools.recipes""" -from typing import ( - Any, - Callable, - Iterable, - Iterator, - List, - Optional, - Tuple, - TypeVar, - Union, -) -from typing_extensions import overload, Type - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - -def take(n: int, iterable: Iterable[_T]) -> List[_T]: ... -def tabulate( - function: Callable[[int], _T], start: int = ... -) -> Iterator[_T]: ... -def tail(n: int, iterable: Iterable[_T]) -> Iterator[_T]: ... -def consume(iterator: Iterable[object], n: Optional[int] = ...) -> None: ... -@overload -def nth(iterable: Iterable[_T], n: int) -> Optional[_T]: ... -@overload -def nth(iterable: Iterable[_T], n: int, default: _U) -> Union[_T, _U]: ... -def all_equal(iterable: Iterable[object]) -> bool: ... -def quantify( - iterable: Iterable[_T], pred: Callable[[_T], bool] = ... -) -> int: ... -def pad_none(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... -def padnone(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... -def ncycles(iterable: Iterable[_T], n: int) -> Iterator[_T]: ... -def dotproduct(vec1: Iterable[object], vec2: Iterable[object]) -> object: ... -def flatten(listOfLists: Iterable[Iterable[_T]]) -> Iterator[_T]: ... -def repeatfunc( - func: Callable[..., _U], times: Optional[int] = ..., *args: Any -) -> Iterator[_U]: ... -def pairwise(iterable: Iterable[_T]) -> Iterator[Tuple[_T, _T]]: ... -@overload -def grouper( - iterable: Iterable[_T], n: int -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def grouper( - iterable: Iterable[_T], n: int, fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -@overload -def grouper( # Deprecated interface - iterable: int, n: Iterable[_T] -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def grouper( # Deprecated interface - iterable: int, n: Iterable[_T], fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def roundrobin(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def partition( - pred: Optional[Callable[[_T], object]], iterable: Iterable[_T] -) -> Tuple[Iterator[_T], Iterator[_T]]: ... -def powerset(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... -) -> Iterator[_T]: ... -def unique_justseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], object]] = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], exception: Type[BaseException], first: None = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException], - first: Callable[[], _U], -) -> Iterator[Union[_T, _U]]: ... -@overload -def first_true( - iterable: Iterable[_T], *, pred: Optional[Callable[[_T], object]] = ... -) -> Optional[_T]: ... -@overload -def first_true( - iterable: Iterable[_T], - default: _U, - pred: Optional[Callable[[_T], object]] = ..., -) -> Union[_T, _U]: ... -def random_product( - *args: Iterable[_T], repeat: int = ... -) -> Tuple[_T, ...]: ... -def random_permutation( - iterable: Iterable[_T], r: Optional[int] = ... -) -> Tuple[_T, ...]: ... -def random_combination(iterable: Iterable[_T], r: int) -> Tuple[_T, ...]: ... -def random_combination_with_replacement( - iterable: Iterable[_T], r: int -) -> Tuple[_T, ...]: ... -def nth_combination( - iterable: Iterable[_T], r: int, index: int -) -> Tuple[_T, ...]: ... -def prepend(value: _T, iterator: Iterable[_U]) -> Iterator[Union[_T, _U]]: ... -def convolve(signal: Iterable[_T], kernel: Iterable[_T]) -> Iterator[_T]: ... diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/ordered_set.py b/contrib/python/setuptools/py3/setuptools/_vendor/ordered_set.py deleted file mode 100644 index 14876000de8..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/ordered_set.py +++ /dev/null @@ -1,488 +0,0 @@ -""" -An OrderedSet is a custom MutableSet that remembers its order, so that every -entry has an index that can be looked up. - -Based on a recipe originally posted to ActiveState Recipes by Raymond Hettiger, -and released under the MIT license. -""" -import itertools as it -from collections import deque - -try: - # Python 3 - from collections.abc import MutableSet, Sequence -except ImportError: - # Python 2.7 - from collections import MutableSet, Sequence - -SLICE_ALL = slice(None) -__version__ = "3.1" - - -def is_iterable(obj): - """ - Are we being asked to look up a list of things, instead of a single thing? - We check for the `__iter__` attribute so that this can cover types that - don't have to be known by this module, such as NumPy arrays. - - Strings, however, should be considered as atomic values to look up, not - iterables. The same goes for tuples, since they are immutable and therefore - valid entries. - - We don't need to check for the Python 2 `unicode` type, because it doesn't - have an `__iter__` attribute anyway. - """ - return ( - hasattr(obj, "__iter__") - and not isinstance(obj, str) - and not isinstance(obj, tuple) - ) - - -class OrderedSet(MutableSet, Sequence): - """ - An OrderedSet is a custom MutableSet that remembers its order, so that - every entry has an index that can be looked up. - - Example: - >>> OrderedSet([1, 1, 2, 3, 2]) - OrderedSet([1, 2, 3]) - """ - - def __init__(self, iterable=None): - self.items = [] - self.map = {} - if iterable is not None: - self |= iterable - - def __len__(self): - """ - Returns the number of unique elements in the ordered set - - Example: - >>> len(OrderedSet([])) - 0 - >>> len(OrderedSet([1, 2])) - 2 - """ - return len(self.items) - - def __getitem__(self, index): - """ - Get the item at a given index. - - If `index` is a slice, you will get back that slice of items, as a - new OrderedSet. - - If `index` is a list or a similar iterable, you'll get a list of - items corresponding to those indices. This is similar to NumPy's - "fancy indexing". The result is not an OrderedSet because you may ask - for duplicate indices, and the number of elements returned should be - the number of elements asked for. - - Example: - >>> oset = OrderedSet([1, 2, 3]) - >>> oset[1] - 2 - """ - if isinstance(index, slice) and index == SLICE_ALL: - return self.copy() - elif is_iterable(index): - return [self.items[i] for i in index] - elif hasattr(index, "__index__") or isinstance(index, slice): - result = self.items[index] - if isinstance(result, list): - return self.__class__(result) - else: - return result - else: - raise TypeError("Don't know how to index an OrderedSet by %r" % index) - - def copy(self): - """ - Return a shallow copy of this object. - - Example: - >>> this = OrderedSet([1, 2, 3]) - >>> other = this.copy() - >>> this == other - True - >>> this is other - False - """ - return self.__class__(self) - - def __getstate__(self): - if len(self) == 0: - # The state can't be an empty list. - # We need to return a truthy value, or else __setstate__ won't be run. - # - # This could have been done more gracefully by always putting the state - # in a tuple, but this way is backwards- and forwards- compatible with - # previous versions of OrderedSet. - return (None,) - else: - return list(self) - - def __setstate__(self, state): - if state == (None,): - self.__init__([]) - else: - self.__init__(state) - - def __contains__(self, key): - """ - Test if the item is in this ordered set - - Example: - >>> 1 in OrderedSet([1, 3, 2]) - True - >>> 5 in OrderedSet([1, 3, 2]) - False - """ - return key in self.map - - def add(self, key): - """ - Add `key` as an item to this OrderedSet, then return its index. - - If `key` is already in the OrderedSet, return the index it already - had. - - Example: - >>> oset = OrderedSet() - >>> oset.append(3) - 0 - >>> print(oset) - OrderedSet([3]) - """ - if key not in self.map: - self.map[key] = len(self.items) - self.items.append(key) - return self.map[key] - - append = add - - def update(self, sequence): - """ - Update the set with the given iterable sequence, then return the index - of the last element inserted. - - Example: - >>> oset = OrderedSet([1, 2, 3]) - >>> oset.update([3, 1, 5, 1, 4]) - 4 - >>> print(oset) - OrderedSet([1, 2, 3, 5, 4]) - """ - item_index = None - try: - for item in sequence: - item_index = self.add(item) - except TypeError: - raise ValueError( - "Argument needs to be an iterable, got %s" % type(sequence) - ) - return item_index - - def index(self, key): - """ - Get the index of a given entry, raising an IndexError if it's not - present. - - `key` can be an iterable of entries that is not a string, in which case - this returns a list of indices. - - Example: - >>> oset = OrderedSet([1, 2, 3]) - >>> oset.index(2) - 1 - """ - if is_iterable(key): - return [self.index(subkey) for subkey in key] - return self.map[key] - - # Provide some compatibility with pd.Index - get_loc = index - get_indexer = index - - def pop(self): - """ - Remove and return the last element from the set. - - Raises KeyError if the set is empty. - - Example: - >>> oset = OrderedSet([1, 2, 3]) - >>> oset.pop() - 3 - """ - if not self.items: - raise KeyError("Set is empty") - - elem = self.items[-1] - del self.items[-1] - del self.map[elem] - return elem - - def discard(self, key): - """ - Remove an element. Do not raise an exception if absent. - - The MutableSet mixin uses this to implement the .remove() method, which - *does* raise an error when asked to remove a non-existent item. - - Example: - >>> oset = OrderedSet([1, 2, 3]) - >>> oset.discard(2) - >>> print(oset) - OrderedSet([1, 3]) - >>> oset.discard(2) - >>> print(oset) - OrderedSet([1, 3]) - """ - if key in self: - i = self.map[key] - del self.items[i] - del self.map[key] - for k, v in self.map.items(): - if v >= i: - self.map[k] = v - 1 - - def clear(self): - """ - Remove all items from this OrderedSet. - """ - del self.items[:] - self.map.clear() - - def __iter__(self): - """ - Example: - >>> list(iter(OrderedSet([1, 2, 3]))) - [1, 2, 3] - """ - return iter(self.items) - - def __reversed__(self): - """ - Example: - >>> list(reversed(OrderedSet([1, 2, 3]))) - [3, 2, 1] - """ - return reversed(self.items) - - def __repr__(self): - if not self: - return "%s()" % (self.__class__.__name__,) - return "%s(%r)" % (self.__class__.__name__, list(self)) - - def __eq__(self, other): - """ - Returns true if the containers have the same items. If `other` is a - Sequence, then order is checked, otherwise it is ignored. - - Example: - >>> oset = OrderedSet([1, 3, 2]) - >>> oset == [1, 3, 2] - True - >>> oset == [1, 2, 3] - False - >>> oset == [2, 3] - False - >>> oset == OrderedSet([3, 2, 1]) - False - """ - # In Python 2 deque is not a Sequence, so treat it as one for - # consistent behavior with Python 3. - if isinstance(other, (Sequence, deque)): - # Check that this OrderedSet contains the same elements, in the - # same order, as the other object. - return list(self) == list(other) - try: - other_as_set = set(other) - except TypeError: - # If `other` can't be converted into a set, it's not equal. - return False - else: - return set(self) == other_as_set - - def union(self, *sets): - """ - Combines all unique items. - Each items order is defined by its first appearance. - - Example: - >>> oset = OrderedSet.union(OrderedSet([3, 1, 4, 1, 5]), [1, 3], [2, 0]) - >>> print(oset) - OrderedSet([3, 1, 4, 5, 2, 0]) - >>> oset.union([8, 9]) - OrderedSet([3, 1, 4, 5, 2, 0, 8, 9]) - >>> oset | {10} - OrderedSet([3, 1, 4, 5, 2, 0, 10]) - """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet - containers = map(list, it.chain([self], sets)) - items = it.chain.from_iterable(containers) - return cls(items) - - def __and__(self, other): - # the parent implementation of this is backwards - return self.intersection(other) - - def intersection(self, *sets): - """ - Returns elements in common between all sets. Order is defined only - by the first set. - - Example: - >>> oset = OrderedSet.intersection(OrderedSet([0, 1, 2, 3]), [1, 2, 3]) - >>> print(oset) - OrderedSet([1, 2, 3]) - >>> oset.intersection([2, 4, 5], [1, 2, 3, 4]) - OrderedSet([2]) - >>> oset.intersection() - OrderedSet([1, 2, 3]) - """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet - if sets: - common = set.intersection(*map(set, sets)) - items = (item for item in self if item in common) - else: - items = self - return cls(items) - - def difference(self, *sets): - """ - Returns all elements that are in this set but not the others. - - Example: - >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2])) - OrderedSet([1, 3]) - >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2]), OrderedSet([3])) - OrderedSet([1]) - >>> OrderedSet([1, 2, 3]) - OrderedSet([2]) - OrderedSet([1, 3]) - >>> OrderedSet([1, 2, 3]).difference() - OrderedSet([1, 2, 3]) - """ - cls = self.__class__ - if sets: - other = set.union(*map(set, sets)) - items = (item for item in self if item not in other) - else: - items = self - return cls(items) - - def issubset(self, other): - """ - Report whether another set contains this set. - - Example: - >>> OrderedSet([1, 2, 3]).issubset({1, 2}) - False - >>> OrderedSet([1, 2, 3]).issubset({1, 2, 3, 4}) - True - >>> OrderedSet([1, 2, 3]).issubset({1, 4, 3, 5}) - False - """ - if len(self) > len(other): # Fast check for obvious cases - return False - return all(item in other for item in self) - - def issuperset(self, other): - """ - Report whether this set contains another set. - - Example: - >>> OrderedSet([1, 2]).issuperset([1, 2, 3]) - False - >>> OrderedSet([1, 2, 3, 4]).issuperset({1, 2, 3}) - True - >>> OrderedSet([1, 4, 3, 5]).issuperset({1, 2, 3}) - False - """ - if len(self) < len(other): # Fast check for obvious cases - return False - return all(item in self for item in other) - - def symmetric_difference(self, other): - """ - Return the symmetric difference of two OrderedSets as a new set. - That is, the new set will contain all elements that are in exactly - one of the sets. - - Their order will be preserved, with elements from `self` preceding - elements from `other`. - - Example: - >>> this = OrderedSet([1, 4, 3, 5, 7]) - >>> other = OrderedSet([9, 7, 1, 3, 2]) - >>> this.symmetric_difference(other) - OrderedSet([4, 5, 9, 2]) - """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet - diff1 = cls(self).difference(other) - diff2 = cls(other).difference(self) - return diff1.union(diff2) - - def _update_items(self, items): - """ - Replace the 'items' list of this OrderedSet with a new one, updating - self.map accordingly. - """ - self.items = items - self.map = {item: idx for (idx, item) in enumerate(items)} - - def difference_update(self, *sets): - """ - Update this OrderedSet to remove items from one or more other sets. - - Example: - >>> this = OrderedSet([1, 2, 3]) - >>> this.difference_update(OrderedSet([2, 4])) - >>> print(this) - OrderedSet([1, 3]) - - >>> this = OrderedSet([1, 2, 3, 4, 5]) - >>> this.difference_update(OrderedSet([2, 4]), OrderedSet([1, 4, 6])) - >>> print(this) - OrderedSet([3, 5]) - """ - items_to_remove = set() - for other in sets: - items_to_remove |= set(other) - self._update_items([item for item in self.items if item not in items_to_remove]) - - def intersection_update(self, other): - """ - Update this OrderedSet to keep only items in another set, preserving - their order in this set. - - Example: - >>> this = OrderedSet([1, 4, 3, 5, 7]) - >>> other = OrderedSet([9, 7, 1, 3, 2]) - >>> this.intersection_update(other) - >>> print(this) - OrderedSet([1, 3, 7]) - """ - other = set(other) - self._update_items([item for item in self.items if item in other]) - - def symmetric_difference_update(self, other): - """ - Update this OrderedSet to remove items from another set, then - add items from the other set that were not present in this set. - - Example: - >>> this = OrderedSet([1, 4, 3, 5, 7]) - >>> other = OrderedSet([9, 7, 1, 3, 2]) - >>> this.symmetric_difference_update(other) - >>> print(this) - OrderedSet([4, 5, 9, 2]) - """ - items_to_add = [item for item in other if item not in self] - items_to_remove = set(other) - self._update_items( - [item for item in self.items if item not in items_to_remove] + items_to_add - ) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py deleted file mode 100644 index e7c0aa12ca9..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -__title__ = "packaging" -__summary__ = "Core utilities for Python packages" -__uri__ = "https://github.com/pypa/packaging" - -__version__ = "24.0" - -__author__ = "Donald Stufft and individual contributors" -__email__ = "[email protected]" - -__license__ = "BSD-2-Clause or Apache-2.0" -__copyright__ = "2014 %s" % __author__ diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py deleted file mode 100644 index 6fb19b30bb5..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -ELF file parser. - -This provides a class ``ELFFile`` that parses an ELF executable in a similar -interface to ``ZipFile``. Only the read interface is implemented. - -Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca -ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html -""" - -import enum -import os -import struct -from typing import IO, Optional, Tuple - - -class ELFInvalid(ValueError): - pass - - -class EIClass(enum.IntEnum): - C32 = 1 - C64 = 2 - - -class EIData(enum.IntEnum): - Lsb = 1 - Msb = 2 - - -class EMachine(enum.IntEnum): - I386 = 3 - S390 = 22 - Arm = 40 - X8664 = 62 - AArc64 = 183 - - -class ELFFile: - """ - Representation of an ELF executable. - """ - - def __init__(self, f: IO[bytes]) -> None: - self._f = f - - try: - ident = self._read("16B") - except struct.error: - raise ELFInvalid("unable to parse identification") - magic = bytes(ident[:4]) - if magic != b"\x7fELF": - raise ELFInvalid(f"invalid magic: {magic!r}") - - self.capacity = ident[4] # Format for program header (bitness). - self.encoding = ident[5] # Data structure encoding (endianness). - - try: - # e_fmt: Format for program header. - # p_fmt: Format for section header. - # p_idx: Indexes to find p_type, p_offset, and p_filesz. - e_fmt, self._p_fmt, self._p_idx = { - (1, 1): ("<HHIIIIIHHH", "<IIIIIIII", (0, 1, 4)), # 32-bit LSB. - (1, 2): (">HHIIIIIHHH", ">IIIIIIII", (0, 1, 4)), # 32-bit MSB. - (2, 1): ("<HHIQQQIHHH", "<IIQQQQQQ", (0, 2, 5)), # 64-bit LSB. - (2, 2): (">HHIQQQIHHH", ">IIQQQQQQ", (0, 2, 5)), # 64-bit MSB. - }[(self.capacity, self.encoding)] - except KeyError: - raise ELFInvalid( - f"unrecognized capacity ({self.capacity}) or " - f"encoding ({self.encoding})" - ) - - try: - ( - _, - self.machine, # Architecture type. - _, - _, - self._e_phoff, # Offset of program header. - _, - self.flags, # Processor-specific flags. - _, - self._e_phentsize, # Size of section. - self._e_phnum, # Number of sections. - ) = self._read(e_fmt) - except struct.error as e: - raise ELFInvalid("unable to parse machine and section information") from e - - def _read(self, fmt: str) -> Tuple[int, ...]: - return struct.unpack(fmt, self._f.read(struct.calcsize(fmt))) - - @property - def interpreter(self) -> Optional[str]: - """ - The path recorded in the ``PT_INTERP`` section header. - """ - for index in range(self._e_phnum): - self._f.seek(self._e_phoff + self._e_phentsize * index) - try: - data = self._read(self._p_fmt) - except struct.error: - continue - if data[self._p_idx[0]] != 3: # Not PT_INTERP. - continue - self._f.seek(data[self._p_idx[1]]) - return os.fsdecode(self._f.read(data[self._p_idx[2]])).strip("\0") - return None diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_manylinux.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_manylinux.py deleted file mode 100644 index ad62505f3ff..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_manylinux.py +++ /dev/null @@ -1,260 +0,0 @@ -import collections -import contextlib -import functools -import os -import re -import sys -import warnings -from typing import Dict, Generator, Iterator, NamedTuple, Optional, Sequence, Tuple - -from ._elffile import EIClass, EIData, ELFFile, EMachine - -EF_ARM_ABIMASK = 0xFF000000 -EF_ARM_ABI_VER5 = 0x05000000 -EF_ARM_ABI_FLOAT_HARD = 0x00000400 - - -# `os.PathLike` not a generic type until Python 3.9, so sticking with `str` -# as the type for `path` until then. -def _parse_elf(path: str) -> Generator[Optional[ELFFile], None, None]: - try: - with open(path, "rb") as f: - yield ELFFile(f) - except (OSError, TypeError, ValueError): - yield None - - -def _is_linux_armhf(executable: str) -> bool: - # hard-float ABI can be detected from the ELF header of the running - # process - # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.Arm - and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5 - and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD - ) - - -def _is_linux_i686(executable: str) -> bool: - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.I386 - ) - - -def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: - if "armv7l" in archs: - return _is_linux_armhf(executable) - if "i686" in archs: - return _is_linux_i686(executable) - allowed_archs = { - "x86_64", - "aarch64", - "ppc64", - "ppc64le", - "s390x", - "loongarch64", - "riscv64", - } - return any(arch in allowed_archs for arch in archs) - - -# If glibc ever changes its major version, we need to know what the last -# minor version was, so we can build the complete list of all versions. -# For now, guess what the highest minor version might be, assume it will -# be 50 for testing. Once this actually happens, update the dictionary -# with the actual value. -_LAST_GLIBC_MINOR: Dict[int, int] = collections.defaultdict(lambda: 50) - - -class _GLibCVersion(NamedTuple): - major: int - minor: int - - -def _glibc_version_string_confstr() -> Optional[str]: - """ - Primary implementation of glibc_version_string using os.confstr. - """ - # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely - # to be broken or missing. This strategy is used in the standard library - # platform module. - # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183 - try: - # Should be a string like "glibc 2.17". - version_string: Optional[str] = os.confstr("CS_GNU_LIBC_VERSION") - assert version_string is not None - _, version = version_string.rsplit() - except (AssertionError, AttributeError, OSError, ValueError): - # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... - return None - return version - - -def _glibc_version_string_ctypes() -> Optional[str]: - """ - Fallback implementation of glibc_version_string using ctypes. - """ - try: - import ctypes - except ImportError: - return None - - # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen - # manpage says, "If filename is NULL, then the returned handle is for the - # main program". This way we can let the linker do the work to figure out - # which libc our process is actually using. - # - # We must also handle the special case where the executable is not a - # dynamically linked executable. This can occur when using musl libc, - # for example. In this situation, dlopen() will error, leading to an - # OSError. Interestingly, at least in the case of musl, there is no - # errno set on the OSError. The single string argument used to construct - # OSError comes from libc itself and is therefore not portable to - # hard code here. In any case, failure to call dlopen() means we - # can proceed, so we bail on our attempt. - try: - process_namespace = ctypes.CDLL(None) - except OSError: - return None - - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return None - - # Call gnu_get_libc_version, which returns a string like "2.5" - gnu_get_libc_version.restype = ctypes.c_char_p - version_str: str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") - - return version_str - - -def _glibc_version_string() -> Optional[str]: - """Returns glibc version string, or None if not using glibc.""" - return _glibc_version_string_confstr() or _glibc_version_string_ctypes() - - -def _parse_glibc_version(version_str: str) -> Tuple[int, int]: - """Parse glibc version. - - We use a regexp instead of str.split because we want to discard any - random junk that might come after the minor version -- this might happen - in patched/forked versions of glibc (e.g. Linaro's version of glibc - uses version strings like "2.20-2014.11"). See gh-3588. - """ - m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) - if not m: - warnings.warn( - f"Expected glibc version with 2 components major.minor," - f" got: {version_str}", - RuntimeWarning, - ) - return -1, -1 - return int(m.group("major")), int(m.group("minor")) - - [email protected]_cache() -def _get_glibc_version() -> Tuple[int, int]: - version_str = _glibc_version_string() - if version_str is None: - return (-1, -1) - return _parse_glibc_version(version_str) - - -# From PEP 513, PEP 600 -def _is_compatible(arch: str, version: _GLibCVersion) -> bool: - sys_glibc = _get_glibc_version() - if sys_glibc < version: - return False - # Check for presence of _manylinux module. - try: - import _manylinux - except ImportError: - return True - if hasattr(_manylinux, "manylinux_compatible"): - result = _manylinux.manylinux_compatible(version[0], version[1], arch) - if result is not None: - return bool(result) - return True - if version == _GLibCVersion(2, 5): - if hasattr(_manylinux, "manylinux1_compatible"): - return bool(_manylinux.manylinux1_compatible) - if version == _GLibCVersion(2, 12): - if hasattr(_manylinux, "manylinux2010_compatible"): - return bool(_manylinux.manylinux2010_compatible) - if version == _GLibCVersion(2, 17): - if hasattr(_manylinux, "manylinux2014_compatible"): - return bool(_manylinux.manylinux2014_compatible) - return True - - -_LEGACY_MANYLINUX_MAP = { - # CentOS 7 w/ glibc 2.17 (PEP 599) - (2, 17): "manylinux2014", - # CentOS 6 w/ glibc 2.12 (PEP 571) - (2, 12): "manylinux2010", - # CentOS 5 w/ glibc 2.5 (PEP 513) - (2, 5): "manylinux1", -} - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate manylinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be manylinux-compatible. - - :returns: An iterator of compatible manylinux tags. - """ - if not _have_compatible_abi(sys.executable, archs): - return - # Oldest glibc to be supported regardless of architecture is (2, 17). - too_old_glibc2 = _GLibCVersion(2, 16) - if set(archs) & {"x86_64", "i686"}: - # On x86/i686 also oldest glibc to be supported is (2, 5). - too_old_glibc2 = _GLibCVersion(2, 4) - current_glibc = _GLibCVersion(*_get_glibc_version()) - glibc_max_list = [current_glibc] - # We can assume compatibility across glibc major versions. - # https://sourceware.org/bugzilla/show_bug.cgi?id=24636 - # - # Build a list of maximum glibc versions so that we can - # output the canonical list of all glibc from current_glibc - # down to too_old_glibc2, including all intermediary versions. - for glibc_major in range(current_glibc.major - 1, 1, -1): - glibc_minor = _LAST_GLIBC_MINOR[glibc_major] - glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor)) - for arch in archs: - for glibc_max in glibc_max_list: - if glibc_max.major == too_old_glibc2.major: - min_minor = too_old_glibc2.minor - else: - # For other glibc major versions oldest supported is (x, 0). - min_minor = -1 - for glibc_minor in range(glibc_max.minor, min_minor, -1): - glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) - tag = "manylinux_{}_{}".format(*glibc_version) - if _is_compatible(arch, glibc_version): - yield f"{tag}_{arch}" - # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. - if glibc_version in _LEGACY_MANYLINUX_MAP: - legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] - if _is_compatible(arch, glibc_version): - yield f"{legacy_tag}_{arch}" diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py deleted file mode 100644 index 86419df9d70..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py +++ /dev/null @@ -1,83 +0,0 @@ -"""PEP 656 support. - -This module implements logic to detect if the currently running Python is -linked against musl, and what musl version is used. -""" - -import functools -import re -import subprocess -import sys -from typing import Iterator, NamedTuple, Optional, Sequence - -from ._elffile import ELFFile - - -class _MuslVersion(NamedTuple): - major: int - minor: int - - -def _parse_musl_version(output: str) -> Optional[_MuslVersion]: - lines = [n for n in (n.strip() for n in output.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) - - [email protected]_cache() -def _get_musl_version(executable: str) -> Optional[_MuslVersion]: - """Detect currently-running musl runtime version. - - This is done by checking the specified executable's dynamic linking - information, and invoking the loader to parse its output for a version - string. If the loader is musl, the output would be something like:: - - musl libc (x86_64) - Version 1.2.2 - Dynamic Program Loader - """ - try: - with open(executable, "rb") as f: - ld = ELFFile(f).interpreter - except (OSError, TypeError, ValueError): - return None - if ld is None or "musl" not in ld: - return None - proc = subprocess.run([ld], stderr=subprocess.PIPE, text=True) - return _parse_musl_version(proc.stderr) - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate musllinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be musllinux-compatible. - - :returns: An iterator of compatible musllinux tags. - """ - sys_musl = _get_musl_version(sys.executable) - if sys_musl is None: # Python not dynamically linked against musl. - return - for arch in archs: - for minor in range(sys_musl.minor, -1, -1): - yield f"musllinux_{sys_musl.major}_{minor}_{arch}" - - -if __name__ == "__main__": # pragma: no cover - import sysconfig - - plat = sysconfig.get_platform() - assert plat.startswith("linux-"), "not linux" - - print("plat:", plat) - print("musl:", _get_musl_version(sys.executable)) - print("tags:", end=" ") - for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])): - print(t, end="\n ") diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_parser.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_parser.py deleted file mode 100644 index 684df75457c..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_parser.py +++ /dev/null @@ -1,356 +0,0 @@ -"""Handwritten parser of dependency specifiers. - -The docstring for each __parse_* function contains ENBF-inspired grammar representing -the implementation. -""" - -import ast -from typing import Any, List, NamedTuple, Optional, Tuple, Union - -from ._tokenizer import DEFAULT_RULES, Tokenizer - - -class Node: - def __init__(self, value: str) -> None: - self.value = value - - def __str__(self) -> str: - return self.value - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" - - def serialize(self) -> str: - raise NotImplementedError - - -class Variable(Node): - def serialize(self) -> str: - return str(self) - - -class Value(Node): - def serialize(self) -> str: - return f'"{self}"' - - -class Op(Node): - def serialize(self) -> str: - return str(self) - - -MarkerVar = Union[Variable, Value] -MarkerItem = Tuple[MarkerVar, Op, MarkerVar] -# MarkerAtom = Union[MarkerItem, List["MarkerAtom"]] -# MarkerList = List[Union["MarkerList", MarkerAtom, str]] -# mypy does not support recursive type definition -# https://github.com/python/mypy/issues/731 -MarkerAtom = Any -MarkerList = List[Any] - - -class ParsedRequirement(NamedTuple): - name: str - url: str - extras: List[str] - specifier: str - marker: Optional[MarkerList] - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for dependency specifier -# -------------------------------------------------------------------------------------- -def parse_requirement(source: str) -> ParsedRequirement: - return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: - """ - requirement = WS? IDENTIFIER WS? extras WS? requirement_details - """ - tokenizer.consume("WS") - - name_token = tokenizer.expect( - "IDENTIFIER", expected="package name at the start of dependency specifier" - ) - name = name_token.text - tokenizer.consume("WS") - - extras = _parse_extras(tokenizer) - tokenizer.consume("WS") - - url, specifier, marker = _parse_requirement_details(tokenizer) - tokenizer.expect("END", expected="end of dependency specifier") - - return ParsedRequirement(name, url, extras, specifier, marker) - - -def _parse_requirement_details( - tokenizer: Tokenizer, -) -> Tuple[str, str, Optional[MarkerList]]: - """ - requirement_details = AT URL (WS requirement_marker?)? - | specifier WS? (requirement_marker)? - """ - - specifier = "" - url = "" - marker = None - - if tokenizer.check("AT"): - tokenizer.read() - tokenizer.consume("WS") - - url_start = tokenizer.position - url = tokenizer.expect("URL", expected="URL after @").text - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - tokenizer.expect("WS", expected="whitespace after URL") - - # The input might end after whitespace. - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, span_start=url_start, after="URL and whitespace" - ) - else: - specifier_start = tokenizer.position - specifier = _parse_specifier(tokenizer) - tokenizer.consume("WS") - - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, - span_start=specifier_start, - after=( - "version specifier" - if specifier - else "name and no valid version specifier" - ), - ) - - return (url, specifier, marker) - - -def _parse_requirement_marker( - tokenizer: Tokenizer, *, span_start: int, after: str -) -> MarkerList: - """ - requirement_marker = SEMICOLON marker WS? - """ - - if not tokenizer.check("SEMICOLON"): - tokenizer.raise_syntax_error( - f"Expected end or semicolon (after {after})", - span_start=span_start, - ) - tokenizer.read() - - marker = _parse_marker(tokenizer) - tokenizer.consume("WS") - - return marker - - -def _parse_extras(tokenizer: Tokenizer) -> List[str]: - """ - extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? - """ - if not tokenizer.check("LEFT_BRACKET", peek=True): - return [] - - with tokenizer.enclosing_tokens( - "LEFT_BRACKET", - "RIGHT_BRACKET", - around="extras", - ): - tokenizer.consume("WS") - extras = _parse_extras_list(tokenizer) - tokenizer.consume("WS") - - return extras - - -def _parse_extras_list(tokenizer: Tokenizer) -> List[str]: - """ - extras_list = identifier (wsp* ',' wsp* identifier)* - """ - extras: List[str] = [] - - if not tokenizer.check("IDENTIFIER"): - return extras - - extras.append(tokenizer.read().text) - - while True: - tokenizer.consume("WS") - if tokenizer.check("IDENTIFIER", peek=True): - tokenizer.raise_syntax_error("Expected comma between extra names") - elif not tokenizer.check("COMMA"): - break - - tokenizer.read() - tokenizer.consume("WS") - - extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") - extras.append(extra_token.text) - - return extras - - -def _parse_specifier(tokenizer: Tokenizer) -> str: - """ - specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS - | WS? version_many WS? - """ - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="version specifier", - ): - tokenizer.consume("WS") - parsed_specifiers = _parse_version_many(tokenizer) - tokenizer.consume("WS") - - return parsed_specifiers - - -def _parse_version_many(tokenizer: Tokenizer) -> str: - """ - version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? - """ - parsed_specifiers = "" - while tokenizer.check("SPECIFIER"): - span_start = tokenizer.position - parsed_specifiers += tokenizer.read().text - if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): - tokenizer.raise_syntax_error( - ".* suffix can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position + 1, - ) - if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): - tokenizer.raise_syntax_error( - "Local version label can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position, - ) - tokenizer.consume("WS") - if not tokenizer.check("COMMA"): - break - parsed_specifiers += tokenizer.read().text - tokenizer.consume("WS") - - return parsed_specifiers - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for marker expression -# -------------------------------------------------------------------------------------- -def parse_marker(source: str) -> MarkerList: - return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList: - retval = _parse_marker(tokenizer) - tokenizer.expect("END", expected="end of marker expression") - return retval - - -def _parse_marker(tokenizer: Tokenizer) -> MarkerList: - """ - marker = marker_atom (BOOLOP marker_atom)+ - """ - expression = [_parse_marker_atom(tokenizer)] - while tokenizer.check("BOOLOP"): - token = tokenizer.read() - expr_right = _parse_marker_atom(tokenizer) - expression.extend((token.text, expr_right)) - return expression - - -def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: - """ - marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? - | WS? marker_item WS? - """ - - tokenizer.consume("WS") - if tokenizer.check("LEFT_PARENTHESIS", peek=True): - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="marker expression", - ): - tokenizer.consume("WS") - marker: MarkerAtom = _parse_marker(tokenizer) - tokenizer.consume("WS") - else: - marker = _parse_marker_item(tokenizer) - tokenizer.consume("WS") - return marker - - -def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: - """ - marker_item = WS? marker_var WS? marker_op WS? marker_var WS? - """ - tokenizer.consume("WS") - marker_var_left = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - marker_op = _parse_marker_op(tokenizer) - tokenizer.consume("WS") - marker_var_right = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - return (marker_var_left, marker_op, marker_var_right) - - -def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: - """ - marker_var = VARIABLE | QUOTED_STRING - """ - if tokenizer.check("VARIABLE"): - return process_env_var(tokenizer.read().text.replace(".", "_")) - elif tokenizer.check("QUOTED_STRING"): - return process_python_str(tokenizer.read().text) - else: - tokenizer.raise_syntax_error( - message="Expected a marker variable or quoted string" - ) - - -def process_env_var(env_var: str) -> Variable: - if env_var in ("platform_python_implementation", "python_implementation"): - return Variable("platform_python_implementation") - else: - return Variable(env_var) - - -def process_python_str(python_str: str) -> Value: - value = ast.literal_eval(python_str) - return Value(str(value)) - - -def _parse_marker_op(tokenizer: Tokenizer) -> Op: - """ - marker_op = IN | NOT IN | OP - """ - if tokenizer.check("IN"): - tokenizer.read() - return Op("in") - elif tokenizer.check("NOT"): - tokenizer.read() - tokenizer.expect("WS", expected="whitespace after 'not'") - tokenizer.expect("IN", expected="'in' after 'not'") - return Op("not in") - elif tokenizer.check("OP"): - return Op(tokenizer.read().text) - else: - return tokenizer.raise_syntax_error( - "Expected marker operator, one of " - "<=, <, !=, ==, >=, >, ~=, ===, in, not in" - ) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py deleted file mode 100644 index 90a6465f968..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py +++ /dev/null @@ -1,61 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - - -class InfinityType: - def __repr__(self) -> str: - return "Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return False - - def __le__(self, other: object) -> bool: - return False - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return True - - def __ge__(self, other: object) -> bool: - return True - - def __neg__(self: object) -> "NegativeInfinityType": - return NegativeInfinity - - -Infinity = InfinityType() - - -class NegativeInfinityType: - def __repr__(self) -> str: - return "-Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return True - - def __le__(self, other: object) -> bool: - return True - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return False - - def __ge__(self, other: object) -> bool: - return False - - def __neg__(self: object) -> InfinityType: - return Infinity - - -NegativeInfinity = NegativeInfinityType() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py deleted file mode 100644 index dd0d648d49a..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py +++ /dev/null @@ -1,192 +0,0 @@ -import contextlib -import re -from dataclasses import dataclass -from typing import Dict, Iterator, NoReturn, Optional, Tuple, Union - -from .specifiers import Specifier - - -@dataclass -class Token: - name: str - text: str - position: int - - -class ParserSyntaxError(Exception): - """The provided source text could not be parsed correctly.""" - - def __init__( - self, - message: str, - *, - source: str, - span: Tuple[int, int], - ) -> None: - self.span = span - self.message = message - self.source = source - - super().__init__() - - def __str__(self) -> str: - marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" - return "\n ".join([self.message, self.source, marker]) - - -DEFAULT_RULES: "Dict[str, Union[str, re.Pattern[str]]]" = { - "LEFT_PARENTHESIS": r"\(", - "RIGHT_PARENTHESIS": r"\)", - "LEFT_BRACKET": r"\[", - "RIGHT_BRACKET": r"\]", - "SEMICOLON": r";", - "COMMA": r",", - "QUOTED_STRING": re.compile( - r""" - ( - ('[^']*') - | - ("[^"]*") - ) - """, - re.VERBOSE, - ), - "OP": r"(===|==|~=|!=|<=|>=|<|>)", - "BOOLOP": r"\b(or|and)\b", - "IN": r"\bin\b", - "NOT": r"\bnot\b", - "VARIABLE": re.compile( - r""" - \b( - python_version - |python_full_version - |os[._]name - |sys[._]platform - |platform_(release|system) - |platform[._](version|machine|python_implementation) - |python_implementation - |implementation_(name|version) - |extra - )\b - """, - re.VERBOSE, - ), - "SPECIFIER": re.compile( - Specifier._operator_regex_str + Specifier._version_regex_str, - re.VERBOSE | re.IGNORECASE, - ), - "AT": r"\@", - "URL": r"[^ \t]+", - "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", - "VERSION_PREFIX_TRAIL": r"\.\*", - "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", - "WS": r"[ \t]+", - "END": r"$", -} - - -class Tokenizer: - """Context-sensitive token parsing. - - Provides methods to examine the input stream to check whether the next token - matches. - """ - - def __init__( - self, - source: str, - *, - rules: "Dict[str, Union[str, re.Pattern[str]]]", - ) -> None: - self.source = source - self.rules: Dict[str, re.Pattern[str]] = { - name: re.compile(pattern) for name, pattern in rules.items() - } - self.next_token: Optional[Token] = None - self.position = 0 - - def consume(self, name: str) -> None: - """Move beyond provided token name, if at current position.""" - if self.check(name): - self.read() - - def check(self, name: str, *, peek: bool = False) -> bool: - """Check whether the next token has the provided name. - - By default, if the check succeeds, the token *must* be read before - another check. If `peek` is set to `True`, the token is not loaded and - would need to be checked again. - """ - assert ( - self.next_token is None - ), f"Cannot check for {name!r}, already have {self.next_token!r}" - assert name in self.rules, f"Unknown token name: {name!r}" - - expression = self.rules[name] - - match = expression.match(self.source, self.position) - if match is None: - return False - if not peek: - self.next_token = Token(name, match[0], self.position) - return True - - def expect(self, name: str, *, expected: str) -> Token: - """Expect a certain token name next, failing with a syntax error otherwise. - - The token is *not* read. - """ - if not self.check(name): - raise self.raise_syntax_error(f"Expected {expected}") - return self.read() - - def read(self) -> Token: - """Consume the next token and return it.""" - token = self.next_token - assert token is not None - - self.position += len(token.text) - self.next_token = None - - return token - - def raise_syntax_error( - self, - message: str, - *, - span_start: Optional[int] = None, - span_end: Optional[int] = None, - ) -> NoReturn: - """Raise ParserSyntaxError at the given position.""" - span = ( - self.position if span_start is None else span_start, - self.position if span_end is None else span_end, - ) - raise ParserSyntaxError( - message, - source=self.source, - span=span, - ) - - @contextlib.contextmanager - def enclosing_tokens( - self, open_token: str, close_token: str, *, around: str - ) -> Iterator[None]: - if self.check(open_token): - open_position = self.position - self.read() - else: - open_position = None - - yield - - if open_position is None: - return - - if not self.check(close_token): - self.raise_syntax_error( - f"Expected matching {close_token} for {open_token}, after {around}", - span_start=open_position, - ) - - self.read() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/markers.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/markers.py deleted file mode 100644 index 8b98fca7233..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/markers.py +++ /dev/null @@ -1,252 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import operator -import os -import platform -import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -from ._parser import ( - MarkerAtom, - MarkerList, - Op, - Value, - Variable, - parse_marker as _parse_marker, -) -from ._tokenizer import ParserSyntaxError -from .specifiers import InvalidSpecifier, Specifier -from .utils import canonicalize_name - -__all__ = [ - "InvalidMarker", - "UndefinedComparison", - "UndefinedEnvironmentName", - "Marker", - "default_environment", -] - -Operator = Callable[[str, str], bool] - - -class InvalidMarker(ValueError): - """ - An invalid marker was found, users should refer to PEP 508. - """ - - -class UndefinedComparison(ValueError): - """ - An invalid operation was attempted on a value that doesn't support it. - """ - - -class UndefinedEnvironmentName(ValueError): - """ - A name was attempted to be used that does not exist inside of the - environment. - """ - - -def _normalize_extra_values(results: Any) -> Any: - """ - Normalize extra values. - """ - if isinstance(results[0], tuple): - lhs, op, rhs = results[0] - if isinstance(lhs, Variable) and lhs.value == "extra": - normalized_extra = canonicalize_name(rhs.value) - rhs = Value(normalized_extra) - elif isinstance(rhs, Variable) and rhs.value == "extra": - normalized_extra = canonicalize_name(lhs.value) - lhs = Value(normalized_extra) - results[0] = lhs, op, rhs - return results - - -def _format_marker( - marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True -) -> str: - - assert isinstance(marker, (list, tuple, str)) - - # Sometimes we have a structure like [[...]] which is a single item list - # where the single item is itself it's own list. In that case we want skip - # the rest of this function so that we don't get extraneous () on the - # outside. - if ( - isinstance(marker, list) - and len(marker) == 1 - and isinstance(marker[0], (list, tuple)) - ): - return _format_marker(marker[0]) - - if isinstance(marker, list): - inner = (_format_marker(m, first=False) for m in marker) - if first: - return " ".join(inner) - else: - return "(" + " ".join(inner) + ")" - elif isinstance(marker, tuple): - return " ".join([m.serialize() for m in marker]) - else: - return marker - - -_operators: Dict[str, Operator] = { - "in": lambda lhs, rhs: lhs in rhs, - "not in": lambda lhs, rhs: lhs not in rhs, - "<": operator.lt, - "<=": operator.le, - "==": operator.eq, - "!=": operator.ne, - ">=": operator.ge, - ">": operator.gt, -} - - -def _eval_op(lhs: str, op: Op, rhs: str) -> bool: - try: - spec = Specifier("".join([op.serialize(), rhs])) - except InvalidSpecifier: - pass - else: - return spec.contains(lhs, prereleases=True) - - oper: Optional[Operator] = _operators.get(op.serialize()) - if oper is None: - raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.") - - return oper(lhs, rhs) - - -def _normalize(*values: str, key: str) -> Tuple[str, ...]: - # PEP 685 – Comparison of extra names for optional distribution dependencies - # https://peps.python.org/pep-0685/ - # > When comparing extra names, tools MUST normalize the names being - # > compared using the semantics outlined in PEP 503 for names - if key == "extra": - return tuple(canonicalize_name(v) for v in values) - - # other environment markers don't have such standards - return values - - -def _evaluate_markers(markers: MarkerList, environment: Dict[str, str]) -> bool: - groups: List[List[bool]] = [[]] - - for marker in markers: - assert isinstance(marker, (list, tuple, str)) - - if isinstance(marker, list): - groups[-1].append(_evaluate_markers(marker, environment)) - elif isinstance(marker, tuple): - lhs, op, rhs = marker - - if isinstance(lhs, Variable): - environment_key = lhs.value - lhs_value = environment[environment_key] - rhs_value = rhs.value - else: - lhs_value = lhs.value - environment_key = rhs.value - rhs_value = environment[environment_key] - - lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key) - groups[-1].append(_eval_op(lhs_value, op, rhs_value)) - else: - assert marker in ["and", "or"] - if marker == "or": - groups.append([]) - - return any(all(item) for item in groups) - - -def format_full_version(info: "sys._version_info") -> str: - version = "{0.major}.{0.minor}.{0.micro}".format(info) - kind = info.releaselevel - if kind != "final": - version += kind[0] + str(info.serial) - return version - - -def default_environment() -> Dict[str, str]: - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name - return { - "implementation_name": implementation_name, - "implementation_version": iver, - "os_name": os.name, - "platform_machine": platform.machine(), - "platform_release": platform.release(), - "platform_system": platform.system(), - "platform_version": platform.version(), - "python_full_version": platform.python_version(), - "platform_python_implementation": platform.python_implementation(), - "python_version": ".".join(platform.python_version_tuple()[:2]), - "sys_platform": sys.platform, - } - - -class Marker: - def __init__(self, marker: str) -> None: - # Note: We create a Marker object without calling this constructor in - # packaging.requirements.Requirement. If any additional logic is - # added here, make sure to mirror/adapt Requirement. - try: - self._markers = _normalize_extra_values(_parse_marker(marker)) - # The attribute `_markers` can be described in terms of a recursive type: - # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]] - # - # For example, the following expression: - # python_version > "3.6" or (python_version == "3.6" and os_name == "unix") - # - # is parsed into: - # [ - # (<Variable('python_version')>, <Op('>')>, <Value('3.6')>), - # 'and', - # [ - # (<Variable('python_version')>, <Op('==')>, <Value('3.6')>), - # 'or', - # (<Variable('os_name')>, <Op('==')>, <Value('unix')>) - # ] - # ] - except ParserSyntaxError as e: - raise InvalidMarker(str(e)) from e - - def __str__(self) -> str: - return _format_marker(self._markers) - - def __repr__(self) -> str: - return f"<Marker('{self}')>" - - def __hash__(self) -> int: - return hash((self.__class__.__name__, str(self))) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Marker): - return NotImplemented - - return str(self) == str(other) - - def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool: - """Evaluate a marker. - - Return the boolean from evaluating the given marker against the - environment. environment is an optional argument to override all or - part of the determined environment. - - The environment is determined from the current Python process. - """ - current_environment = default_environment() - current_environment["extra"] = "" - if environment is not None: - current_environment.update(environment) - # The API used to allow setting extra to None. We need to handle this - # case for backwards compatibility. - if current_environment["extra"] is None: - current_environment["extra"] = "" - - return _evaluate_markers(self._markers, current_environment) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py deleted file mode 100644 index fb274930799..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py +++ /dev/null @@ -1,825 +0,0 @@ -import email.feedparser -import email.header -import email.message -import email.parser -import email.policy -import sys -import typing -from typing import ( - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - Union, - cast, -) - -from . import requirements, specifiers, utils, version as version_module - -T = typing.TypeVar("T") -if sys.version_info[:2] >= (3, 8): # pragma: no cover - from typing import Literal, TypedDict -else: # pragma: no cover - if typing.TYPE_CHECKING: - from typing_extensions import Literal, TypedDict - else: - try: - from typing_extensions import Literal, TypedDict - except ImportError: - - class Literal: - def __init_subclass__(*_args, **_kwargs): - pass - - class TypedDict: - def __init_subclass__(*_args, **_kwargs): - pass - - -try: - ExceptionGroup -except NameError: # pragma: no cover - - class ExceptionGroup(Exception): # noqa: N818 - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: List[Exception] - - def __init__(self, message: str, exceptions: List[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" - -else: # pragma: no cover - ExceptionGroup = ExceptionGroup - - -class InvalidMetadata(ValueError): - """A metadata field contains invalid data.""" - - field: str - """The name of the field that contains invalid data.""" - - def __init__(self, field: str, message: str) -> None: - self.field = field - super().__init__(message) - - -# The RawMetadata class attempts to make as few assumptions about the underlying -# serialization formats as possible. The idea is that as long as a serialization -# formats offer some very basic primitives in *some* way then we can support -# serializing to and from that format. -class RawMetadata(TypedDict, total=False): - """A dictionary of raw core metadata. - - Each field in core metadata maps to a key of this dictionary (when data is - provided). The key is lower-case and underscores are used instead of dashes - compared to the equivalent core metadata field. Any core metadata field that - can be specified multiple times or can hold multiple values in a single - field have a key with a plural name. See :class:`Metadata` whose attributes - match the keys of this dictionary. - - Core metadata fields that can be specified multiple times are stored as a - list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - - """ - - # Metadata 1.0 - PEP 241 - metadata_version: str - name: str - version: str - platforms: List[str] - summary: str - description: str - keywords: List[str] - home_page: str - author: str - author_email: str - license: str - - # Metadata 1.1 - PEP 314 - supported_platforms: List[str] - download_url: str - classifiers: List[str] - requires: List[str] - provides: List[str] - obsoletes: List[str] - - # Metadata 1.2 - PEP 345 - maintainer: str - maintainer_email: str - requires_dist: List[str] - provides_dist: List[str] - obsoletes_dist: List[str] - requires_python: str - requires_external: List[str] - project_urls: Dict[str, str] - - # Metadata 2.0 - # PEP 426 attempted to completely revamp the metadata format - # but got stuck without ever being able to build consensus on - # it and ultimately ended up withdrawn. - # - # However, a number of tools had started emitting METADATA with - # `2.0` Metadata-Version, so for historical reasons, this version - # was skipped. - - # Metadata 2.1 - PEP 566 - description_content_type: str - provides_extra: List[str] - - # Metadata 2.2 - PEP 643 - dynamic: List[str] - - # Metadata 2.3 - PEP 685 - # No new fields were added in PEP 685, just some edge case were - # tightened up to provide better interoptability. - - -_STRING_FIELDS = { - "author", - "author_email", - "description", - "description_content_type", - "download_url", - "home_page", - "license", - "maintainer", - "maintainer_email", - "metadata_version", - "name", - "requires_python", - "summary", - "version", -} - -_LIST_FIELDS = { - "classifiers", - "dynamic", - "obsoletes", - "obsoletes_dist", - "platforms", - "provides", - "provides_dist", - "provides_extra", - "requires", - "requires_dist", - "requires_external", - "supported_platforms", -} - -_DICT_FIELDS = { - "project_urls", -} - - -def _parse_keywords(data: str) -> List[str]: - """Split a string of comma-separate keyboards into a list of keywords.""" - return [k.strip() for k in data.split(",")] - - -def _parse_project_urls(data: List[str]) -> Dict[str, str]: - """Parse a list of label/URL string pairings separated by a comma.""" - urls = {} - for pair in data: - # Our logic is slightly tricky here as we want to try and do - # *something* reasonable with malformed data. - # - # The main thing that we have to worry about, is data that does - # not have a ',' at all to split the label from the Value. There - # isn't a singular right answer here, and we will fail validation - # later on (if the caller is validating) so it doesn't *really* - # matter, but since the missing value has to be an empty str - # and our return value is dict[str, str], if we let the key - # be the missing value, then they'd have multiple '' values that - # overwrite each other in a accumulating dict. - # - # The other potentional issue is that it's possible to have the - # same label multiple times in the metadata, with no solid "right" - # answer with what to do in that case. As such, we'll do the only - # thing we can, which is treat the field as unparseable and add it - # to our list of unparsed fields. - parts = [p.strip() for p in pair.split(",", 1)] - parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items - - # TODO: The spec doesn't say anything about if the keys should be - # considered case sensitive or not... logically they should - # be case-preserving and case-insensitive, but doing that - # would open up more cases where we might have duplicate - # entries. - label, url = parts - if label in urls: - # The label already exists in our set of urls, so this field - # is unparseable, and we can just add the whole thing to our - # unparseable data and stop processing it. - raise KeyError("duplicate labels in project urls") - urls[label] = url - - return urls - - -def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: - """Get the body of the message.""" - # If our source is a str, then our caller has managed encodings for us, - # and we don't need to deal with it. - if isinstance(source, str): - payload: str = msg.get_payload() - return payload - # If our source is a bytes, then we're managing the encoding and we need - # to deal with it. - else: - bpayload: bytes = msg.get_payload(decode=True) - try: - return bpayload.decode("utf8", "strict") - except UnicodeDecodeError: - raise ValueError("payload in an invalid encoding") - - -# The various parse_FORMAT functions here are intended to be as lenient as -# possible in their parsing, while still returning a correctly typed -# RawMetadata. -# -# To aid in this, we also generally want to do as little touching of the -# data as possible, except where there are possibly some historic holdovers -# that make valid data awkward to work with. -# -# While this is a lower level, intermediate format than our ``Metadata`` -# class, some light touch ups can make a massive difference in usability. - -# Map METADATA fields to RawMetadata. -_EMAIL_TO_RAW_MAPPING = { - "author": "author", - "author-email": "author_email", - "classifier": "classifiers", - "description": "description", - "description-content-type": "description_content_type", - "download-url": "download_url", - "dynamic": "dynamic", - "home-page": "home_page", - "keywords": "keywords", - "license": "license", - "maintainer": "maintainer", - "maintainer-email": "maintainer_email", - "metadata-version": "metadata_version", - "name": "name", - "obsoletes": "obsoletes", - "obsoletes-dist": "obsoletes_dist", - "platform": "platforms", - "project-url": "project_urls", - "provides": "provides", - "provides-dist": "provides_dist", - "provides-extra": "provides_extra", - "requires": "requires", - "requires-dist": "requires_dist", - "requires-external": "requires_external", - "requires-python": "requires_python", - "summary": "summary", - "supported-platform": "supported_platforms", - "version": "version", -} -_RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()} - - -def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: - """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``). - - This function returns a two-item tuple of dicts. The first dict is of - recognized fields from the core metadata specification. Fields that can be - parsed and translated into Python's built-in types are converted - appropriately. All other fields are left as-is. Fields that are allowed to - appear multiple times are stored as lists. - - The second dict contains all other fields from the metadata. This includes - any unrecognized fields. It also includes any fields which are expected to - be parsed into a built-in type but were not formatted appropriately. Finally, - any fields that are expected to appear only once but are repeated are - included in this dict. - - """ - raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} - unparsed: Dict[str, List[str]] = {} - - if isinstance(data, str): - parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) - else: - parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) - - # We have to wrap parsed.keys() in a set, because in the case of multiple - # values for a key (a list), the key will appear multiple times in the - # list of keys, but we're avoiding that by using get_all(). - for name in frozenset(parsed.keys()): - # Header names in RFC are case insensitive, so we'll normalize to all - # lower case to make comparisons easier. - name = name.lower() - - # We use get_all() here, even for fields that aren't multiple use, - # because otherwise someone could have e.g. two Name fields, and we - # would just silently ignore it rather than doing something about it. - headers = parsed.get_all(name) or [] - - # The way the email module works when parsing bytes is that it - # unconditionally decodes the bytes as ascii using the surrogateescape - # handler. When you pull that data back out (such as with get_all() ), - # it looks to see if the str has any surrogate escapes, and if it does - # it wraps it in a Header object instead of returning the string. - # - # As such, we'll look for those Header objects, and fix up the encoding. - value = [] - # Flag if we have run into any issues processing the headers, thus - # signalling that the data belongs in 'unparsed'. - valid_encoding = True - for h in headers: - # It's unclear if this can return more types than just a Header or - # a str, so we'll just assert here to make sure. - assert isinstance(h, (email.header.Header, str)) - - # If it's a header object, we need to do our little dance to get - # the real data out of it. In cases where there is invalid data - # we're going to end up with mojibake, but there's no obvious, good - # way around that without reimplementing parts of the Header object - # ourselves. - # - # That should be fine since, if mojibacked happens, this key is - # going into the unparsed dict anyways. - if isinstance(h, email.header.Header): - # The Header object stores it's data as chunks, and each chunk - # can be independently encoded, so we'll need to check each - # of them. - chunks: List[Tuple[bytes, Optional[str]]] = [] - for bin, encoding in email.header.decode_header(h): - try: - bin.decode("utf8", "strict") - except UnicodeDecodeError: - # Enable mojibake. - encoding = "latin1" - valid_encoding = False - else: - encoding = "utf8" - chunks.append((bin, encoding)) - - # Turn our chunks back into a Header object, then let that - # Header object do the right thing to turn them into a - # string for us. - value.append(str(email.header.make_header(chunks))) - # This is already a string, so just add it. - else: - value.append(h) - - # We've processed all of our values to get them into a list of str, - # but we may have mojibake data, in which case this is an unparsed - # field. - if not valid_encoding: - unparsed[name] = value - continue - - raw_name = _EMAIL_TO_RAW_MAPPING.get(name) - if raw_name is None: - # This is a bit of a weird situation, we've encountered a key that - # we don't know what it means, so we don't know whether it's meant - # to be a list or not. - # - # Since we can't really tell one way or another, we'll just leave it - # as a list, even though it may be a single item list, because that's - # what makes the most sense for email headers. - unparsed[name] = value - continue - - # If this is one of our string fields, then we'll check to see if our - # value is a list of a single item. If it is then we'll assume that - # it was emitted as a single string, and unwrap the str from inside - # the list. - # - # If it's any other kind of data, then we haven't the faintest clue - # what we should parse it as, and we have to just add it to our list - # of unparsed stuff. - if raw_name in _STRING_FIELDS and len(value) == 1: - raw[raw_name] = value[0] - # If this is one of our list of string fields, then we can just assign - # the value, since email *only* has strings, and our get_all() call - # above ensures that this is a list. - elif raw_name in _LIST_FIELDS: - raw[raw_name] = value - # Special Case: Keywords - # The keywords field is implemented in the metadata spec as a str, - # but it conceptually is a list of strings, and is serialized using - # ", ".join(keywords), so we'll do some light data massaging to turn - # this into what it logically is. - elif raw_name == "keywords" and len(value) == 1: - raw[raw_name] = _parse_keywords(value[0]) - # Special Case: Project-URL - # The project urls is implemented in the metadata spec as a list of - # specially-formatted strings that represent a key and a value, which - # is fundamentally a mapping, however the email format doesn't support - # mappings in a sane way, so it was crammed into a list of strings - # instead. - # - # We will do a little light data massaging to turn this into a map as - # it logically should be. - elif raw_name == "project_urls": - try: - raw[raw_name] = _parse_project_urls(value) - except KeyError: - unparsed[name] = value - # Nothing that we've done has managed to parse this, so it'll just - # throw it in our unparseable data and move on. - else: - unparsed[name] = value - - # We need to support getting the Description from the message payload in - # addition to getting it from the the headers. This does mean, though, there - # is the possibility of it being set both ways, in which case we put both - # in 'unparsed' since we don't know which is right. - try: - payload = _get_payload(parsed, data) - except ValueError: - unparsed.setdefault("description", []).append( - parsed.get_payload(decode=isinstance(data, bytes)) - ) - else: - if payload: - # Check to see if we've already got a description, if so then both - # it, and this body move to unparseable. - if "description" in raw: - description_header = cast(str, raw.pop("description")) - unparsed.setdefault("description", []).extend( - [description_header, payload] - ) - elif "description" in unparsed: - unparsed["description"].append(payload) - else: - raw["description"] = payload - - # We need to cast our `raw` to a metadata, because a TypedDict only support - # literal key names, but we're computing our key names on purpose, but the - # way this function is implemented, our `TypedDict` can only have valid key - # names. - return cast(RawMetadata, raw), unparsed - - -_NOT_FOUND = object() - - -# Keep the two values in sync. -_VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] -_MetadataVersion = Literal["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] - -_REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"]) - - -class _Validator(Generic[T]): - """Validate a metadata field. - - All _process_*() methods correspond to a core metadata field. The method is - called with the field's raw value. If the raw value is valid it is returned - in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field). - If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause - as appropriate). - """ - - name: str - raw_name: str - added: _MetadataVersion - - def __init__( - self, - *, - added: _MetadataVersion = "1.0", - ) -> None: - self.added = added - - def __set_name__(self, _owner: "Metadata", name: str) -> None: - self.name = name - self.raw_name = _RAW_TO_EMAIL_MAPPING[name] - - def __get__(self, instance: "Metadata", _owner: Type["Metadata"]) -> T: - # With Python 3.8, the caching can be replaced with functools.cached_property(). - # No need to check the cache as attribute lookup will resolve into the - # instance's __dict__ before __get__ is called. - cache = instance.__dict__ - value = instance._raw.get(self.name) - - # To make the _process_* methods easier, we'll check if the value is None - # and if this field is NOT a required attribute, and if both of those - # things are true, we'll skip the the converter. This will mean that the - # converters never have to deal with the None union. - if self.name in _REQUIRED_ATTRS or value is not None: - try: - converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}") - except AttributeError: - pass - else: - value = converter(value) - - cache[self.name] = value - try: - del instance._raw[self.name] # type: ignore[misc] - except KeyError: - pass - - return cast(T, value) - - def _invalid_metadata( - self, msg: str, cause: Optional[Exception] = None - ) -> InvalidMetadata: - exc = InvalidMetadata( - self.raw_name, msg.format_map({"field": repr(self.raw_name)}) - ) - exc.__cause__ = cause - return exc - - def _process_metadata_version(self, value: str) -> _MetadataVersion: - # Implicitly makes Metadata-Version required. - if value not in _VALID_METADATA_VERSIONS: - raise self._invalid_metadata(f"{value!r} is not a valid metadata version") - return cast(_MetadataVersion, value) - - def _process_name(self, value: str) -> str: - if not value: - raise self._invalid_metadata("{field} is a required field") - # Validate the name as a side-effect. - try: - utils.canonicalize_name(value, validate=True) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - else: - return value - - def _process_version(self, value: str) -> version_module.Version: - if not value: - raise self._invalid_metadata("{field} is a required field") - try: - return version_module.parse(value) - except version_module.InvalidVersion as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_summary(self, value: str) -> str: - """Check the field contains no newlines.""" - if "\n" in value: - raise self._invalid_metadata("{field} must be a single line") - return value - - def _process_description_content_type(self, value: str) -> str: - content_types = {"text/plain", "text/x-rst", "text/markdown"} - message = email.message.EmailMessage() - message["content-type"] = value - - content_type, parameters = ( - # Defaults to `text/plain` if parsing failed. - message.get_content_type().lower(), - message["content-type"].params, - ) - # Check if content-type is valid or defaulted to `text/plain` and thus was - # not parseable. - if content_type not in content_types or content_type not in value.lower(): - raise self._invalid_metadata( - f"{{field}} must be one of {list(content_types)}, not {value!r}" - ) - - charset = parameters.get("charset", "UTF-8") - if charset != "UTF-8": - raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" - ) - - markdown_variants = {"GFM", "CommonMark"} - variant = parameters.get("variant", "GFM") # Use an acceptable default. - if content_type == "text/markdown" and variant not in markdown_variants: - raise self._invalid_metadata( - f"valid Markdown variants for {{field}} are {list(markdown_variants)}, " - f"not {variant!r}", - ) - return value - - def _process_dynamic(self, value: List[str]) -> List[str]: - for dynamic_field in map(str.lower, value): - if dynamic_field in {"name", "version", "metadata-version"}: - raise self._invalid_metadata( - f"{value!r} is not allowed as a dynamic field" - ) - elif dynamic_field not in _EMAIL_TO_RAW_MAPPING: - raise self._invalid_metadata(f"{value!r} is not a valid dynamic field") - return list(map(str.lower, value)) - - def _process_provides_extra( - self, - value: List[str], - ) -> List[utils.NormalizedName]: - normalized_names = [] - try: - for name in value: - normalized_names.append(utils.canonicalize_name(name, validate=True)) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{name!r} is invalid for {{field}}", cause=exc - ) - else: - return normalized_names - - def _process_requires_python(self, value: str) -> specifiers.SpecifierSet: - try: - return specifiers.SpecifierSet(value) - except specifiers.InvalidSpecifier as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_requires_dist( - self, - value: List[str], - ) -> List[requirements.Requirement]: - reqs = [] - try: - for req in value: - reqs.append(requirements.Requirement(req)) - except requirements.InvalidRequirement as exc: - raise self._invalid_metadata(f"{req!r} is invalid for {{field}}", cause=exc) - else: - return reqs - - -class Metadata: - """Representation of distribution metadata. - - Compared to :class:`RawMetadata`, this class provides objects representing - metadata fields instead of only using built-in types. Any invalid metadata - will cause :exc:`InvalidMetadata` to be raised (with a - :py:attr:`~BaseException.__cause__` attribute as appropriate). - """ - - _raw: RawMetadata - - @classmethod - def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> "Metadata": - """Create an instance from :class:`RawMetadata`. - - If *validate* is true, all metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - ins = cls() - ins._raw = data.copy() # Mutations occur due to caching enriched values. - - if validate: - exceptions: List[Exception] = [] - try: - metadata_version = ins.metadata_version - metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None - - # Make sure to check for the fields that are present, the required - # fields (so their absence can be reported). - fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS - # Remove fields that have already been checked. - fields_to_check -= {"metadata_version"} - - for key in fields_to_check: - try: - if metadata_version: - # Can't use getattr() as that triggers descriptor protocol which - # will fail due to no value for the instance argument. - try: - field_metadata_version = cls.__dict__[key].added - except KeyError: - exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) - continue - field_age = _VALID_METADATA_VERSIONS.index( - field_metadata_version - ) - if field_age > metadata_age: - field = _RAW_TO_EMAIL_MAPPING[key] - exc = InvalidMetadata( - field, - "{field} introduced in metadata version " - "{field_metadata_version}, not {metadata_version}", - ) - exceptions.append(exc) - continue - getattr(ins, key) - except InvalidMetadata as exc: - exceptions.append(exc) - - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) - - return ins - - @classmethod - def from_email( - cls, data: Union[bytes, str], *, validate: bool = True - ) -> "Metadata": - """Parse metadata from email headers. - - If *validate* is true, the metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - raw, unparsed = parse_email(data) - - if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) - - try: - return cls.from_raw(raw, validate=validate) - except ExceptionGroup as exc_group: - raise ExceptionGroup( - "invalid or unparsed metadata", exc_group.exceptions - ) from None - - metadata_version: _Validator[_MetadataVersion] = _Validator() - """:external:ref:`core-metadata-metadata-version` - (required; validated to be a valid metadata version)""" - name: _Validator[str] = _Validator() - """:external:ref:`core-metadata-name` - (required; validated using :func:`~packaging.utils.canonicalize_name` and its - *validate* parameter)""" - version: _Validator[version_module.Version] = _Validator() - """:external:ref:`core-metadata-version` (required)""" - dynamic: _Validator[Optional[List[str]]] = _Validator( - added="2.2", - ) - """:external:ref:`core-metadata-dynamic` - (validated against core metadata field names and lowercased)""" - platforms: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-platform`""" - supported_platforms: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-supported-platform`""" - summary: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-summary` (validated to contain no newlines)""" - description: _Validator[Optional[str]] = _Validator() # TODO 2.1: can be in body - """:external:ref:`core-metadata-description`""" - description_content_type: _Validator[Optional[str]] = _Validator(added="2.1") - """:external:ref:`core-metadata-description-content-type` (validated)""" - keywords: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-keywords`""" - home_page: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-home-page`""" - download_url: _Validator[Optional[str]] = _Validator(added="1.1") - """:external:ref:`core-metadata-download-url`""" - author: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author`""" - author_email: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author-email`""" - maintainer: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer`""" - maintainer_email: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer-email`""" - license: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-license`""" - classifiers: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-classifier`""" - requires_dist: _Validator[Optional[List[requirements.Requirement]]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-dist`""" - requires_python: _Validator[Optional[specifiers.SpecifierSet]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-python`""" - # Because `Requires-External` allows for non-PEP 440 version specifiers, we - # don't do any processing on the values. - requires_external: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-requires-external`""" - project_urls: _Validator[Optional[Dict[str, str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-project-url`""" - # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation - # regardless of metadata version. - provides_extra: _Validator[Optional[List[utils.NormalizedName]]] = _Validator( - added="2.1", - ) - """:external:ref:`core-metadata-provides-extra`""" - provides_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-provides-dist`""" - obsoletes_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-obsoletes-dist`""" - requires: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Requires`` (deprecated)""" - provides: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Provides`` (deprecated)""" - obsoletes: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Obsoletes`` (deprecated)""" diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py deleted file mode 100644 index bdc43a7e98d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py +++ /dev/null @@ -1,90 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from typing import Any, Iterator, Optional, Set - -from ._parser import parse_requirement as _parse_requirement -from ._tokenizer import ParserSyntaxError -from .markers import Marker, _normalize_extra_values -from .specifiers import SpecifierSet -from .utils import canonicalize_name - - -class InvalidRequirement(ValueError): - """ - An invalid requirement was found, users should refer to PEP 508. - """ - - -class Requirement: - """Parse a requirement. - - Parse a given requirement string into its parts, such as name, specifier, - URL, and extras. Raises InvalidRequirement on a badly-formed requirement - string. - """ - - # TODO: Can we test whether something is contained within a requirement? - # If so how do we do that? Do we need to test against the _name_ of - # the thing as well as the version? What about the markers? - # TODO: Can we normalize the name and extra name? - - def __init__(self, requirement_string: str) -> None: - try: - parsed = _parse_requirement(requirement_string) - except ParserSyntaxError as e: - raise InvalidRequirement(str(e)) from e - - self.name: str = parsed.name - self.url: Optional[str] = parsed.url or None - self.extras: Set[str] = set(parsed.extras or []) - self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) - self.marker: Optional[Marker] = None - if parsed.marker is not None: - self.marker = Marker.__new__(Marker) - self.marker._markers = _normalize_extra_values(parsed.marker) - - def _iter_parts(self, name: str) -> Iterator[str]: - yield name - - if self.extras: - formatted_extras = ",".join(sorted(self.extras)) - yield f"[{formatted_extras}]" - - if self.specifier: - yield str(self.specifier) - - if self.url: - yield f"@ {self.url}" - if self.marker: - yield " " - - if self.marker: - yield f"; {self.marker}" - - def __str__(self) -> str: - return "".join(self._iter_parts(self.name)) - - def __repr__(self) -> str: - return f"<Requirement('{self}')>" - - def __hash__(self) -> int: - return hash( - ( - self.__class__.__name__, - *self._iter_parts(canonicalize_name(self.name)), - ) - ) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Requirement): - return NotImplemented - - return ( - canonicalize_name(self.name) == canonicalize_name(other.name) - and self.extras == other.extras - and self.specifier == other.specifier - and self.url == other.url - and self.marker == other.marker - ) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/specifiers.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/specifiers.py deleted file mode 100644 index 2d015bab595..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/specifiers.py +++ /dev/null @@ -1,1017 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.specifiers import Specifier, SpecifierSet, InvalidSpecifier - from packaging.version import Version -""" - -import abc -import itertools -import re -from typing import Callable, Iterable, Iterator, List, Optional, Tuple, TypeVar, Union - -from .utils import canonicalize_version -from .version import Version - -UnparsedVersion = Union[Version, str] -UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) -CallableOperator = Callable[[Version, str], bool] - - -def _coerce_version(version: UnparsedVersion) -> Version: - if not isinstance(version, Version): - version = Version(version) - return version - - -class InvalidSpecifier(ValueError): - """ - Raised when attempting to create a :class:`Specifier` with a specifier - string that is invalid. - - >>> Specifier("lolwat") - Traceback (most recent call last): - ... - packaging.specifiers.InvalidSpecifier: Invalid specifier: 'lolwat' - """ - - -class BaseSpecifier(metaclass=abc.ABCMeta): - @abc.abstractmethod - def __str__(self) -> str: - """ - Returns the str representation of this Specifier-like object. This - should be representative of the Specifier itself. - """ - - @abc.abstractmethod - def __hash__(self) -> int: - """ - Returns a hash value for this Specifier-like object. - """ - - @abc.abstractmethod - def __eq__(self, other: object) -> bool: - """ - Returns a boolean representing whether or not the two Specifier-like - objects are equal. - - :param other: The other object to check against. - """ - - @property - @abc.abstractmethod - def prereleases(self) -> Optional[bool]: - """Whether or not pre-releases as a whole are allowed. - - This can be set to either ``True`` or ``False`` to explicitly enable or disable - prereleases or it can be set to ``None`` (the default) to use default semantics. - """ - - @prereleases.setter - def prereleases(self, value: bool) -> None: - """Setter for :attr:`prereleases`. - - :param value: The value to set. - """ - - @abc.abstractmethod - def contains(self, item: str, prereleases: Optional[bool] = None) -> bool: - """ - Determines if the given item is contained within this specifier. - """ - - @abc.abstractmethod - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """ - Takes an iterable of items and filters them so that only items which - are contained within this specifier are allowed in it. - """ - - -class Specifier(BaseSpecifier): - """This class abstracts handling of version specifiers. - - .. tip:: - - It is generally not required to instantiate this manually. You should instead - prefer to work with :class:`SpecifierSet` instead, which can parse - comma-separated version specifiers (which is what package metadata contains). - """ - - _operator_regex_str = r""" - (?P<operator>(~=|==|!=|<=|>=|<|>|===)) - """ - _version_regex_str = r""" - (?P<version> - (?: - # The identity operators allow for an escape hatch that will - # do an exact string match of the version you wish to install. - # This will not be parsed by PEP 440 and we cannot determine - # any semantic meaning from it. This operator is discouraged - # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator - \s* - [^\s;)]* # The arbitrary version can be just about anything, - # we match everything except for whitespace, a - # semi-colon for marker support, and a closing paren - # since versions can be enclosed in them. - ) - | - (?: - # The (non)equality operators allow for wild card and local - # versions to be specified so we have to define these two - # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - - # You cannot use a wild card and a pre-release, post-release, a dev or - # local version together so group them with a | and make them optional. - (?: - \.\* # Wild card syntax of .* - | - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local - )? - ) - | - (?: - # The compatible operator requires at least two digits in the - # release segment. - (?<=~=) # Only match for the compatible operator - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - | - (?: - # All other operators only allow a sub set of what the - # (non)equality operators do. Specifically they do not allow - # local versions to be specified nor do they allow the prefix - # matching wild cards. - (?<!==|!=|~=) # We have special cases for these - # operators so we want to make sure they - # don't match here. - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - ) - """ - - _regex = re.compile( - r"^\s*" + _operator_regex_str + _version_regex_str + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) - - _operators = { - "~=": "compatible", - "==": "equal", - "!=": "not_equal", - "<=": "less_than_equal", - ">=": "greater_than_equal", - "<": "less_than", - ">": "greater_than", - "===": "arbitrary", - } - - def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: - """Initialize a Specifier instance. - - :param spec: - The string representation of a specifier which will be parsed and - normalized before use. - :param prereleases: - This tells the specifier if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - :raises InvalidSpecifier: - If the given specifier is invalid (i.e. bad syntax). - """ - match = self._regex.search(spec) - if not match: - raise InvalidSpecifier(f"Invalid specifier: '{spec}'") - - self._spec: Tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) - - # Store whether or not this Specifier should accept prereleases - self._prereleases = prereleases - - # https://github.com/python/mypy/pull/13475#pullrequestreview-1079784515 - @property # type: ignore[override] - def prereleases(self) -> bool: - # If there is an explicit prereleases set for this, then we'll just - # blindly use that. - if self._prereleases is not None: - return self._prereleases - - # Look at all of our specifiers and determine if they are inclusive - # operators, and if they are if they are including an explicit - # prerelease. - operator, version = self._spec - if operator in ["==", ">=", "<=", "~=", "==="]: - # The == specifier can include a trailing .*, if it does we - # want to remove before parsing. - if operator == "==" and version.endswith(".*"): - version = version[:-2] - - # Parse the version, and if it is a pre-release than this - # specifier allows pre-releases. - if Version(version).is_prerelease: - return True - - return False - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - @property - def operator(self) -> str: - """The operator of this specifier. - - >>> Specifier("==1.2.3").operator - '==' - """ - return self._spec[0] - - @property - def version(self) -> str: - """The version of this specifier. - - >>> Specifier("==1.2.3").version - '1.2.3' - """ - return self._spec[1] - - def __repr__(self) -> str: - """A representation of the Specifier that shows all internal state. - - >>> Specifier('>=1.0.0') - <Specifier('>=1.0.0')> - >>> Specifier('>=1.0.0', prereleases=False) - <Specifier('>=1.0.0', prereleases=False)> - >>> Specifier('>=1.0.0', prereleases=True) - <Specifier('>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<{self.__class__.__name__}({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the Specifier that can be round-tripped. - - >>> str(Specifier('>=1.0.0')) - '>=1.0.0' - >>> str(Specifier('>=1.0.0', prereleases=False)) - '>=1.0.0' - """ - return "{}{}".format(*self._spec) - - @property - def _canonical_spec(self) -> Tuple[str, str]: - canonical_version = canonicalize_version( - self._spec[1], - strip_trailing_zero=(self._spec[0] != "~="), - ) - return self._spec[0], canonical_version - - def __hash__(self) -> int: - return hash(self._canonical_spec) - - def __eq__(self, other: object) -> bool: - """Whether or not the two Specifier-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> Specifier("==1.2.3") == Specifier("== 1.2.3.0") - True - >>> (Specifier("==1.2.3", prereleases=False) == - ... Specifier("==1.2.3", prereleases=True)) - True - >>> Specifier("==1.2.3") == "==1.2.3" - True - >>> Specifier("==1.2.3") == Specifier("==1.2.4") - False - >>> Specifier("==1.2.3") == Specifier("~=1.2.3") - False - """ - if isinstance(other, str): - try: - other = self.__class__(str(other)) - except InvalidSpecifier: - return NotImplemented - elif not isinstance(other, self.__class__): - return NotImplemented - - return self._canonical_spec == other._canonical_spec - - def _get_operator(self, op: str) -> CallableOperator: - operator_callable: CallableOperator = getattr( - self, f"_compare_{self._operators[op]}" - ) - return operator_callable - - def _compare_compatible(self, prospective: Version, spec: str) -> bool: - - # Compatible releases have an equivalent combination of >= and ==. That - # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to - # implement this in terms of the other specifiers instead of - # implementing it ourselves. The only thing we need to do is construct - # the other specifiers. - - # We want everything but the last item in the version, but we want to - # ignore suffix segments. - prefix = _version_join( - list(itertools.takewhile(_is_not_suffix, _version_split(spec)))[:-1] - ) - - # Add the prefix notation to the end of our string - prefix += ".*" - - return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( - prospective, prefix - ) - - def _compare_equal(self, prospective: Version, spec: str) -> bool: - - # We need special logic to handle prefix matching - if spec.endswith(".*"): - # In the case of prefix matching we want to ignore local segment. - normalized_prospective = canonicalize_version( - prospective.public, strip_trailing_zero=False - ) - # Get the normalized version string ignoring the trailing .* - normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) - # Split the spec out by bangs and dots, and pretend that there is - # an implicit dot in between a release segment and a pre-release segment. - split_spec = _version_split(normalized_spec) - - # Split the prospective version out by bangs and dots, and pretend - # that there is an implicit dot in between a release segment and - # a pre-release segment. - split_prospective = _version_split(normalized_prospective) - - # 0-pad the prospective version before shortening it to get the correct - # shortened version. - padded_prospective, _ = _pad_version(split_prospective, split_spec) - - # Shorten the prospective version to be the same length as the spec - # so that we can determine if the specifier is a prefix of the - # prospective version or not. - shortened_prospective = padded_prospective[: len(split_spec)] - - return shortened_prospective == split_spec - else: - # Convert our spec string into a Version - spec_version = Version(spec) - - # If the specifier does not have a local segment, then we want to - # act as if the prospective version also does not have a local - # segment. - if not spec_version.local: - prospective = Version(prospective.public) - - return prospective == spec_version - - def _compare_not_equal(self, prospective: Version, spec: str) -> bool: - return not self._compare_equal(prospective, spec) - - def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) <= Version(spec) - - def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) >= Version(spec) - - def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is less than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective < spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a pre-release version, that we do not accept pre-release - # versions for the version mentioned in the specifier (e.g. <3.1 should - # not match 3.1.dev0, but should match 3.0.dev0). - if not spec.is_prerelease and prospective.is_prerelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # less than the spec version *and* it's not a pre-release of the same - # version in the spec. - return True - - def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is greater than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective > spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a post-release version, that we do not accept - # post-release versions for the version mentioned in the specifier - # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). - if not spec.is_postrelease and prospective.is_postrelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is technically greater than, to match. - if prospective.local is not None: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # greater than the spec version *and* it's not a pre-release of the - # same version in the spec. - return True - - def _compare_arbitrary(self, prospective: Version, spec: str) -> bool: - return str(prospective).lower() == str(spec).lower() - - def __contains__(self, item: Union[str, Version]) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in Specifier(">=1.2.3") - True - >>> Version("1.2.3") in Specifier(">=1.2.3") - True - >>> "1.0.0" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, item: UnparsedVersion, prereleases: Optional[bool] = None - ) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this Specifier. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> Specifier(">=1.2.3").contains("1.2.3") - True - >>> Specifier(">=1.2.3").contains(Version("1.2.3")) - True - >>> Specifier(">=1.2.3").contains("1.0.0") - False - >>> Specifier(">=1.2.3").contains("1.3.0a1") - False - >>> Specifier(">=1.2.3", prereleases=True).contains("1.3.0a1") - True - >>> Specifier(">=1.2.3").contains("1.3.0a1", prereleases=True) - True - """ - - # Determine if prereleases are to be allowed or not. - if prereleases is None: - prereleases = self.prereleases - - # Normalize item to a Version, this allows us to have a shortcut for - # "2.0" in Specifier(">=2") - normalized_item = _coerce_version(item) - - # Determine if we should be supporting prereleases in this specifier - # or not, if we do not support prereleases than we can short circuit - # logic if this version is a prereleases. - if normalized_item.is_prerelease and not prereleases: - return False - - # Actually do the comparison to determine if this item is contained - # within this Specifier or not. - operator_callable: CallableOperator = self._get_operator(self.operator) - return operator_callable(normalized_item, self.version) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifier. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(Specifier().contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.2.3", "1.3", Version("1.4")])) - ['1.2.3', '1.3', <Version('1.4')>] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.5a1"])) - ['1.5a1'] - >>> list(Specifier(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - """ - - yielded = False - found_prereleases = [] - - kw = {"prereleases": prereleases if prereleases is not None else True} - - # Attempt to iterate over all the values in the iterable and if any of - # them match, yield them. - for version in iterable: - parsed_version = _coerce_version(version) - - if self.contains(parsed_version, **kw): - # If our version is a prerelease, and we were not set to allow - # prereleases, then we'll store it for later in case nothing - # else matches this specifier. - if parsed_version.is_prerelease and not ( - prereleases or self.prereleases - ): - found_prereleases.append(version) - # Either this is not a prerelease, or we should have been - # accepting prereleases from the beginning. - else: - yielded = True - yield version - - # Now that we've iterated over everything, determine if we've yielded - # any values, and if we have not and we have any prereleases stored up - # then we will go ahead and yield the prereleases. - if not yielded and found_prereleases: - for version in found_prereleases: - yield version - - -_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") - - -def _version_split(version: str) -> List[str]: - """Split version into components. - - The split components are intended for version comparison. The logic does - not attempt to retain the original version string, so joining the - components back with :func:`_version_join` may not produce the original - version string. - """ - result: List[str] = [] - - epoch, _, rest = version.rpartition("!") - result.append(epoch or "0") - - for item in rest.split("."): - match = _prefix_regex.search(item) - if match: - result.extend(match.groups()) - else: - result.append(item) - return result - - -def _version_join(components: List[str]) -> str: - """Join split version components into a version string. - - This function assumes the input came from :func:`_version_split`, where the - first component must be the epoch (either empty or numeric), and all other - components numeric. - """ - epoch, *rest = components - return f"{epoch}!{'.'.join(rest)}" - - -def _is_not_suffix(segment: str) -> bool: - return not any( - segment.startswith(prefix) for prefix in ("dev", "a", "b", "rc", "post") - ) - - -def _pad_version(left: List[str], right: List[str]) -> Tuple[List[str], List[str]]: - left_split, right_split = [], [] - - # Get the release segment of our versions - left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) - right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) - - # Get the rest of our versions - left_split.append(left[len(left_split[0]) :]) - right_split.append(right[len(right_split[0]) :]) - - # Insert our padding - left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) - right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - - return ( - list(itertools.chain.from_iterable(left_split)), - list(itertools.chain.from_iterable(right_split)), - ) - - -class SpecifierSet(BaseSpecifier): - """This class abstracts handling of a set of version specifiers. - - It can be passed a single specifier (``>=3.0``), a comma-separated list of - specifiers (``>=3.0,!=3.1``), or no specifier at all. - """ - - def __init__( - self, specifiers: str = "", prereleases: Optional[bool] = None - ) -> None: - """Initialize a SpecifierSet instance. - - :param specifiers: - The string representation of a specifier or a comma-separated list of - specifiers which will be parsed and normalized before use. - :param prereleases: - This tells the SpecifierSet if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - - :raises InvalidSpecifier: - If the given ``specifiers`` are not parseable than this exception will be - raised. - """ - - # Split on `,` to break each individual specifier into it's own item, and - # strip each item to remove leading/trailing whitespace. - split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] - - # Make each individual specifier a Specifier and save in a frozen set for later. - self._specs = frozenset(map(Specifier, split_specifiers)) - - # Store our prereleases value so we can use it later to determine if - # we accept prereleases or not. - self._prereleases = prereleases - - @property - def prereleases(self) -> Optional[bool]: - # If we have been given an explicit prerelease modifier, then we'll - # pass that through here. - if self._prereleases is not None: - return self._prereleases - - # If we don't have any specifiers, and we don't have a forced value, - # then we'll just return None since we don't know if this should have - # pre-releases or not. - if not self._specs: - return None - - # Otherwise we'll see if any of the given specifiers accept - # prereleases, if any of them do we'll return True, otherwise False. - return any(s.prereleases for s in self._specs) - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - def __repr__(self) -> str: - """A representation of the specifier set that shows all internal state. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> SpecifierSet('>=1.0.0,!=2.0.0') - <SpecifierSet('!=2.0.0,>=1.0.0')> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=False) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=False)> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=True) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<SpecifierSet({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the specifier set that can be round-tripped. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> str(SpecifierSet(">=1.0.0,!=1.0.1")) - '!=1.0.1,>=1.0.0' - >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) - '!=1.0.1,>=1.0.0' - """ - return ",".join(sorted(str(s) for s in self._specs)) - - def __hash__(self) -> int: - return hash(self._specs) - - def __and__(self, other: Union["SpecifierSet", str]) -> "SpecifierSet": - """Return a SpecifierSet which is a combination of the two sets. - - :param other: The other object to combine with. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") & '<=2.0.0,!=2.0.1' - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - >>> SpecifierSet(">=1.0.0,!=1.0.1") & SpecifierSet('<=2.0.0,!=2.0.1') - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - """ - if isinstance(other, str): - other = SpecifierSet(other) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - specifier = SpecifierSet() - specifier._specs = frozenset(self._specs | other._specs) - - if self._prereleases is None and other._prereleases is not None: - specifier._prereleases = other._prereleases - elif self._prereleases is not None and other._prereleases is None: - specifier._prereleases = self._prereleases - elif self._prereleases == other._prereleases: - specifier._prereleases = self._prereleases - else: - raise ValueError( - "Cannot combine SpecifierSets with True and False prerelease " - "overrides." - ) - - return specifier - - def __eq__(self, other: object) -> bool: - """Whether or not the two SpecifierSet-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> (SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False) == - ... SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True)) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == ">=1.0.0,!=1.0.1" - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.2") - False - """ - if isinstance(other, (str, Specifier)): - other = SpecifierSet(str(other)) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - return self._specs == other._specs - - def __len__(self) -> int: - """Returns the number of specifiers in this specifier set.""" - return len(self._specs) - - def __iter__(self) -> Iterator[Specifier]: - """ - Returns an iterator over all the underlying :class:`Specifier` instances - in this specifier set. - - >>> sorted(SpecifierSet(">=1.0.0,!=1.0.1"), key=str) - [<Specifier('!=1.0.1')>, <Specifier('>=1.0.0')>] - """ - return iter(self._specs) - - def __contains__(self, item: UnparsedVersion) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> Version("1.2.3") in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> "1.0.1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, - item: UnparsedVersion, - prereleases: Optional[bool] = None, - installed: Optional[bool] = None, - ) -> bool: - """Return whether or not the item is contained in this SpecifierSet. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this SpecifierSet. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.2.3") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains(Version("1.2.3")) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.0.1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True).contains("1.3.0a1") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1", prereleases=True) - True - """ - # Ensure that our item is a Version instance. - if not isinstance(item, Version): - item = Version(item) - - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # We can determine if we're going to allow pre-releases by looking to - # see if any of the underlying items supports them. If none of them do - # and this item is a pre-release then we do not allow it and we can - # short circuit that here. - # Note: This means that 1.0.dev1 would not be contained in something - # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 - if not prereleases and item.is_prerelease: - return False - - if installed and item.is_prerelease: - item = Version(item.base_version) - - # We simply dispatch to the underlying specs here to make sure that the - # given version is contained within all of them. - # Note: This use of all() here means that an empty set of specifiers - # will always return True, this is an explicit design decision. - return all(s.contains(item, prereleases=prereleases) for s in self._specs) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifiers in this set. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(SpecifierSet(...).contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", Version("1.4")])) - ['1.3', <Version('1.4')>] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.5a1"])) - [] - >>> list(SpecifierSet(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - - An "empty" SpecifierSet will filter items based on the presence of prerelease - versions in the set. - - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet("").filter(["1.5a1"])) - ['1.5a1'] - >>> list(SpecifierSet("", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - """ - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # If we have any specifiers, then we want to wrap our iterable in the - # filter method for each one, this will act as a logical AND amongst - # each specifier. - if self._specs: - for spec in self._specs: - iterable = spec.filter(iterable, prereleases=bool(prereleases)) - return iter(iterable) - # If we do not have any specifiers, then we need to have a rough filter - # which will filter out any pre-releases, unless there are no final - # releases. - else: - filtered: List[UnparsedVersionVar] = [] - found_prereleases: List[UnparsedVersionVar] = [] - - for item in iterable: - parsed_version = _coerce_version(item) - - # Store any item which is a pre-release for later unless we've - # already found a final version or we are accepting prereleases - if parsed_version.is_prerelease and not prereleases: - if not filtered: - found_prereleases.append(item) - else: - filtered.append(item) - - # If we've found no items except for pre-releases, then we'll go - # ahead and use the pre-releases - if not filtered and found_prereleases and prereleases is None: - return iter(found_prereleases) - - return iter(filtered) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py deleted file mode 100644 index 89f1926137d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py +++ /dev/null @@ -1,571 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import logging -import platform -import re -import struct -import subprocess -import sys -import sysconfig -from importlib.machinery import EXTENSION_SUFFIXES -from typing import ( - Dict, - FrozenSet, - Iterable, - Iterator, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) - -from . import _manylinux, _musllinux - -logger = logging.getLogger(__name__) - -PythonVersion = Sequence[int] -MacVersion = Tuple[int, int] - -INTERPRETER_SHORT_NAMES: Dict[str, str] = { - "python": "py", # Generic. - "cpython": "cp", - "pypy": "pp", - "ironpython": "ip", - "jython": "jy", -} - - -_32_BIT_INTERPRETER = struct.calcsize("P") == 4 - - -class Tag: - """ - A representation of the tag triple for a wheel. - - Instances are considered immutable and thus are hashable. Equality checking - is also supported. - """ - - __slots__ = ["_interpreter", "_abi", "_platform", "_hash"] - - def __init__(self, interpreter: str, abi: str, platform: str) -> None: - self._interpreter = interpreter.lower() - self._abi = abi.lower() - self._platform = platform.lower() - # The __hash__ of every single element in a Set[Tag] will be evaluated each time - # that a set calls its `.disjoint()` method, which may be called hundreds of - # times when scanning a page of links for packages with tags matching that - # Set[Tag]. Pre-computing the value here produces significant speedups for - # downstream consumers. - self._hash = hash((self._interpreter, self._abi, self._platform)) - - @property - def interpreter(self) -> str: - return self._interpreter - - @property - def abi(self) -> str: - return self._abi - - @property - def platform(self) -> str: - return self._platform - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Tag): - return NotImplemented - - return ( - (self._hash == other._hash) # Short-circuit ASAP for perf reasons. - and (self._platform == other._platform) - and (self._abi == other._abi) - and (self._interpreter == other._interpreter) - ) - - def __hash__(self) -> int: - return self._hash - - def __str__(self) -> str: - return f"{self._interpreter}-{self._abi}-{self._platform}" - - def __repr__(self) -> str: - return f"<{self} @ {id(self)}>" - - -def parse_tag(tag: str) -> FrozenSet[Tag]: - """ - Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. - - Returning a set is required due to the possibility that the tag is a - compressed tag set. - """ - tags = set() - interpreters, abis, platforms = tag.split("-") - for interpreter in interpreters.split("."): - for abi in abis.split("."): - for platform_ in platforms.split("."): - tags.add(Tag(interpreter, abi, platform_)) - return frozenset(tags) - - -def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]: - value: Union[int, str, None] = sysconfig.get_config_var(name) - if value is None and warn: - logger.debug( - "Config variable '%s' is unset, Python ABI tag may be incorrect", name - ) - return value - - -def _normalize_string(string: str) -> str: - return string.replace(".", "_").replace("-", "_").replace(" ", "_") - - -def _is_threaded_cpython(abis: List[str]) -> bool: - """ - Determine if the ABI corresponds to a threaded (`--disable-gil`) build. - - The threaded builds are indicated by a "t" in the abiflags. - """ - if len(abis) == 0: - return False - # expect e.g., cp313 - m = re.match(r"cp\d+(.*)", abis[0]) - if not m: - return False - abiflags = m.group(1) - return "t" in abiflags - - -def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: - """ - Determine if the Python version supports abi3. - - PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) - builds do not support abi3. - """ - return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading - - -def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> List[str]: - py_version = tuple(py_version) # To allow for version comparison. - abis = [] - version = _version_nodot(py_version[:2]) - threading = debug = pymalloc = ucs4 = "" - with_debug = _get_config_var("Py_DEBUG", warn) - has_refcount = hasattr(sys, "gettotalrefcount") - # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled - # extension modules is the best option. - # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 - has_ext = "_d.pyd" in EXTENSION_SUFFIXES - if with_debug or (with_debug is None and (has_refcount or has_ext)): - debug = "d" - if py_version >= (3, 13) and _get_config_var("Py_GIL_DISABLED", warn): - threading = "t" - if py_version < (3, 8): - with_pymalloc = _get_config_var("WITH_PYMALLOC", warn) - if with_pymalloc or with_pymalloc is None: - pymalloc = "m" - if py_version < (3, 3): - unicode_size = _get_config_var("Py_UNICODE_SIZE", warn) - if unicode_size == 4 or ( - unicode_size is None and sys.maxunicode == 0x10FFFF - ): - ucs4 = "u" - elif debug: - # Debug builds can also load "normal" extension modules. - # We can also assume no UCS-4 or pymalloc requirement. - abis.append(f"cp{version}{threading}") - abis.insert(0, f"cp{version}{threading}{debug}{pymalloc}{ucs4}") - return abis - - -def cpython_tags( - python_version: Optional[PythonVersion] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a CPython interpreter. - - The tags consist of: - - cp<python_version>-<abi>-<platform> - - cp<python_version>-abi3-<platform> - - cp<python_version>-none-<platform> - - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. - - If python_version only specifies a major version then user-provided ABIs and - the 'none' ABItag will be used. - - If 'abi3' or 'none' are specified in 'abis' then they will be yielded at - their normal position and not at the beginning. - """ - if not python_version: - python_version = sys.version_info[:2] - - interpreter = f"cp{_version_nodot(python_version[:2])}" - - if abis is None: - if len(python_version) > 1: - abis = _cpython_abis(python_version, warn) - else: - abis = [] - abis = list(abis) - # 'abi3' and 'none' are explicitly handled later. - for explicit_abi in ("abi3", "none"): - try: - abis.remove(explicit_abi) - except ValueError: - pass - - platforms = list(platforms or platform_tags()) - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - threading = _is_threaded_cpython(abis) - use_abi3 = _abi3_applies(python_version, threading) - if use_abi3: - yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) - yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) - - if use_abi3: - for minor_version in range(python_version[1] - 1, 1, -1): - for platform_ in platforms: - interpreter = "cp{version}".format( - version=_version_nodot((python_version[0], minor_version)) - ) - yield Tag(interpreter, "abi3", platform_) - - -def _generic_abi() -> List[str]: - """ - Return the ABI tag based on EXT_SUFFIX. - """ - # The following are examples of `EXT_SUFFIX`. - # We want to keep the parts which are related to the ABI and remove the - # parts which are related to the platform: - # - linux: '.cpython-310-x86_64-linux-gnu.so' => cp310 - # - mac: '.cpython-310-darwin.so' => cp310 - # - win: '.cp310-win_amd64.pyd' => cp310 - # - win: '.pyd' => cp37 (uses _cpython_abis()) - # - pypy: '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73 - # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib' - # => graalpy_38_native - - ext_suffix = _get_config_var("EXT_SUFFIX", warn=True) - if not isinstance(ext_suffix, str) or ext_suffix[0] != ".": - raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')") - parts = ext_suffix.split(".") - if len(parts) < 3: - # CPython3.7 and earlier uses ".pyd" on Windows. - return _cpython_abis(sys.version_info[:2]) - soabi = parts[1] - if soabi.startswith("cpython"): - # non-windows - abi = "cp" + soabi.split("-")[1] - elif soabi.startswith("cp"): - # windows - abi = soabi.split("-")[0] - elif soabi.startswith("pypy"): - abi = "-".join(soabi.split("-")[:2]) - elif soabi.startswith("graalpy"): - abi = "-".join(soabi.split("-")[:3]) - elif soabi: - # pyston, ironpython, others? - abi = soabi - else: - return [] - return [_normalize_string(abi)] - - -def generic_tags( - interpreter: Optional[str] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a generic interpreter. - - The tags consist of: - - <interpreter>-<abi>-<platform> - - The "none" ABI will be added if it was not explicitly provided. - """ - if not interpreter: - interp_name = interpreter_name() - interp_version = interpreter_version(warn=warn) - interpreter = "".join([interp_name, interp_version]) - if abis is None: - abis = _generic_abi() - else: - abis = list(abis) - platforms = list(platforms or platform_tags()) - if "none" not in abis: - abis.append("none") - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - -def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]: - """ - Yields Python versions in descending order. - - After the latest version, the major-only version will be yielded, and then - all previous versions of that major version. - """ - if len(py_version) > 1: - yield f"py{_version_nodot(py_version[:2])}" - yield f"py{py_version[0]}" - if len(py_version) > 1: - for minor in range(py_version[1] - 1, -1, -1): - yield f"py{_version_nodot((py_version[0], minor))}" - - -def compatible_tags( - python_version: Optional[PythonVersion] = None, - interpreter: Optional[str] = None, - platforms: Optional[Iterable[str]] = None, -) -> Iterator[Tag]: - """ - Yields the sequence of tags that are compatible with a specific version of Python. - - The tags consist of: - - py*-none-<platform> - - <interpreter>-none-any # ... if `interpreter` is provided. - - py*-none-any - """ - if not python_version: - python_version = sys.version_info[:2] - platforms = list(platforms or platform_tags()) - for version in _py_interpreter_range(python_version): - for platform_ in platforms: - yield Tag(version, "none", platform_) - if interpreter: - yield Tag(interpreter, "none", "any") - for version in _py_interpreter_range(python_version): - yield Tag(version, "none", "any") - - -def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str: - if not is_32bit: - return arch - - if arch.startswith("ppc"): - return "ppc" - - return "i386" - - -def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> List[str]: - formats = [cpu_arch] - if cpu_arch == "x86_64": - if version < (10, 4): - return [] - formats.extend(["intel", "fat64", "fat32"]) - - elif cpu_arch == "i386": - if version < (10, 4): - return [] - formats.extend(["intel", "fat32", "fat"]) - - elif cpu_arch == "ppc64": - # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? - if version > (10, 5) or version < (10, 4): - return [] - formats.append("fat64") - - elif cpu_arch == "ppc": - if version > (10, 6): - return [] - formats.extend(["fat32", "fat"]) - - if cpu_arch in {"arm64", "x86_64"}: - formats.append("universal2") - - if cpu_arch in {"x86_64", "i386", "ppc64", "ppc", "intel"}: - formats.append("universal") - - return formats - - -def mac_platforms( - version: Optional[MacVersion] = None, arch: Optional[str] = None -) -> Iterator[str]: - """ - Yields the platform tags for a macOS system. - - The `version` parameter is a two-item tuple specifying the macOS version to - generate platform tags for. The `arch` parameter is the CPU architecture to - generate platform tags for. Both parameters default to the appropriate value - for the current system. - """ - version_str, _, cpu_arch = platform.mac_ver() - if version is None: - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - if version == (10, 16): - # When built against an older macOS SDK, Python will report macOS 10.16 - # instead of the real version. - version_str = subprocess.run( - [ - sys.executable, - "-sS", - "-c", - "import platform; print(platform.mac_ver()[0])", - ], - check=True, - env={"SYSTEM_VERSION_COMPAT": "0"}, - stdout=subprocess.PIPE, - text=True, - ).stdout - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - else: - version = version - if arch is None: - arch = _mac_arch(cpu_arch) - else: - arch = arch - - if (10, 0) <= version and version < (11, 0): - # Prior to Mac OS 11, each yearly release of Mac OS bumped the - # "minor" version number. The major version was always 10. - for minor_version in range(version[1], -1, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=10, minor=minor_version, binary_format=binary_format - ) - - if version >= (11, 0): - # Starting with Mac OS 11, each yearly release bumps the major version - # number. The minor versions are now the midyear updates. - for major_version in range(version[0], 10, -1): - compat_version = major_version, 0 - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=major_version, minor=0, binary_format=binary_format - ) - - if version >= (11, 0): - # Mac OS 11 on x86_64 is compatible with binaries from previous releases. - # Arm64 support was introduced in 11.0, so no Arm binaries from previous - # releases exist. - # - # However, the "universal2" binary format can have a - # macOS version earlier than 11.0 when the x86_64 part of the binary supports - # that version of macOS. - if arch == "x86_64": - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - else: - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_format = "universal2" - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - - -def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: - linux = _normalize_string(sysconfig.get_platform()) - if not linux.startswith("linux_"): - # we should never be here, just yield the sysconfig one and return - yield linux - return - if is_32bit: - if linux == "linux_x86_64": - linux = "linux_i686" - elif linux == "linux_aarch64": - linux = "linux_armv8l" - _, arch = linux.split("_", 1) - archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch]) - yield from _manylinux.platform_tags(archs) - yield from _musllinux.platform_tags(archs) - for arch in archs: - yield f"linux_{arch}" - - -def _generic_platforms() -> Iterator[str]: - yield _normalize_string(sysconfig.get_platform()) - - -def platform_tags() -> Iterator[str]: - """ - Provides the platform tags for this installation. - """ - if platform.system() == "Darwin": - return mac_platforms() - elif platform.system() == "Linux": - return _linux_platforms() - else: - return _generic_platforms() - - -def interpreter_name() -> str: - """ - Returns the name of the running interpreter. - - Some implementations have a reserved, two-letter abbreviation which will - be returned when appropriate. - """ - name = sys.implementation.name - return INTERPRETER_SHORT_NAMES.get(name) or name - - -def interpreter_version(*, warn: bool = False) -> str: - """ - Returns the version of the running interpreter. - """ - version = _get_config_var("py_version_nodot", warn=warn) - if version: - version = str(version) - else: - version = _version_nodot(sys.version_info[:2]) - return version - - -def _version_nodot(version: PythonVersion) -> str: - return "".join(map(str, version)) - - -def sys_tags(*, warn: bool = False) -> Iterator[Tag]: - """ - Returns the sequence of tag triples for the running interpreter. - - The order of the sequence corresponds to priority order for the - interpreter, from most to least important. - """ - - interp_name = interpreter_name() - if interp_name == "cp": - yield from cpython_tags(warn=warn) - else: - yield from generic_tags() - - if interp_name == "pp": - interp = "pp3" - elif interp_name == "cp": - interp = "cp" + interpreter_version(warn=warn) - else: - interp = None - yield from compatible_tags(interpreter=interp) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py deleted file mode 100644 index c2c2f75aa80..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py +++ /dev/null @@ -1,172 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import re -from typing import FrozenSet, NewType, Tuple, Union, cast - -from .tags import Tag, parse_tag -from .version import InvalidVersion, Version - -BuildTag = Union[Tuple[()], Tuple[int, str]] -NormalizedName = NewType("NormalizedName", str) - - -class InvalidName(ValueError): - """ - An invalid distribution name; users should refer to the packaging user guide. - """ - - -class InvalidWheelFilename(ValueError): - """ - An invalid wheel filename was found, users should refer to PEP 427. - """ - - -class InvalidSdistFilename(ValueError): - """ - An invalid sdist filename was found, users should refer to the packaging user guide. - """ - - -# Core metadata spec for `Name` -_validate_regex = re.compile( - r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE -) -_canonicalize_regex = re.compile(r"[-_.]+") -_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") -# PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") - - -def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: - if validate and not _validate_regex.match(name): - raise InvalidName(f"name is invalid: {name!r}") - # This is taken from PEP 503. - value = _canonicalize_regex.sub("-", name).lower() - return cast(NormalizedName, value) - - -def is_normalized_name(name: str) -> bool: - return _normalized_regex.match(name) is not None - - -def canonicalize_version( - version: Union[Version, str], *, strip_trailing_zero: bool = True -) -> str: - """ - This is very similar to Version.__str__, but has one subtle difference - with the way it handles the release segment. - """ - if isinstance(version, str): - try: - parsed = Version(version) - except InvalidVersion: - # Legacy versions cannot be normalized - return version - else: - parsed = version - - parts = [] - - # Epoch - if parsed.epoch != 0: - parts.append(f"{parsed.epoch}!") - - # Release segment - release_segment = ".".join(str(x) for x in parsed.release) - if strip_trailing_zero: - # NB: This strips trailing '.0's to normalize - release_segment = re.sub(r"(\.0)+$", "", release_segment) - parts.append(release_segment) - - # Pre-release - if parsed.pre is not None: - parts.append("".join(str(x) for x in parsed.pre)) - - # Post-release - if parsed.post is not None: - parts.append(f".post{parsed.post}") - - # Development release - if parsed.dev is not None: - parts.append(f".dev{parsed.dev}") - - # Local version segment - if parsed.local is not None: - parts.append(f"+{parsed.local}") - - return "".join(parts) - - -def parse_wheel_filename( - filename: str, -) -> Tuple[NormalizedName, Version, BuildTag, FrozenSet[Tag]]: - if not filename.endswith(".whl"): - raise InvalidWheelFilename( - f"Invalid wheel filename (extension must be '.whl'): {filename}" - ) - - filename = filename[:-4] - dashes = filename.count("-") - if dashes not in (4, 5): - raise InvalidWheelFilename( - f"Invalid wheel filename (wrong number of parts): {filename}" - ) - - parts = filename.split("-", dashes - 2) - name_part = parts[0] - # See PEP 427 for the rules on escaping the project name. - if "__" in name_part or re.match(r"^[\w\d._]*$", name_part, re.UNICODE) is None: - raise InvalidWheelFilename(f"Invalid project name: {filename}") - name = canonicalize_name(name_part) - - try: - version = Version(parts[1]) - except InvalidVersion as e: - raise InvalidWheelFilename( - f"Invalid wheel filename (invalid version): {filename}" - ) from e - - if dashes == 5: - build_part = parts[2] - build_match = _build_tag_regex.match(build_part) - if build_match is None: - raise InvalidWheelFilename( - f"Invalid build number: {build_part} in '{filename}'" - ) - build = cast(BuildTag, (int(build_match.group(1)), build_match.group(2))) - else: - build = () - tags = parse_tag(parts[-1]) - return (name, version, build, tags) - - -def parse_sdist_filename(filename: str) -> Tuple[NormalizedName, Version]: - if filename.endswith(".tar.gz"): - file_stem = filename[: -len(".tar.gz")] - elif filename.endswith(".zip"): - file_stem = filename[: -len(".zip")] - else: - raise InvalidSdistFilename( - f"Invalid sdist filename (extension must be '.tar.gz' or '.zip'):" - f" {filename}" - ) - - # We are requiring a PEP 440 version, which cannot contain dashes, - # so we split on the last dash. - name_part, sep, version_part = file_stem.rpartition("-") - if not sep: - raise InvalidSdistFilename(f"Invalid sdist filename: {filename}") - - name = canonicalize_name(name_part) - - try: - version = Version(version_part) - except InvalidVersion as e: - raise InvalidSdistFilename( - f"Invalid sdist filename (invalid version): {filename}" - ) from e - - return (name, version) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py deleted file mode 100644 index 5faab9bd0dc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py +++ /dev/null @@ -1,563 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.version import parse, Version -""" - -import itertools -import re -from typing import Any, Callable, NamedTuple, Optional, SupportsInt, Tuple, Union - -from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType - -__all__ = ["VERSION_PATTERN", "parse", "Version", "InvalidVersion"] - -LocalType = Tuple[Union[int, str], ...] - -CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] -CmpLocalType = Union[ - NegativeInfinityType, - Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], -] -CmpKey = Tuple[ - int, - Tuple[int, ...], - CmpPrePostDevType, - CmpPrePostDevType, - CmpPrePostDevType, - CmpLocalType, -] -VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] - - -class _Version(NamedTuple): - epoch: int - release: Tuple[int, ...] - dev: Optional[Tuple[str, int]] - pre: Optional[Tuple[str, int]] - post: Optional[Tuple[str, int]] - local: Optional[LocalType] - - -def parse(version: str) -> "Version": - """Parse the given version string. - - >>> parse('1.0.dev1') - <Version('1.0.dev1')> - - :param version: The version string to parse. - :raises InvalidVersion: When the version string is not a valid version. - """ - return Version(version) - - -class InvalidVersion(ValueError): - """Raised when a version string is not a valid version. - - >>> Version("invalid") - Traceback (most recent call last): - ... - packaging.version.InvalidVersion: Invalid version: 'invalid' - """ - - -class _BaseVersion: - _key: Tuple[Any, ...] - - def __hash__(self) -> int: - return hash(self._key) - - # Please keep the duplicated `isinstance` check - # in the six comparisons hereunder - # unless you find a way to avoid adding overhead function calls. - def __lt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key < other._key - - def __le__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key <= other._key - - def __eq__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key == other._key - - def __ge__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key >= other._key - - def __gt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key > other._key - - def __ne__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key != other._key - - -# Deliberately not anchored to the start and end of the string, to make it -# easier for 3rd party code to reuse -_VERSION_PATTERN = r""" - v? - (?: - (?:(?P<epoch>[0-9]+)!)? # epoch - (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment - (?P<pre> # pre-release - [-_\.]? - (?P<pre_l>alpha|a|beta|b|preview|pre|c|rc) - [-_\.]? - (?P<pre_n>[0-9]+)? - )? - (?P<post> # post release - (?:-(?P<post_n1>[0-9]+)) - | - (?: - [-_\.]? - (?P<post_l>post|rev|r) - [-_\.]? - (?P<post_n2>[0-9]+)? - ) - )? - (?P<dev> # dev release - [-_\.]? - (?P<dev_l>dev) - [-_\.]? - (?P<dev_n>[0-9]+)? - )? - ) - (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version -""" - -VERSION_PATTERN = _VERSION_PATTERN -""" -A string containing the regular expression used to match a valid version. - -The pattern is not anchored at either end, and is intended for embedding in larger -expressions (for example, matching a version number as part of a file name). The -regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` -flags set. - -:meta hide-value: -""" - - -class Version(_BaseVersion): - """This class abstracts handling of a project's versions. - - A :class:`Version` instance is comparison aware and can be compared and - sorted using the standard Python interfaces. - - >>> v1 = Version("1.0a5") - >>> v2 = Version("1.0") - >>> v1 - <Version('1.0a5')> - >>> v2 - <Version('1.0')> - >>> v1 < v2 - True - >>> v1 == v2 - False - >>> v1 > v2 - False - >>> v1 >= v2 - False - >>> v1 <= v2 - True - """ - - _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) - _key: CmpKey - - def __init__(self, version: str) -> None: - """Initialize a Version object. - - :param version: - The string representation of a version which will be parsed and normalized - before use. - :raises InvalidVersion: - If the ``version`` does not conform to PEP 440 in any way then this - exception will be raised. - """ - - # Validate the version and parse it into pieces - match = self._regex.search(version) - if not match: - raise InvalidVersion(f"Invalid version: '{version}'") - - # Store the parsed out pieces of the version - self._version = _Version( - epoch=int(match.group("epoch")) if match.group("epoch") else 0, - release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), - post=_parse_letter_version( - match.group("post_l"), match.group("post_n1") or match.group("post_n2") - ), - dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), - local=_parse_local_version(match.group("local")), - ) - - # Generate a key which will be used for sorting - self._key = _cmpkey( - self._version.epoch, - self._version.release, - self._version.pre, - self._version.post, - self._version.dev, - self._version.local, - ) - - def __repr__(self) -> str: - """A representation of the Version that shows all internal state. - - >>> Version('1.0.0') - <Version('1.0.0')> - """ - return f"<Version('{self}')>" - - def __str__(self) -> str: - """A string representation of the version that can be rounded-tripped. - - >>> str(Version("1.0a5")) - '1.0a5' - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - # Pre-release - if self.pre is not None: - parts.append("".join(str(x) for x in self.pre)) - - # Post-release - if self.post is not None: - parts.append(f".post{self.post}") - - # Development release - if self.dev is not None: - parts.append(f".dev{self.dev}") - - # Local version segment - if self.local is not None: - parts.append(f"+{self.local}") - - return "".join(parts) - - @property - def epoch(self) -> int: - """The epoch of the version. - - >>> Version("2.0.0").epoch - 0 - >>> Version("1!2.0.0").epoch - 1 - """ - return self._version.epoch - - @property - def release(self) -> Tuple[int, ...]: - """The components of the "release" segment of the version. - - >>> Version("1.2.3").release - (1, 2, 3) - >>> Version("2.0.0").release - (2, 0, 0) - >>> Version("1!2.0.0.post0").release - (2, 0, 0) - - Includes trailing zeroes but not the epoch or any pre-release / development / - post-release suffixes. - """ - return self._version.release - - @property - def pre(self) -> Optional[Tuple[str, int]]: - """The pre-release segment of the version. - - >>> print(Version("1.2.3").pre) - None - >>> Version("1.2.3a1").pre - ('a', 1) - >>> Version("1.2.3b1").pre - ('b', 1) - >>> Version("1.2.3rc1").pre - ('rc', 1) - """ - return self._version.pre - - @property - def post(self) -> Optional[int]: - """The post-release number of the version. - - >>> print(Version("1.2.3").post) - None - >>> Version("1.2.3.post1").post - 1 - """ - return self._version.post[1] if self._version.post else None - - @property - def dev(self) -> Optional[int]: - """The development number of the version. - - >>> print(Version("1.2.3").dev) - None - >>> Version("1.2.3.dev1").dev - 1 - """ - return self._version.dev[1] if self._version.dev else None - - @property - def local(self) -> Optional[str]: - """The local version segment of the version. - - >>> print(Version("1.2.3").local) - None - >>> Version("1.2.3+abc").local - 'abc' - """ - if self._version.local: - return ".".join(str(x) for x in self._version.local) - else: - return None - - @property - def public(self) -> str: - """The public portion of the version. - - >>> Version("1.2.3").public - '1.2.3' - >>> Version("1.2.3+abc").public - '1.2.3' - >>> Version("1.2.3+abc.dev1").public - '1.2.3' - """ - return str(self).split("+", 1)[0] - - @property - def base_version(self) -> str: - """The "base version" of the version. - - >>> Version("1.2.3").base_version - '1.2.3' - >>> Version("1.2.3+abc").base_version - '1.2.3' - >>> Version("1!1.2.3+abc.dev1").base_version - '1!1.2.3' - - The "base version" is the public version of the project without any pre or post - release markers. - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - return "".join(parts) - - @property - def is_prerelease(self) -> bool: - """Whether this version is a pre-release. - - >>> Version("1.2.3").is_prerelease - False - >>> Version("1.2.3a1").is_prerelease - True - >>> Version("1.2.3b1").is_prerelease - True - >>> Version("1.2.3rc1").is_prerelease - True - >>> Version("1.2.3dev1").is_prerelease - True - """ - return self.dev is not None or self.pre is not None - - @property - def is_postrelease(self) -> bool: - """Whether this version is a post-release. - - >>> Version("1.2.3").is_postrelease - False - >>> Version("1.2.3.post1").is_postrelease - True - """ - return self.post is not None - - @property - def is_devrelease(self) -> bool: - """Whether this version is a development release. - - >>> Version("1.2.3").is_devrelease - False - >>> Version("1.2.3.dev1").is_devrelease - True - """ - return self.dev is not None - - @property - def major(self) -> int: - """The first item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").major - 1 - """ - return self.release[0] if len(self.release) >= 1 else 0 - - @property - def minor(self) -> int: - """The second item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").minor - 2 - >>> Version("1").minor - 0 - """ - return self.release[1] if len(self.release) >= 2 else 0 - - @property - def micro(self) -> int: - """The third item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").micro - 3 - >>> Version("1").micro - 0 - """ - return self.release[2] if len(self.release) >= 3 else 0 - - -def _parse_letter_version( - letter: Optional[str], number: Union[str, bytes, SupportsInt, None] -) -> Optional[Tuple[str, int]]: - - if letter: - # We consider there to be an implicit 0 in a pre-release if there is - # not a numeral associated with it. - if number is None: - number = 0 - - # We normalize any letters to their lower case form - letter = letter.lower() - - # We consider some words to be alternate spellings of other words and - # in those cases we want to normalize the spellings to our preferred - # spelling. - if letter == "alpha": - letter = "a" - elif letter == "beta": - letter = "b" - elif letter in ["c", "pre", "preview"]: - letter = "rc" - elif letter in ["rev", "r"]: - letter = "post" - - return letter, int(number) - if not letter and number: - # We assume if we are given a number, but we are not given a letter - # then this is using the implicit post release syntax (e.g. 1.0-1) - letter = "post" - - return letter, int(number) - - return None - - -_local_version_separators = re.compile(r"[\._-]") - - -def _parse_local_version(local: Optional[str]) -> Optional[LocalType]: - """ - Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). - """ - if local is not None: - return tuple( - part.lower() if not part.isdigit() else int(part) - for part in _local_version_separators.split(local) - ) - return None - - -def _cmpkey( - epoch: int, - release: Tuple[int, ...], - pre: Optional[Tuple[str, int]], - post: Optional[Tuple[str, int]], - dev: Optional[Tuple[str, int]], - local: Optional[LocalType], -) -> CmpKey: - - # When we compare a release version, we want to compare it with all of the - # trailing zeros removed. So we'll use a reverse the list, drop all the now - # leading zeros until we come to something non zero, then take the rest - # re-reverse it back into the correct order and make it a tuple and use - # that for our sorting key. - _release = tuple( - reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) - ) - - # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. - # We'll do this by abusing the pre segment, but we _only_ want to do this - # if there is not a pre or a post segment. If we have one of those then - # the normal sorting rules will handle this case correctly. - if pre is None and post is None and dev is not None: - _pre: CmpPrePostDevType = NegativeInfinity - # Versions without a pre-release (except as noted above) should sort after - # those with one. - elif pre is None: - _pre = Infinity - else: - _pre = pre - - # Versions without a post segment should sort before those with one. - if post is None: - _post: CmpPrePostDevType = NegativeInfinity - - else: - _post = post - - # Versions without a development segment should sort after those with one. - if dev is None: - _dev: CmpPrePostDevType = Infinity - - else: - _dev = dev - - if local is None: - # Versions without a local segment should sort before those with one. - _local: CmpLocalType = NegativeInfinity - else: - # Versions with a local segment need that segment parsed to implement - # the sorting rules in PEP440. - # - Alpha numeric segments sort before numeric segments - # - Alpha numeric segments sort lexicographically - # - Numeric segments sort numerically - # - Shorter versions sort before longer versions when the prefixes - # match exactly - _local = tuple( - (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local - ) - - return epoch, _release, _pre, _post, _dev, _local diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py deleted file mode 100644 index 4c6ec97ec69..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -__all__ = ("loads", "load", "TOMLDecodeError") -__version__ = "2.0.1" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT - -from ._parser import TOMLDecodeError, load, loads - -# Pretend this exception was created here. -TOMLDecodeError.__module__ = __name__ diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py deleted file mode 100644 index f1bb0aa19a5..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py +++ /dev/null @@ -1,691 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from __future__ import annotations - -from collections.abc import Iterable -import string -from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple - -from ._re import ( - RE_DATETIME, - RE_LOCALTIME, - RE_NUMBER, - match_to_datetime, - match_to_localtime, - match_to_number, -) -from ._types import Key, ParseFloat, Pos - -ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) - -# Neither of these sets include quotation mark or backslash. They are -# currently handled as separate cases in the parser functions. -ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t") -ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n") - -ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS -ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS - -ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS - -TOML_WS = frozenset(" \t") -TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") -KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) - -BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( - { - "\\b": "\u0008", # backspace - "\\t": "\u0009", # tab - "\\n": "\u000A", # linefeed - "\\f": "\u000C", # form feed - "\\r": "\u000D", # carriage return - '\\"': "\u0022", # quote - "\\\\": "\u005C", # backslash - } -) - - -class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" - - -def load(__fp: BinaryIO, *, parse_float: ParseFloat = float) -> dict[str, Any]: - """Parse TOML from a binary file object.""" - b = __fp.read() - try: - s = b.decode() - except AttributeError: - raise TypeError( - "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`" - ) from None - return loads(s, parse_float=parse_float) - - -def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901 - """Parse TOML from a string.""" - - # The spec allows converting "\r\n" to "\n", even in string - # literals. Let's do so to simplify parsing. - src = __s.replace("\r\n", "\n") - pos = 0 - out = Output(NestedDict(), Flags()) - header: Key = () - parse_float = make_safe_parse_float(parse_float) - - # Parse one statement at a time - # (typically means one line in TOML source) - while True: - # 1. Skip line leading whitespace - pos = skip_chars(src, pos, TOML_WS) - - # 2. Parse rules. Expect one of the following: - # - end of file - # - end of line - # - comment - # - key/value pair - # - append dict to list (and move to its namespace) - # - create dict (and move to its namespace) - # Skip trailing whitespace when applicable. - try: - char = src[pos] - except IndexError: - break - if char == "\n": - pos += 1 - continue - if char in KEY_INITIAL_CHARS: - pos = key_value_rule(src, pos, out, header, parse_float) - pos = skip_chars(src, pos, TOML_WS) - elif char == "[": - try: - second_char: str | None = src[pos + 1] - except IndexError: - second_char = None - out.flags.finalize_pending() - if second_char == "[": - pos, header = create_list_rule(src, pos, out) - else: - pos, header = create_dict_rule(src, pos, out) - pos = skip_chars(src, pos, TOML_WS) - elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") - - # 3. Skip comment - pos = skip_comment(src, pos) - - # 4. Expect end of line or end of file - try: - char = src[pos] - except IndexError: - break - if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" - ) - pos += 1 - - return out.data.dict - - -class Flags: - """Flags that map to parsed keys/namespaces.""" - - # Marks an immutable namespace (inline array or inline table). - FROZEN = 0 - # Marks a nest that has been explicitly created and can no longer - # be opened using the "[table]" syntax. - EXPLICIT_NEST = 1 - - def __init__(self) -> None: - self._flags: dict[str, dict] = {} - self._pending_flags: set[tuple[Key, int]] = set() - - def add_pending(self, key: Key, flag: int) -> None: - self._pending_flags.add((key, flag)) - - def finalize_pending(self) -> None: - for key, flag in self._pending_flags: - self.set(key, flag, recursive=False) - self._pending_flags.clear() - - def unset_all(self, key: Key) -> None: - cont = self._flags - for k in key[:-1]: - if k not in cont: - return - cont = cont[k]["nested"] - cont.pop(key[-1], None) - - def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003 - cont = self._flags - key_parent, key_stem = key[:-1], key[-1] - for k in key_parent: - if k not in cont: - cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} - cont = cont[k]["nested"] - if key_stem not in cont: - cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}} - cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag) - - def is_(self, key: Key, flag: int) -> bool: - if not key: - return False # document root has no flags - cont = self._flags - for k in key[:-1]: - if k not in cont: - return False - inner_cont = cont[k] - if flag in inner_cont["recursive_flags"]: - return True - cont = inner_cont["nested"] - key_stem = key[-1] - if key_stem in cont: - cont = cont[key_stem] - return flag in cont["flags"] or flag in cont["recursive_flags"] - return False - - -class NestedDict: - def __init__(self) -> None: - # The parsed content of the TOML document - self.dict: dict[str, Any] = {} - - def get_or_create_nest( - self, - key: Key, - *, - access_lists: bool = True, - ) -> dict: - cont: Any = self.dict - for k in key: - if k not in cont: - cont[k] = {} - cont = cont[k] - if access_lists and isinstance(cont, list): - cont = cont[-1] - if not isinstance(cont, dict): - raise KeyError("There is no nest behind this key") - return cont - - def append_nest_to_list(self, key: Key) -> None: - cont = self.get_or_create_nest(key[:-1]) - last_key = key[-1] - if last_key in cont: - list_ = cont[last_key] - if not isinstance(list_, list): - raise KeyError("An object other than list found behind this key") - list_.append({}) - else: - cont[last_key] = [{}] - - -class Output(NamedTuple): - data: NestedDict - flags: Flags - - -def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: - try: - while src[pos] in chars: - pos += 1 - except IndexError: - pass - return pos - - -def skip_until( - src: str, - pos: Pos, - expect: str, - *, - error_on: frozenset[str], - error_on_eof: bool, -) -> Pos: - try: - new_pos = src.index(expect, pos) - except ValueError: - new_pos = len(src) - if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None - - if not error_on.isdisjoint(src[pos:new_pos]): - while src[pos] not in error_on: - pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") - return new_pos - - -def skip_comment(src: str, pos: Pos) -> Pos: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char == "#": - return skip_until( - src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False - ) - return pos - - -def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos: - while True: - pos_before_skip = pos - pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) - pos = skip_comment(src, pos) - if pos == pos_before_skip: - return pos - - -def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: - pos += 1 # Skip "[" - pos = skip_chars(src, pos, TOML_WS) - pos, key = parse_key(src, pos) - - if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") - out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) - try: - out.data.get_or_create_nest(key) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - - if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") - return pos + 1, key - - -def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: - pos += 2 # Skip "[[" - pos = skip_chars(src, pos, TOML_WS) - pos, key = parse_key(src, pos) - - if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") - # Free the namespace now that it points to another empty list item... - out.flags.unset_all(key) - # ...but this key precisely is still prohibited from table declaration - out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) - try: - out.data.append_nest_to_list(key) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - - if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") - return pos + 2, key - - -def key_value_rule( - src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat -) -> Pos: - pos, key, value = parse_key_value_pair(src, pos, parse_float) - key_parent, key_stem = key[:-1], key[-1] - abs_key_parent = header + key_parent - - relative_path_cont_keys = (header + key[:i] for i in range(1, len(key))) - for cont_key in relative_path_cont_keys: - # Check that dotted key syntax does not redefine an existing table - if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") - # Containers in the relative path can't be opened with the table syntax or - # dotted key/value syntax in following table sections. - out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) - - if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" - ) - - try: - nest = out.data.get_or_create_nest(abs_key_parent) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") - # Mark inline table and array namespaces recursively immutable - if isinstance(value, (dict, list)): - out.flags.set(header + key, Flags.FROZEN, recursive=True) - nest[key_stem] = value - return pos - - -def parse_key_value_pair( - src: str, pos: Pos, parse_float: ParseFloat -) -> tuple[Pos, Key, Any]: - pos, key = parse_key(src, pos) - try: - char: str | None = src[pos] - except IndexError: - char = None - if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - pos, value = parse_value(src, pos, parse_float) - return pos, key, value - - -def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: - pos, key_part = parse_key_part(src, pos) - key: Key = (key_part,) - pos = skip_chars(src, pos, TOML_WS) - while True: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char != ".": - return pos, key - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - pos, key_part = parse_key_part(src, pos) - key += (key_part,) - pos = skip_chars(src, pos, TOML_WS) - - -def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char in BARE_KEY_CHARS: - start_pos = pos - pos = skip_chars(src, pos, BARE_KEY_CHARS) - return pos, src[start_pos:pos] - if char == "'": - return parse_literal_str(src, pos) - if char == '"': - return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") - - -def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: - pos += 1 - return parse_basic_str(src, pos, multiline=False) - - -def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]: - pos += 1 - array: list = [] - - pos = skip_comments_and_array_ws(src, pos) - if src.startswith("]", pos): - return pos + 1, array - while True: - pos, val = parse_value(src, pos, parse_float) - array.append(val) - pos = skip_comments_and_array_ws(src, pos) - - c = src[pos : pos + 1] - if c == "]": - return pos + 1, array - if c != ",": - raise suffixed_err(src, pos, "Unclosed array") - pos += 1 - - pos = skip_comments_and_array_ws(src, pos) - if src.startswith("]", pos): - return pos + 1, array - - -def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]: - pos += 1 - nested_dict = NestedDict() - flags = Flags() - - pos = skip_chars(src, pos, TOML_WS) - if src.startswith("}", pos): - return pos + 1, nested_dict.dict - while True: - pos, key, value = parse_key_value_pair(src, pos, parse_float) - key_parent, key_stem = key[:-1], key[-1] - if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") - try: - nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") - nest[key_stem] = value - pos = skip_chars(src, pos, TOML_WS) - c = src[pos : pos + 1] - if c == "}": - return pos + 1, nested_dict.dict - if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") - if isinstance(value, (dict, list)): - flags.set(key, Flags.FROZEN, recursive=True) - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - - -def parse_basic_str_escape( - src: str, pos: Pos, *, multiline: bool = False -) -> tuple[Pos, str]: - escape_id = src[pos : pos + 2] - pos += 2 - if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}: - # Skip whitespace until next non-whitespace character or end of - # the doc. Error if non-whitespace is found before newline. - if escape_id != "\\\n": - pos = skip_chars(src, pos, TOML_WS) - try: - char = src[pos] - except IndexError: - return pos, "" - if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") - pos += 1 - pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) - return pos, "" - if escape_id == "\\u": - return parse_hex_char(src, pos, 4) - if escape_id == "\\U": - return parse_hex_char(src, pos, 8) - try: - return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] - except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None - - -def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: - return parse_basic_str_escape(src, pos, multiline=True) - - -def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: - hex_str = src[pos : pos + hex_len] - if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") - pos += hex_len - hex_int = int(hex_str, 16) - if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") - return pos, chr(hex_int) - - -def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]: - pos += 1 # Skip starting apostrophe - start_pos = pos - pos = skip_until( - src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True - ) - return pos + 1, src[start_pos:pos] # Skip ending apostrophe - - -def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]: - pos += 3 - if src.startswith("\n", pos): - pos += 1 - - if literal: - delim = "'" - end_pos = skip_until( - src, - pos, - "'''", - error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS, - error_on_eof=True, - ) - result = src[pos:end_pos] - pos = end_pos + 3 - else: - delim = '"' - pos, result = parse_basic_str(src, pos, multiline=True) - - # Add at maximum two extra apostrophes/quotes if the end sequence - # is 4 or 5 chars long instead of just 3. - if not src.startswith(delim, pos): - return pos, result - pos += 1 - if not src.startswith(delim, pos): - return pos, result + delim - pos += 1 - return pos, result + (delim * 2) - - -def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: - if multiline: - error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS - parse_escapes = parse_basic_str_escape_multiline - else: - error_on = ILLEGAL_BASIC_STR_CHARS - parse_escapes = parse_basic_str_escape - result = "" - start_pos = pos - while True: - try: - char = src[pos] - except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None - if char == '"': - if not multiline: - return pos + 1, result + src[start_pos:pos] - if src.startswith('"""', pos): - return pos + 3, result + src[start_pos:pos] - pos += 1 - continue - if char == "\\": - result += src[start_pos:pos] - pos, parsed_escape = parse_escapes(src, pos) - result += parsed_escape - start_pos = pos - continue - if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") - pos += 1 - - -def parse_value( # noqa: C901 - src: str, pos: Pos, parse_float: ParseFloat -) -> tuple[Pos, Any]: - try: - char: str | None = src[pos] - except IndexError: - char = None - - # IMPORTANT: order conditions based on speed of checking and likelihood - - # Basic strings - if char == '"': - if src.startswith('"""', pos): - return parse_multiline_str(src, pos, literal=False) - return parse_one_line_basic_str(src, pos) - - # Literal strings - if char == "'": - if src.startswith("'''", pos): - return parse_multiline_str(src, pos, literal=True) - return parse_literal_str(src, pos) - - # Booleans - if char == "t": - if src.startswith("true", pos): - return pos + 4, True - if char == "f": - if src.startswith("false", pos): - return pos + 5, False - - # Arrays - if char == "[": - return parse_array(src, pos, parse_float) - - # Inline tables - if char == "{": - return parse_inline_table(src, pos, parse_float) - - # Dates and times - datetime_match = RE_DATETIME.match(src, pos) - if datetime_match: - try: - datetime_obj = match_to_datetime(datetime_match) - except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e - return datetime_match.end(), datetime_obj - localtime_match = RE_LOCALTIME.match(src, pos) - if localtime_match: - return localtime_match.end(), match_to_localtime(localtime_match) - - # Integers and "normal" floats. - # The regex will greedily match any type starting with a decimal - # char, so needs to be located after handling of dates and times. - number_match = RE_NUMBER.match(src, pos) - if number_match: - return number_match.end(), match_to_number(number_match, parse_float) - - # Special floats - first_three = src[pos : pos + 3] - if first_three in {"inf", "nan"}: - return pos + 3, parse_float(first_three) - first_four = src[pos : pos + 4] - if first_four in {"-inf", "+inf", "-nan", "+nan"}: - return pos + 4, parse_float(first_four) - - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") - - -def is_unicode_scalar_value(codepoint: int) -> bool: - return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111) - - -def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat: - """A decorator to make `parse_float` safe. - - `parse_float` must not return dicts or lists, because these types - would be mixed with parsed TOML tables and arrays, thus confusing - the parser. The returned decorated callable raises `ValueError` - instead of returning illegal types. - """ - # The default `float` callable never returns illegal types. Optimize it. - if parse_float is float: # type: ignore[comparison-overlap] - return float - - def safe_parse_float(float_str: str) -> Any: - float_value = parse_float(float_str) - if isinstance(float_value, (dict, list)): - raise ValueError("parse_float must not return dicts or lists") - return float_value - - return safe_parse_float diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py deleted file mode 100644 index 994bb7493fd..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from __future__ import annotations - -from datetime import date, datetime, time, timedelta, timezone, tzinfo -from functools import lru_cache -import re -from typing import Any - -from ._types import ParseFloat - -# E.g. -# - 00:32:00.999999 -# - 00:32:00 -_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?" - -RE_NUMBER = re.compile( - r""" -0 -(?: - x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex - | - b[01](?:_?[01])* # bin - | - o[0-7](?:_?[0-7])* # oct -) -| -[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part -(?P<floatpart> - (?:\.[0-9](?:_?[0-9])*)? # optional fractional part - (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part -) -""", - flags=re.VERBOSE, -) -RE_LOCALTIME = re.compile(_TIME_RE_STR) -RE_DATETIME = re.compile( - rf""" -([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27 -(?: - [Tt ] - {_TIME_RE_STR} - (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset -)? -""", - flags=re.VERBOSE, -) - - -def match_to_datetime(match: re.Match) -> datetime | date: - """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. - - Raises ValueError if the match does not correspond to a valid date - or datetime. - """ - ( - year_str, - month_str, - day_str, - hour_str, - minute_str, - sec_str, - micros_str, - zulu_time, - offset_sign_str, - offset_hour_str, - offset_minute_str, - ) = match.groups() - year, month, day = int(year_str), int(month_str), int(day_str) - if hour_str is None: - return date(year, month, day) - hour, minute, sec = int(hour_str), int(minute_str), int(sec_str) - micros = int(micros_str.ljust(6, "0")) if micros_str else 0 - if offset_sign_str: - tz: tzinfo | None = cached_tz( - offset_hour_str, offset_minute_str, offset_sign_str - ) - elif zulu_time: - tz = timezone.utc - else: # local date-time - tz = None - return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) - - -@lru_cache(maxsize=None) -def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: - sign = 1 if sign_str == "+" else -1 - return timezone( - timedelta( - hours=sign * int(hour_str), - minutes=sign * int(minute_str), - ) - ) - - -def match_to_localtime(match: re.Match) -> time: - hour_str, minute_str, sec_str, micros_str = match.groups() - micros = int(micros_str.ljust(6, "0")) if micros_str else 0 - return time(int(hour_str), int(minute_str), int(sec_str), micros) - - -def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any: - if match.group("floatpart"): - return parse_float(match.group()) - return int(match.group(), 0) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py deleted file mode 100644 index d949412e03b..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from typing import Any, Callable, Tuple - -# Type annotations -ParseFloat = Callable[[str], Any] -Key = Tuple[str, ...] -Pos = int diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed deleted file mode 100644 index 7632ecf7754..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed +++ /dev/null @@ -1 +0,0 @@ -# Marker file for PEP 561 diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/wheel/__init__.py deleted file mode 100644 index a773bbbcd7d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import annotations - -__version__ = "0.43.0" diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/macosx_libfile.py b/contrib/python/setuptools/py3/setuptools/_vendor/wheel/macosx_libfile.py deleted file mode 100644 index 8953c3f8051..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/macosx_libfile.py +++ /dev/null @@ -1,469 +0,0 @@ -""" -This module contains function to analyse dynamic library -headers to extract system information - -Currently only for MacOSX - -Library file on macosx system starts with Mach-O or Fat field. -This can be distinguish by first 32 bites and it is called magic number. -Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means -reversed bytes order. -Both fields can occur in two types: 32 and 64 bytes. - -FAT field inform that this library contains few version of library -(typically for different types version). It contains -information where Mach-O headers starts. - -Each section started with Mach-O header contains one library -(So if file starts with this field it contains only one version). - -After filed Mach-O there are section fields. -Each of them starts with two fields: -cmd - magic number for this command -cmdsize - total size occupied by this section information. - -In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier) -and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting, -because them contains information about minimal system version. - -Important remarks: -- For fat files this implementation looks for maximum number version. - It not check if it is 32 or 64 and do not compare it with currently built package. - So it is possible to false report higher version that needed. -- All structures signatures are taken form macosx header files. -- I think that binary format will be more stable than `otool` output. - and if apple introduce some changes both implementation will need to be updated. -- The system compile will set the deployment target no lower than - 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment - target when the arm64 target is 11.0. -""" - -from __future__ import annotations - -import ctypes -import os -import sys - -"""here the needed const and struct from mach-o header files""" - -FAT_MAGIC = 0xCAFEBABE -FAT_CIGAM = 0xBEBAFECA -FAT_MAGIC_64 = 0xCAFEBABF -FAT_CIGAM_64 = 0xBFBAFECA -MH_MAGIC = 0xFEEDFACE -MH_CIGAM = 0xCEFAEDFE -MH_MAGIC_64 = 0xFEEDFACF -MH_CIGAM_64 = 0xCFFAEDFE - -LC_VERSION_MIN_MACOSX = 0x24 -LC_BUILD_VERSION = 0x32 - -CPU_TYPE_ARM64 = 0x0100000C - -mach_header_fields = [ - ("magic", ctypes.c_uint32), - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("filetype", ctypes.c_uint32), - ("ncmds", ctypes.c_uint32), - ("sizeofcmds", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct mach_header { - uint32_t magic; /* mach magic number identifier */ - cpu_type_t cputype; /* cpu specifier */ - cpu_subtype_t cpusubtype; /* machine specifier */ - uint32_t filetype; /* type of file */ - uint32_t ncmds; /* number of load commands */ - uint32_t sizeofcmds; /* the size of all the load commands */ - uint32_t flags; /* flags */ -}; -typedef integer_t cpu_type_t; -typedef integer_t cpu_subtype_t; -""" - -mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)] -""" -struct mach_header_64 { - uint32_t magic; /* mach magic number identifier */ - cpu_type_t cputype; /* cpu specifier */ - cpu_subtype_t cpusubtype; /* machine specifier */ - uint32_t filetype; /* type of file */ - uint32_t ncmds; /* number of load commands */ - uint32_t sizeofcmds; /* the size of all the load commands */ - uint32_t flags; /* flags */ - uint32_t reserved; /* reserved */ -}; -""" - -fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)] -""" -struct fat_header { - uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */ - uint32_t nfat_arch; /* number of structs that follow */ -}; -""" - -fat_arch_fields = [ - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("offset", ctypes.c_uint32), - ("size", ctypes.c_uint32), - ("align", ctypes.c_uint32), -] -""" -struct fat_arch { - cpu_type_t cputype; /* cpu specifier (int) */ - cpu_subtype_t cpusubtype; /* machine specifier (int) */ - uint32_t offset; /* file offset to this object file */ - uint32_t size; /* size of this object file */ - uint32_t align; /* alignment as a power of 2 */ -}; -""" - -fat_arch_64_fields = [ - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("offset", ctypes.c_uint64), - ("size", ctypes.c_uint64), - ("align", ctypes.c_uint32), - ("reserved", ctypes.c_uint32), -] -""" -struct fat_arch_64 { - cpu_type_t cputype; /* cpu specifier (int) */ - cpu_subtype_t cpusubtype; /* machine specifier (int) */ - uint64_t offset; /* file offset to this object file */ - uint64_t size; /* size of this object file */ - uint32_t align; /* alignment as a power of 2 */ - uint32_t reserved; /* reserved */ -}; -""" - -segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)] -"""base for reading segment info""" - -segment_command_fields = [ - ("cmd", ctypes.c_uint32), - ("cmdsize", ctypes.c_uint32), - ("segname", ctypes.c_char * 16), - ("vmaddr", ctypes.c_uint32), - ("vmsize", ctypes.c_uint32), - ("fileoff", ctypes.c_uint32), - ("filesize", ctypes.c_uint32), - ("maxprot", ctypes.c_int), - ("initprot", ctypes.c_int), - ("nsects", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct segment_command { /* for 32-bit architectures */ - uint32_t cmd; /* LC_SEGMENT */ - uint32_t cmdsize; /* includes sizeof section structs */ - char segname[16]; /* segment name */ - uint32_t vmaddr; /* memory address of this segment */ - uint32_t vmsize; /* memory size of this segment */ - uint32_t fileoff; /* file offset of this segment */ - uint32_t filesize; /* amount to map from the file */ - vm_prot_t maxprot; /* maximum VM protection */ - vm_prot_t initprot; /* initial VM protection */ - uint32_t nsects; /* number of sections in segment */ - uint32_t flags; /* flags */ -}; -typedef int vm_prot_t; -""" - -segment_command_fields_64 = [ - ("cmd", ctypes.c_uint32), - ("cmdsize", ctypes.c_uint32), - ("segname", ctypes.c_char * 16), - ("vmaddr", ctypes.c_uint64), - ("vmsize", ctypes.c_uint64), - ("fileoff", ctypes.c_uint64), - ("filesize", ctypes.c_uint64), - ("maxprot", ctypes.c_int), - ("initprot", ctypes.c_int), - ("nsects", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct segment_command_64 { /* for 64-bit architectures */ - uint32_t cmd; /* LC_SEGMENT_64 */ - uint32_t cmdsize; /* includes sizeof section_64 structs */ - char segname[16]; /* segment name */ - uint64_t vmaddr; /* memory address of this segment */ - uint64_t vmsize; /* memory size of this segment */ - uint64_t fileoff; /* file offset of this segment */ - uint64_t filesize; /* amount to map from the file */ - vm_prot_t maxprot; /* maximum VM protection */ - vm_prot_t initprot; /* initial VM protection */ - uint32_t nsects; /* number of sections in segment */ - uint32_t flags; /* flags */ -}; -""" - -version_min_command_fields = segment_base_fields + [ - ("version", ctypes.c_uint32), - ("sdk", ctypes.c_uint32), -] -""" -struct version_min_command { - uint32_t cmd; /* LC_VERSION_MIN_MACOSX or - LC_VERSION_MIN_IPHONEOS or - LC_VERSION_MIN_WATCHOS or - LC_VERSION_MIN_TVOS */ - uint32_t cmdsize; /* sizeof(struct min_version_command) */ - uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ -}; -""" - -build_version_command_fields = segment_base_fields + [ - ("platform", ctypes.c_uint32), - ("minos", ctypes.c_uint32), - ("sdk", ctypes.c_uint32), - ("ntools", ctypes.c_uint32), -] -""" -struct build_version_command { - uint32_t cmd; /* LC_BUILD_VERSION */ - uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ - /* ntools * sizeof(struct build_tool_version) */ - uint32_t platform; /* platform */ - uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t ntools; /* number of tool entries following this */ -}; -""" - - -def swap32(x): - return ( - ((x << 24) & 0xFF000000) - | ((x << 8) & 0x00FF0000) - | ((x >> 8) & 0x0000FF00) - | ((x >> 24) & 0x000000FF) - ) - - -def get_base_class_and_magic_number(lib_file, seek=None): - if seek is None: - seek = lib_file.tell() - else: - lib_file.seek(seek) - magic_number = ctypes.c_uint32.from_buffer_copy( - lib_file.read(ctypes.sizeof(ctypes.c_uint32)) - ).value - - # Handle wrong byte order - if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]: - if sys.byteorder == "little": - BaseClass = ctypes.BigEndianStructure - else: - BaseClass = ctypes.LittleEndianStructure - - magic_number = swap32(magic_number) - else: - BaseClass = ctypes.Structure - - lib_file.seek(seek) - return BaseClass, magic_number - - -def read_data(struct_class, lib_file): - return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class))) - - -def extract_macosx_min_system_version(path_to_lib): - with open(path_to_lib, "rb") as lib_file: - BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0) - if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]: - return - - if magic_number in [FAT_MAGIC, FAT_CIGAM_64]: - - class FatHeader(BaseClass): - _fields_ = fat_header_fields - - fat_header = read_data(FatHeader, lib_file) - if magic_number == FAT_MAGIC: - - class FatArch(BaseClass): - _fields_ = fat_arch_fields - - else: - - class FatArch(BaseClass): - _fields_ = fat_arch_64_fields - - fat_arch_list = [ - read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch) - ] - - versions_list = [] - for el in fat_arch_list: - try: - version = read_mach_header(lib_file, el.offset) - if version is not None: - if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1: - # Xcode will not set the deployment target below 11.0.0 - # for the arm64 architecture. Ignore the arm64 deployment - # in fat binaries when the target is 11.0.0, that way - # the other architectures can select a lower deployment - # target. - # This is safe because there is no arm64 variant for - # macOS 10.15 or earlier. - if version == (11, 0, 0): - continue - versions_list.append(version) - except ValueError: - pass - - if len(versions_list) > 0: - return max(versions_list) - else: - return None - - else: - try: - return read_mach_header(lib_file, 0) - except ValueError: - """when some error during read library files""" - return None - - -def read_mach_header(lib_file, seek=None): - """ - This function parses a Mach-O header and extracts - information about the minimal macOS version. - - :param lib_file: reference to opened library file with pointer - """ - base_class, magic_number = get_base_class_and_magic_number(lib_file, seek) - arch = "32" if magic_number == MH_MAGIC else "64" - - class SegmentBase(base_class): - _fields_ = segment_base_fields - - if arch == "32": - - class MachHeader(base_class): - _fields_ = mach_header_fields - - else: - - class MachHeader(base_class): - _fields_ = mach_header_fields_64 - - mach_header = read_data(MachHeader, lib_file) - for _i in range(mach_header.ncmds): - pos = lib_file.tell() - segment_base = read_data(SegmentBase, lib_file) - lib_file.seek(pos) - if segment_base.cmd == LC_VERSION_MIN_MACOSX: - - class VersionMinCommand(base_class): - _fields_ = version_min_command_fields - - version_info = read_data(VersionMinCommand, lib_file) - return parse_version(version_info.version) - elif segment_base.cmd == LC_BUILD_VERSION: - - class VersionBuild(base_class): - _fields_ = build_version_command_fields - - version_info = read_data(VersionBuild, lib_file) - return parse_version(version_info.minos) - else: - lib_file.seek(pos + segment_base.cmdsize) - continue - - -def parse_version(version): - x = (version & 0xFFFF0000) >> 16 - y = (version & 0x0000FF00) >> 8 - z = version & 0x000000FF - return x, y, z - - -def calculate_macosx_platform_tag(archive_root, platform_tag): - """ - Calculate proper macosx platform tag basing on files which are included to wheel - - Example platform tag `macosx-10.14-x86_64` - """ - prefix, base_version, suffix = platform_tag.split("-") - base_version = tuple(int(x) for x in base_version.split(".")) - base_version = base_version[:2] - if base_version[0] > 10: - base_version = (base_version[0], 0) - assert len(base_version) == 2 - if "MACOSX_DEPLOYMENT_TARGET" in os.environ: - deploy_target = tuple( - int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".") - ) - deploy_target = deploy_target[:2] - if deploy_target[0] > 10: - deploy_target = (deploy_target[0], 0) - if deploy_target < base_version: - sys.stderr.write( - "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than " - "the version on which the Python interpreter was compiled ({}), and " - "will be ignored.\n".format( - ".".join(str(x) for x in deploy_target), - ".".join(str(x) for x in base_version), - ) - ) - else: - base_version = deploy_target - - assert len(base_version) == 2 - start_version = base_version - versions_dict = {} - for dirpath, _dirnames, filenames in os.walk(archive_root): - for filename in filenames: - if filename.endswith(".dylib") or filename.endswith(".so"): - lib_path = os.path.join(dirpath, filename) - min_ver = extract_macosx_min_system_version(lib_path) - if min_ver is not None: - min_ver = min_ver[0:2] - if min_ver[0] > 10: - min_ver = (min_ver[0], 0) - versions_dict[lib_path] = min_ver - - if len(versions_dict) > 0: - base_version = max(base_version, max(versions_dict.values())) - - # macosx platform tag do not support minor bugfix release - fin_base_version = "_".join([str(x) for x in base_version]) - if start_version < base_version: - problematic_files = [k for k, v in versions_dict.items() if v > start_version] - problematic_files = "\n".join(problematic_files) - if len(problematic_files) == 1: - files_form = "this file" - else: - files_form = "these files" - error_message = ( - "[WARNING] This wheel needs a higher macOS version than {} " - "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least " - + fin_base_version - + " or recreate " - + files_form - + " with lower " - "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files - ) - - if "MACOSX_DEPLOYMENT_TARGET" in os.environ: - error_message = error_message.format( - "is set in MACOSX_DEPLOYMENT_TARGET variable." - ) - else: - error_message = error_message.format( - "the version your Python interpreter is compiled against." - ) - - sys.stderr.write(error_message) - - platform_tag = prefix + "_" + fin_base_version + "_" + suffix - return platform_tag diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/metadata.py b/contrib/python/setuptools/py3/setuptools/_vendor/wheel/metadata.py deleted file mode 100644 index 341f614ceb3..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/metadata.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Tools for converting old- to new-style metadata. -""" - -from __future__ import annotations - -import functools -import itertools -import os.path -import re -import textwrap -from email.message import Message -from email.parser import Parser -from typing import Iterator - -from ..packaging.requirements import Requirement - - -def _nonblank(str): - return str and not str.startswith("#") - - -def yield_lines(iterable): - r""" - Yield valid lines of a string or iterable. - >>> list(yield_lines('')) - [] - >>> list(yield_lines(['foo', 'bar'])) - ['foo', 'bar'] - >>> list(yield_lines('foo\nbar')) - ['foo', 'bar'] - >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) - ['foo', 'baz #comment'] - >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) - ['foo', 'bar', 'baz', 'bing'] - """ - return itertools.chain.from_iterable(map(yield_lines, iterable)) - - -@yield_lines.register(str) -def _(text): - return filter(_nonblank, map(str.strip, text.splitlines())) - - -def split_sections(s): - """Split a string or iterable thereof into (section, content) pairs - Each ``section`` is a stripped version of the section header ("[section]") - and each ``content`` is a list of stripped lines excluding blank lines and - comment-only lines. If there are any such lines before the first section - header, they're returned in a first ``section`` of ``None``. - """ - section = None - content = [] - for line in yield_lines(s): - if line.startswith("["): - if line.endswith("]"): - if section or content: - yield section, content - section = line[1:-1].strip() - content = [] - else: - raise ValueError("Invalid section heading", line) - else: - content.append(line) - - # wrap up last segment - yield section, content - - -def safe_extra(extra): - """Convert an arbitrary string to a standard 'extra' name - Any runs of non-alphanumeric characters are replaced with a single '_', - and the result is always lowercased. - """ - return re.sub("[^A-Za-z0-9.-]+", "_", extra).lower() - - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub("[^A-Za-z0-9.]+", "-", name) - - -def requires_to_requires_dist(requirement: Requirement) -> str: - """Return the version specifier for a requirement in PEP 345/566 fashion.""" - if getattr(requirement, "url", None): - return " @ " + requirement.url - - requires_dist = [] - for spec in requirement.specifier: - requires_dist.append(spec.operator + spec.version) - - if requires_dist: - return " " + ",".join(sorted(requires_dist)) - else: - return "" - - -def convert_requirements(requirements: list[str]) -> Iterator[str]: - """Yield Requires-Dist: strings for parsed requirements strings.""" - for req in requirements: - parsed_requirement = Requirement(req) - spec = requires_to_requires_dist(parsed_requirement) - extras = ",".join(sorted(safe_extra(e) for e in parsed_requirement.extras)) - if extras: - extras = f"[{extras}]" - - yield safe_name(parsed_requirement.name) + extras + spec - - -def generate_requirements( - extras_require: dict[str, list[str]], -) -> Iterator[tuple[str, str]]: - """ - Convert requirements from a setup()-style dictionary to - ('Requires-Dist', 'requirement') and ('Provides-Extra', 'extra') tuples. - - extras_require is a dictionary of {extra: [requirements]} as passed to setup(), - using the empty extra {'': [requirements]} to hold install_requires. - """ - for extra, depends in extras_require.items(): - condition = "" - extra = extra or "" - if ":" in extra: # setuptools extra:condition syntax - extra, condition = extra.split(":", 1) - - extra = safe_extra(extra) - if extra: - yield "Provides-Extra", extra - if condition: - condition = "(" + condition + ") and " - condition += "extra == '%s'" % extra - - if condition: - condition = " ; " + condition - - for new_req in convert_requirements(depends): - yield "Requires-Dist", new_req + condition - - -def pkginfo_to_metadata(egg_info_path: str, pkginfo_path: str) -> Message: - """ - Convert .egg-info directory with PKG-INFO to the Metadata 2.1 format - """ - with open(pkginfo_path, encoding="utf-8") as headers: - pkg_info = Parser().parse(headers) - - pkg_info.replace_header("Metadata-Version", "2.1") - # Those will be regenerated from `requires.txt`. - del pkg_info["Provides-Extra"] - del pkg_info["Requires-Dist"] - requires_path = os.path.join(egg_info_path, "requires.txt") - if os.path.exists(requires_path): - with open(requires_path, encoding="utf-8") as requires_file: - requires = requires_file.read() - - parsed_requirements = sorted(split_sections(requires), key=lambda x: x[0] or "") - for extra, reqs in parsed_requirements: - for key, value in generate_requirements({extra: reqs}): - if (key, value) not in pkg_info.items(): - pkg_info[key] = value - - description = pkg_info["Description"] - if description: - description_lines = pkg_info["Description"].splitlines() - dedented_description = "\n".join( - # if the first line of long_description is blank, - # the first line here will be indented. - ( - description_lines[0].lstrip(), - textwrap.dedent("\n".join(description_lines[1:])), - "\n", - ) - ) - pkg_info.set_payload(dedented_description) - del pkg_info["Description"] - - return pkg_info diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/util.py b/contrib/python/setuptools/py3/setuptools/_vendor/wheel/util.py deleted file mode 100644 index d98d98cb52b..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/util.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -import base64 -import logging - -log = logging.getLogger("wheel") - -# ensure Python logging is configured -try: - __import__("setuptools.logging") -except ImportError: - # setuptools < ?? - from . import _setuptools_logging - - _setuptools_logging.configure() - - -def urlsafe_b64encode(data: bytes) -> bytes: - """urlsafe_b64encode without padding""" - return base64.urlsafe_b64encode(data).rstrip(b"=") - - -def urlsafe_b64decode(data: bytes) -> bytes: - """urlsafe_b64decode without padding""" - pad = b"=" * (4 - (len(data) & 3)) - return base64.urlsafe_b64decode(data + pad) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/wheelfile.py b/contrib/python/setuptools/py3/setuptools/_vendor/wheel/wheelfile.py deleted file mode 100644 index 83a31772bda..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/wheelfile.py +++ /dev/null @@ -1,199 +0,0 @@ -from __future__ import annotations - -import csv -import hashlib -import os.path -import re -import stat -import time -from io import StringIO, TextIOWrapper -from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo - -from .util import log, urlsafe_b64decode, urlsafe_b64encode - -# Non-greedy matching of an optional build number may be too clever (more -# invalid wheel filenames will match). Separate regex for .dist-info? -WHEEL_INFO_RE = re.compile( - r"""^(?P<namever>(?P<name>[^\s-]+?)-(?P<ver>[^\s-]+?))(-(?P<build>\d[^\s-]*))? - -(?P<pyver>[^\s-]+?)-(?P<abi>[^\s-]+?)-(?P<plat>\S+)\.whl$""", - re.VERBOSE, -) -MINIMUM_TIMESTAMP = 315532800 # 1980-01-01 00:00:00 UTC - - -def get_zipinfo_datetime(timestamp=None): - # Some applications need reproducible .whl files, but they can't do this without - # forcing the timestamp of the individual ZipInfo objects. See issue #143. - timestamp = int(os.environ.get("SOURCE_DATE_EPOCH", timestamp or time.time())) - timestamp = max(timestamp, MINIMUM_TIMESTAMP) - return time.gmtime(timestamp)[0:6] - - -class WheelFile(ZipFile): - """A ZipFile derivative class that also reads SHA-256 hashes from - .dist-info/RECORD and checks any read files against those. - """ - - _default_algorithm = hashlib.sha256 - - def __init__(self, file, mode="r", compression=ZIP_DEFLATED): - basename = os.path.basename(file) - self.parsed_filename = WHEEL_INFO_RE.match(basename) - if not basename.endswith(".whl") or self.parsed_filename is None: - raise WheelError(f"Bad wheel filename {basename!r}") - - ZipFile.__init__(self, file, mode, compression=compression, allowZip64=True) - - self.dist_info_path = "{}.dist-info".format( - self.parsed_filename.group("namever") - ) - self.record_path = self.dist_info_path + "/RECORD" - self._file_hashes = {} - self._file_sizes = {} - if mode == "r": - # Ignore RECORD and any embedded wheel signatures - self._file_hashes[self.record_path] = None, None - self._file_hashes[self.record_path + ".jws"] = None, None - self._file_hashes[self.record_path + ".p7s"] = None, None - - # Fill in the expected hashes by reading them from RECORD - try: - record = self.open(self.record_path) - except KeyError: - raise WheelError(f"Missing {self.record_path} file") from None - - with record: - for line in csv.reader( - TextIOWrapper(record, newline="", encoding="utf-8") - ): - path, hash_sum, size = line - if not hash_sum: - continue - - algorithm, hash_sum = hash_sum.split("=") - try: - hashlib.new(algorithm) - except ValueError: - raise WheelError( - f"Unsupported hash algorithm: {algorithm}" - ) from None - - if algorithm.lower() in {"md5", "sha1"}: - raise WheelError( - f"Weak hash algorithm ({algorithm}) is not permitted by " - f"PEP 427" - ) - - self._file_hashes[path] = ( - algorithm, - urlsafe_b64decode(hash_sum.encode("ascii")), - ) - - def open(self, name_or_info, mode="r", pwd=None): - def _update_crc(newdata): - eof = ef._eof - update_crc_orig(newdata) - running_hash.update(newdata) - if eof and running_hash.digest() != expected_hash: - raise WheelError(f"Hash mismatch for file '{ef_name}'") - - ef_name = ( - name_or_info.filename if isinstance(name_or_info, ZipInfo) else name_or_info - ) - if ( - mode == "r" - and not ef_name.endswith("/") - and ef_name not in self._file_hashes - ): - raise WheelError(f"No hash found for file '{ef_name}'") - - ef = ZipFile.open(self, name_or_info, mode, pwd) - if mode == "r" and not ef_name.endswith("/"): - algorithm, expected_hash = self._file_hashes[ef_name] - if expected_hash is not None: - # Monkey patch the _update_crc method to also check for the hash from - # RECORD - running_hash = hashlib.new(algorithm) - update_crc_orig, ef._update_crc = ef._update_crc, _update_crc - - return ef - - def write_files(self, base_dir): - log.info(f"creating '{self.filename}' and adding '{base_dir}' to it") - deferred = [] - for root, dirnames, filenames in os.walk(base_dir): - # Sort the directory names so that `os.walk` will walk them in a - # defined order on the next iteration. - dirnames.sort() - for name in sorted(filenames): - path = os.path.normpath(os.path.join(root, name)) - if os.path.isfile(path): - arcname = os.path.relpath(path, base_dir).replace(os.path.sep, "/") - if arcname == self.record_path: - pass - elif root.endswith(".dist-info"): - deferred.append((path, arcname)) - else: - self.write(path, arcname) - - deferred.sort() - for path, arcname in deferred: - self.write(path, arcname) - - def write(self, filename, arcname=None, compress_type=None): - with open(filename, "rb") as f: - st = os.fstat(f.fileno()) - data = f.read() - - zinfo = ZipInfo( - arcname or filename, date_time=get_zipinfo_datetime(st.st_mtime) - ) - zinfo.external_attr = (stat.S_IMODE(st.st_mode) | stat.S_IFMT(st.st_mode)) << 16 - zinfo.compress_type = compress_type or self.compression - self.writestr(zinfo, data, compress_type) - - def writestr(self, zinfo_or_arcname, data, compress_type=None): - if isinstance(zinfo_or_arcname, str): - zinfo_or_arcname = ZipInfo( - zinfo_or_arcname, date_time=get_zipinfo_datetime() - ) - zinfo_or_arcname.compress_type = self.compression - zinfo_or_arcname.external_attr = (0o664 | stat.S_IFREG) << 16 - - if isinstance(data, str): - data = data.encode("utf-8") - - ZipFile.writestr(self, zinfo_or_arcname, data, compress_type) - fname = ( - zinfo_or_arcname.filename - if isinstance(zinfo_or_arcname, ZipInfo) - else zinfo_or_arcname - ) - log.info(f"adding '{fname}'") - if fname != self.record_path: - hash_ = self._default_algorithm(data) - self._file_hashes[fname] = ( - hash_.name, - urlsafe_b64encode(hash_.digest()).decode("ascii"), - ) - self._file_sizes[fname] = len(data) - - def close(self): - # Write RECORD - if self.fp is not None and self.mode == "w" and self._file_hashes: - data = StringIO() - writer = csv.writer(data, delimiter=",", quotechar='"', lineterminator="\n") - writer.writerows( - ( - (fname, algorithm + "=" + hash_, self._file_sizes[fname]) - for fname, (algorithm, hash_) in self._file_hashes.items() - ) - ) - writer.writerow((format(self.record_path), "", "")) - self.writestr(self.record_path, data.getvalue()) - - ZipFile.close(self) - - -class WheelError(Exception): - pass diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py b/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py deleted file mode 100644 index 26b723c1fd3..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py +++ /dev/null @@ -1,329 +0,0 @@ -import io -import posixpath -import zipfile -import itertools -import contextlib -import sys -import pathlib - -if sys.version_info < (3, 7): - from collections import OrderedDict -else: - OrderedDict = dict - - -__all__ = ['Path'] - - -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = OrderedDict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(zipfile.ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, zipfile.ZipFile): - return cls(_pathlib_compat(source)) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -def _pathlib_compat(path): - """ - For path-like objects, convert to a filename for compatibility - on Python 3.6.1 and earlier. - """ - try: - return path.__fspath__() - except AttributeError: - return str(path) - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = zipfile.ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix - - @property - def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes - - @property - def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *map(_pathlib_compat, other)) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) diff --git a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py index 1f1967e7aa3..ef35d183e86 100644 --- a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py +++ b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py @@ -15,8 +15,8 @@ from itertools import filterfalse from typing import Dict, Mapping, TypeVar from .. import _reqs -from ..extern.jaraco.text import yield_lines -from ..extern.packaging.requirements import Requirement +from jaraco.text import yield_lines +from packaging.requirements import Requirement # dict can work as an ordered set diff --git a/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py b/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py index d8cdd4e4060..5b9bcec60cf 100644 --- a/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py +++ b/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py @@ -23,10 +23,10 @@ from typing import TYPE_CHECKING, Callable, Iterable, Literal, Sequence, cast from zipfile import ZIP_DEFLATED, ZIP_STORED from .. import Command, __version__ -from ..extern.wheel.metadata import pkginfo_to_metadata -from ..extern.packaging import tags -from ..extern.packaging import version as _packaging_version -from ..extern.wheel.wheelfile import WheelFile +from wheel.metadata import pkginfo_to_metadata +from packaging import tags +from packaging import version as _packaging_version +from wheel.wheelfile import WheelFile if TYPE_CHECKING: import types @@ -67,8 +67,8 @@ def python_tag() -> str: def get_platform(archive_root: str | None) -> str: """Return our platform name 'win32', 'linux_x86_64'""" result = sysconfig.get_platform() - if result.startswith("macosx") and archive_root is not None: - from ..extern.wheel.macosx_libfile import calculate_macosx_platform_tag + if result.startswith("macosx") and archive_root is not None: # pragma: no cover + from wheel.macosx_libfile import calculate_macosx_platform_tag result = calculate_macosx_platform_tag(archive_root, result) elif _is_32bit_interpreter(): @@ -451,7 +451,7 @@ class bdist_wheel(Command): def write_wheelfile( self, wheelfile_base: str, generator: str = f"setuptools ({__version__})" - ): + ) -> None: from email.message import Message msg = Message() @@ -525,7 +525,7 @@ class bdist_wheel(Command): return files - def egg2dist(self, egginfo_path: str, distinfo_path: str): + def egg2dist(self, egginfo_path: str, distinfo_path: str) -> None: """Convert an .egg-info directory into a .dist-info directory""" def adios(p: str) -> None: diff --git a/contrib/python/setuptools/py3/setuptools/command/build_py.py b/contrib/python/setuptools/py3/setuptools/command/build_py.py index ab49874635f..15a4f63fdd4 100644 --- a/contrib/python/setuptools/py3/setuptools/command/build_py.py +++ b/contrib/python/setuptools/py3/setuptools/command/build_py.py @@ -13,7 +13,7 @@ import stat from pathlib import Path from typing import Iterable, Iterator -from ..extern.more_itertools import unique_everseen +from more_itertools import unique_everseen from ..warnings import SetuptoolsDeprecationWarning diff --git a/contrib/python/setuptools/py3/setuptools/command/easy_install.py b/contrib/python/setuptools/py3/setuptools/command/easy_install.py index e6ce3fcc055..36114d40ed9 100644 --- a/contrib/python/setuptools/py3/setuptools/command/easy_install.py +++ b/contrib/python/setuptools/py3/setuptools/command/easy_install.py @@ -76,7 +76,7 @@ from pkg_resources import ( import pkg_resources from ..compat import py39, py311 from .._path import ensure_directory -from ..extern.jaraco.text import yield_lines +from jaraco.text import yield_lines # Turn on PEP440Warnings diff --git a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py index ae31bb4c79b..49fd609b151 100644 --- a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py +++ b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py @@ -299,7 +299,7 @@ class editable_wheel(Command): build = self.get_finalized_command("build") for name in build.get_sub_commands(): cmd = self.get_finalized_command(name) - if name == "build_py" and type(cmd) != build_py_cls: + if name == "build_py" and type(cmd) is not build_py_cls: self._safely_run(name) else: self.run_command(name) @@ -333,7 +333,7 @@ class editable_wheel(Command): ) def _create_wheel_file(self, bdist_wheel): - from ..extern.wheel.wheelfile import WheelFile + from wheel.wheelfile import WheelFile dist_info = self.get_finalized_command("dist_info") dist_name = dist_info.name @@ -443,8 +443,7 @@ class _LinkTree(_StaticPth): ): self.auxiliary_dir = Path(auxiliary_dir) self.build_lib = Path(build_lib).resolve() - # TODO: Update typeshed distutils stubs to overload non-None return type by default - self._file = dist.get_command_obj("build_py").copy_file # type: ignore[union-attr] + self._file = dist.get_command_obj("build_py").copy_file super().__init__(dist, name, [self.auxiliary_dir]) def __call__(self, wheel: WheelFile, files: list[str], mapping: dict[str, str]): @@ -462,9 +461,7 @@ class _LinkTree(_StaticPth): dest = self.auxiliary_dir / relative_output if not dest.parent.is_dir(): dest.parent.mkdir(parents=True) - # TODO: Update typeshed distutils stubs so distutils.cmd.Command.copy_file, accepts PathLike - # same with methods used by copy_file - self._file(src_file, dest, link=link) # type: ignore[arg-type] + self._file(src_file, dest, link=link) def _create_links(self, outputs, output_mapping): self.auxiliary_dir.mkdir(parents=True, exist_ok=True) @@ -805,7 +802,7 @@ PATH_PLACEHOLDER = {name!r} + ".__path_hook__" class _EditableFinder: # MetaPathFinder @classmethod - def find_spec(cls, fullname: str, _path=None, _target=None) -> ModuleSpec | None: + def find_spec(cls, fullname: str, path=None, target=None) -> ModuleSpec | None: # type: ignore # Top-level packages and modules (we know these exist in the FS) if fullname in MAPPING: pkg_path = MAPPING[fullname] @@ -851,7 +848,7 @@ class _EditableNamespaceFinder: # PathEntryFinder return [*paths, PATH_PLACEHOLDER] @classmethod - def find_spec(cls, fullname: str, _target=None) -> ModuleSpec | None: + def find_spec(cls, fullname: str, target=None) -> ModuleSpec | None: # type: ignore if fullname in NAMESPACES: spec = ModuleSpec(fullname, None, is_package=True) spec.submodule_search_locations = cls._paths(fullname) diff --git a/contrib/python/setuptools/py3/setuptools/command/egg_info.py b/contrib/python/setuptools/py3/setuptools/command/egg_info.py index 2f203033413..30b62f5f2ef 100644 --- a/contrib/python/setuptools/py3/setuptools/command/egg_info.py +++ b/contrib/python/setuptools/py3/setuptools/command/egg_info.py @@ -27,7 +27,7 @@ from setuptools.command import bdist_egg import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob -from setuptools.extern import packaging +import packaging from ..warnings import SetuptoolsDeprecationWarning @@ -250,17 +250,6 @@ class egg_info(InfoCommon, Command): # self.distribution.metadata.version = self.egg_version - # If we bootstrapped around the lack of a PKG-INFO, as might be the - # case in a fresh checkout, make sure that any special tags get added - # to the version info - # - pd = self.distribution._patched_dist - key = getattr(pd, "key", None) or getattr(pd, "name", None) - if pd is not None and key == self.egg_name.lower(): - pd._version = self.egg_version - pd._parsed_version = packaging.version.Version(self.egg_version) - self.distribution._patched_dist = None - def _get_egg_basename(self, py_version=PY_MAJOR, platform=None): """Compute filename of the output egg. Private API.""" return _egg_basename(self.egg_name, self.egg_version, py_version, platform) diff --git a/contrib/python/setuptools/py3/setuptools/command/install.py b/contrib/python/setuptools/py3/setuptools/command/install.py index c49fcda9397..f1ea2adf1d0 100644 --- a/contrib/python/setuptools/py3/setuptools/command/install.py +++ b/contrib/python/setuptools/py3/setuptools/command/install.py @@ -1,9 +1,12 @@ +from __future__ import annotations + +from collections.abc import Callable from distutils.errors import DistutilsArgError import inspect import glob import platform import distutils.command.install as orig -from typing import cast +from typing import Any, ClassVar, cast import setuptools from ..warnings import SetuptoolsDeprecationWarning, SetuptoolsWarning @@ -29,7 +32,9 @@ class install(orig.install): 'old-and-unmanageable', 'single-version-externally-managed', ] - new_commands = [ + # Type the same as distutils.command.install.install.sub_commands + # Must keep the second tuple item potentially None due to invariance + new_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] = [ ('install_egg_info', lambda self: True), ('install_scripts', lambda self: True), ] diff --git a/contrib/python/setuptools/py3/setuptools/command/install_lib.py b/contrib/python/setuptools/py3/setuptools/command/install_lib.py index 5e74be247e5..3c77c6ebc66 100644 --- a/contrib/python/setuptools/py3/setuptools/command/install_lib.py +++ b/contrib/python/setuptools/py3/setuptools/command/install_lib.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import sys from itertools import product, starmap @@ -92,21 +93,21 @@ class install_lib(orig.install_lib): preserve_times=True, preserve_symlinks=False, level=1, - ): + ) -> list[str]: assert preserve_mode and preserve_times and not preserve_symlinks exclude = self.get_exclusions() if not exclude: - return orig.install_lib.copy_tree(self, infile, outfile) # type: ignore[arg-type] # Fixed upstream + return orig.install_lib.copy_tree(self, infile, outfile) # Exclude namespace package __init__.py* files from the output from setuptools.archive_util import unpack_directory from distutils import log - outfiles = [] + outfiles: list[str] = [] - def pf(src, dst): + def pf(src: str, dst: str): if dst in exclude: log.warn("Skipping installation of %s (namespace package)", dst) return False diff --git a/contrib/python/setuptools/py3/setuptools/command/test.py b/contrib/python/setuptools/py3/setuptools/command/test.py index af1349e1c6e..fbdf9fb9423 100644 --- a/contrib/python/setuptools/py3/setuptools/command/test.py +++ b/contrib/python/setuptools/py3/setuptools/command/test.py @@ -19,8 +19,8 @@ from pkg_resources import ( ) from .._importlib import metadata from setuptools import Command -from setuptools.extern.more_itertools import unique_everseen -from setuptools.extern.jaraco.functools import pass_none +from more_itertools import unique_everseen +from jaraco.functools import pass_none class ScanningLoader(TestLoader): diff --git a/contrib/python/setuptools/py3/setuptools/command/upload_docs.py b/contrib/python/setuptools/py3/setuptools/command/upload_docs.py index 3fbbb62553f..32c9abd796f 100644 --- a/contrib/python/setuptools/py3/setuptools/command/upload_docs.py +++ b/contrib/python/setuptools/py3/setuptools/command/upload_docs.py @@ -50,7 +50,7 @@ class upload_docs(upload): and metadata.entry_points(group='distutils.commands', name='build_sphinx') ) - sub_commands = [('build_sphinx', has_sphinx)] # type: ignore[list-item] # TODO: Fix in typeshed distutils stubs + sub_commands = [('build_sphinx', has_sphinx)] def initialize_options(self): upload.initialize_options(self) diff --git a/contrib/python/setuptools/py3/setuptools/compat/py310.py b/contrib/python/setuptools/py3/setuptools/compat/py310.py index f7d53d6de94..cc875c004b6 100644 --- a/contrib/python/setuptools/py3/setuptools/compat/py310.py +++ b/contrib/python/setuptools/py3/setuptools/compat/py310.py @@ -7,4 +7,4 @@ __all__ = ['tomllib'] if sys.version_info >= (3, 11): import tomllib else: # pragma: no cover - from setuptools.extern import tomli as tomllib + import tomli as tomllib diff --git a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py index f44271c5ddb..6cc59d2b951 100644 --- a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py @@ -203,8 +203,8 @@ def _project_urls(dist: Distribution, val: dict, _root_dir): _set_config(dist, "project_urls", val) -def _python_requires(dist: Distribution, val: dict, _root_dir): - from setuptools.extern.packaging.specifiers import SpecifierSet +def _python_requires(dist: Distribution, val: str, _root_dir): + from packaging.specifiers import SpecifierSet _set_config(dist, "python_requires", SpecifierSet(val)) diff --git a/contrib/python/setuptools/py3/setuptools/config/expand.py b/contrib/python/setuptools/py3/setuptools/config/expand.py index e5f5dc586e8..de6339fa424 100644 --- a/contrib/python/setuptools/py3/setuptools/config/expand.py +++ b/contrib/python/setuptools/py3/setuptools/config/expand.py @@ -31,6 +31,7 @@ from importlib.machinery import ModuleSpec, all_suffixes from itertools import chain from typing import ( TYPE_CHECKING, + Any, Callable, Iterable, Iterator, @@ -122,7 +123,7 @@ def read_files( (By default ``root_dir`` is the current directory). """ - from setuptools.extern.more_itertools import always_iterable + from more_itertools import always_iterable root_dir = os.path.abspath(root_dir or os.getcwd()) _filepaths = (os.path.join(root_dir, path) for path in always_iterable(filepaths)) @@ -158,7 +159,7 @@ def read_attr( attr_desc: str, package_dir: Mapping[str, str] | None = None, root_dir: StrPath | None = None, -): +) -> Any: """Reads the value of an attribute from a module. This function will try to read the attributed statically first @@ -287,7 +288,7 @@ def find_packages( :rtype: list """ from setuptools.discovery import construct_package_dir - from setuptools.extern.more_itertools import unique_everseen, always_iterable + from more_itertools import unique_everseen, always_iterable if namespaces: from setuptools.discovery import PEP420PackageFinder as PackageFinder diff --git a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py index d41c956cbd4..a83e43bb356 100644 --- a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py @@ -15,7 +15,7 @@ import logging import os from contextlib import contextmanager from functools import partial -from typing import TYPE_CHECKING, Callable, Mapping +from typing import TYPE_CHECKING, Any, Callable, Mapping from .._path import StrPath from ..errors import FileError, InvalidConfigError @@ -76,7 +76,7 @@ def read_configuration( expand=True, ignore_option_errors=False, dist: Distribution | None = None, -): +) -> dict[str, Any]: """Read given configuration file and returns options from it as a dict. :param str|unicode filepath: Path to configuration file in the ``pyproject.toml`` @@ -278,7 +278,7 @@ class _ConfigExpander: def _expand_directive( self, specifier: str, directive, package_dir: Mapping[str, str] ): - from setuptools.extern.more_itertools import always_iterable + from more_itertools import always_iterable with _ignore_errors(self.ignore_option_errors): root_dir = self.root_dir diff --git a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py index 80ebe3d9bd5..772b8d00e02 100644 --- a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py +++ b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py @@ -24,6 +24,7 @@ from typing import ( Dict, Generic, Iterable, + Iterator, Tuple, TypeVar, Union, @@ -31,10 +32,10 @@ from typing import ( from .._path import StrPath from ..errors import FileError, OptionError -from ..extern.packaging.markers import default_environment as marker_env -from ..extern.packaging.requirements import InvalidRequirement, Requirement -from ..extern.packaging.specifiers import SpecifierSet -from ..extern.packaging.version import InvalidVersion, Version +from packaging.markers import default_environment as marker_env +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version from ..warnings import SetuptoolsDeprecationWarning from . import expand @@ -260,7 +261,9 @@ class ConfigHandler(Generic[Target]): """ @classmethod - def _section_options(cls, options: AllCommandOptions): + def _section_options( + cls, options: AllCommandOptions + ) -> Iterator[tuple[str, SingleCommandOptions]]: for full_name, value in options.items(): pre, sep, name = full_name.partition(cls.section_prefix) if pre: diff --git a/contrib/python/setuptools/py3/setuptools/depends.py b/contrib/python/setuptools/py3/setuptools/depends.py index 2226b6784af..871a0925ef7 100644 --- a/contrib/python/setuptools/py3/setuptools/depends.py +++ b/contrib/python/setuptools/py3/setuptools/depends.py @@ -6,7 +6,7 @@ import dis from . import _imp from ._imp import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE -from .extern.packaging.version import Version +from packaging.version import Version __all__ = ['Require', 'find_module'] diff --git a/contrib/python/setuptools/py3/setuptools/dist.py b/contrib/python/setuptools/py3/setuptools/dist.py index 32e8d43c64a..b4496ab9860 100644 --- a/contrib/python/setuptools/py3/setuptools/dist.py +++ b/contrib/python/setuptools/py3/setuptools/dist.py @@ -6,7 +6,6 @@ import numbers import os import re import sys -from contextlib import suppress from glob import iglob from pathlib import Path from typing import TYPE_CHECKING, MutableMapping @@ -21,14 +20,13 @@ from distutils.errors import DistutilsOptionError, DistutilsSetupError from distutils.fancy_getopt import translate_longopt from distutils.util import strtobool -from .extern.more_itertools import partition, unique_everseen -from .extern.ordered_set import OrderedSet -from .extern.packaging.markers import InvalidMarker, Marker -from .extern.packaging.specifiers import InvalidSpecifier, SpecifierSet -from .extern.packaging.version import Version +from more_itertools import partition, unique_everseen +from ordered_set import OrderedSet +from packaging.markers import InvalidMarker, Marker +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import Version from . import _entry_points -from . import _normalization from . import _reqs from . import command as _ # noqa -- imported for side-effects from ._importlib import metadata @@ -269,24 +267,9 @@ class Distribution(_Distribution): 'extras_require': dict, } - _patched_dist = None # Used by build_py, editable_wheel and install_lib commands for legacy namespaces namespace_packages: list[str] #: :meta private: DEPRECATED - def patch_missing_pkg_info(self, attrs): - # Fake up a replacement for the data that would normally come from - # PKG-INFO, but which might not yet be built if this is a fresh - # checkout. - # - if not attrs or 'name' not in attrs or 'version' not in attrs: - return - name = _normalization.safe_name(str(attrs['name'])).lower() - with suppress(metadata.PackageNotFoundError): - dist = metadata.distribution(name) - if dist is not None and not dist.read_text('PKG-INFO'): - dist._version = _normalization.safe_version(str(attrs['version'])) - self._patched_dist = dist - def __init__(self, attrs: MutableMapping | None = None) -> None: have_package_data = hasattr(self, "package_data") if not have_package_data: @@ -295,7 +278,6 @@ class Distribution(_Distribution): self.dist_files: list[tuple[str, str, str]] = [] # Filter-out setuptools' specific options. self.src_root = attrs.pop("src_root", None) - self.patch_missing_pkg_info(attrs) self.dependency_links = attrs.pop('dependency_links', []) self.setup_requires = attrs.pop('setup_requires', []) for ep in metadata.entry_points(group='distutils.setup_keywords'): diff --git a/contrib/python/setuptools/py3/setuptools/extern/__init__.py b/contrib/python/setuptools/py3/setuptools/extern/__init__.py deleted file mode 100644 index f9b6eea70db..00000000000 --- a/contrib/python/setuptools/py3/setuptools/extern/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -import importlib.util -import sys - - -class VendorImporter: - """ - A PEP 302 meta path importer for finding optionally-vendored - or otherwise naturally-installed packages from root_name. - """ - - def __init__(self, root_name, vendored_names=(), vendor_pkg=None): - self.root_name = root_name - self.vendored_names = set(vendored_names) - self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') - - @property - def search_path(self): - """ - Search first the vendor package then as a natural package. - """ - yield self.vendor_pkg + '.' - yield '' - - def _module_matches_namespace(self, fullname): - """Figure out if the target module is vendored.""" - root, base, target = fullname.partition(self.root_name + '.') - return not root and any(map(target.startswith, self.vendored_names)) - - def load_module(self, fullname): - """ - Iterate over the search path to locate and load fullname. - """ - root, base, target = fullname.partition(self.root_name + '.') - for prefix in self.search_path: - extant = prefix + target - try: - __import__(extant) - except ImportError: - continue - mod = sys.modules[extant] - sys.modules[fullname] = mod - return mod - else: - raise ImportError( - "The '{target}' package is required; " - "normally this is bundled with this package so if you get " - "this warning, consult the packager of your " - "distribution.".format(**locals()) - ) - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - pass - - def find_spec(self, fullname, path=None, target=None): - """Return a module spec for vendored names.""" - return ( - importlib.util.spec_from_loader(fullname, self) - if self._module_matches_namespace(fullname) - else None - ) - - def install(self): - """ - Install this importer into sys.meta_path if not already present. - """ - if self not in sys.meta_path: - sys.meta_path.append(self) - - -# [[[cog -# import cog -# from tools.vendored import yield_top_level -# names = "\n".join(f" {x!r}," for x in yield_top_level('setuptools')) -# cog.outl(f"names = (\n{names}\n)") -# ]]] -names = ( - 'backports', - 'importlib_metadata', - 'importlib_resources', - 'jaraco', - 'more_itertools', - 'ordered_set', - 'packaging', - 'tomli', - 'wheel', - 'zipp', -) -# [[[end]]] -VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/contrib/python/setuptools/py3/setuptools/msvc.py b/contrib/python/setuptools/py3/setuptools/msvc.py index a3d350fe509..2768059213f 100644 --- a/contrib/python/setuptools/py3/setuptools/msvc.py +++ b/contrib/python/setuptools/py3/setuptools/msvc.py @@ -23,7 +23,8 @@ import itertools import subprocess import distutils.errors from typing import TYPE_CHECKING -from setuptools.extern.more_itertools import unique_everseen + +from more_itertools import unique_everseen # https://github.com/python/mypy/issues/8166 if not TYPE_CHECKING and platform.system() == 'Windows': diff --git a/contrib/python/setuptools/py3/setuptools/package_index.py b/contrib/python/setuptools/py3/setuptools/package_index.py index 2c807f6b4e6..c24c783762b 100644 --- a/contrib/python/setuptools/py3/setuptools/package_index.py +++ b/contrib/python/setuptools/py3/setuptools/package_index.py @@ -39,7 +39,8 @@ from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.wheel import Wheel -from setuptools.extern.more_itertools import unique_everseen + +from more_itertools import unique_everseen from .unicode_utils import _read_utf8_with_fallback, _cfg_read_utf8_with_fallback diff --git a/contrib/python/setuptools/py3/setuptools/warnings.py b/contrib/python/setuptools/py3/setuptools/warnings.py index 5d9cca6c375..8c94bc96e60 100644 --- a/contrib/python/setuptools/py3/setuptools/warnings.py +++ b/contrib/python/setuptools/py3/setuptools/warnings.py @@ -32,7 +32,7 @@ class SetuptoolsWarning(UserWarning): see_url: str | None = None, stacklevel: int = 2, **kwargs, - ): + ) -> None: """Private: reserved for ``setuptools`` internal use only""" # Default values: summary_ = summary or getattr(cls, "_SUMMARY", None) or "" @@ -56,7 +56,7 @@ class SetuptoolsWarning(UserWarning): due_date: date | None = None, see_url: str | None = None, format_args: dict | None = None, - ): + ) -> str: """Private: reserved for ``setuptools`` internal use only""" today = date.today() summary = cleandoc(summary).format_map(format_args or {}) diff --git a/contrib/python/setuptools/py3/setuptools/wheel.py b/contrib/python/setuptools/py3/setuptools/wheel.py index e06daec4d0e..a05cd98d1f5 100644 --- a/contrib/python/setuptools/py3/setuptools/wheel.py +++ b/contrib/python/setuptools/py3/setuptools/wheel.py @@ -9,12 +9,13 @@ import re import zipfile import contextlib +from packaging.version import Version as parse_version +from packaging.tags import sys_tags +from packaging.utils import canonicalize_name + from distutils.util import get_platform import setuptools -from setuptools.extern.packaging.version import Version as parse_version -from setuptools.extern.packaging.tags import sys_tags -from setuptools.extern.packaging.utils import canonicalize_name from setuptools.command.egg_info import write_requirements, _egg_basename from setuptools.archive_util import _unpack_zipfile_obj diff --git a/contrib/python/setuptools/py3/ya.make b/contrib/python/setuptools/py3/ya.make index 38047edfe40..189d7113414 100644 --- a/contrib/python/setuptools/py3/ya.make +++ b/contrib/python/setuptools/py3/ya.make @@ -2,11 +2,21 @@ PY3_LIBRARY() -VERSION(70.3.0) +VERSION(71.1.0) LICENSE(MIT) PEERDIR( + contrib/python/jaraco.context + contrib/python/jaraco.functools + contrib/python/jaraco.text + contrib/python/more-itertools + contrib/python/ordered-set + contrib/python/packaging + contrib/python/platformdirs + contrib/python/typeguard + contrib/python/typing-extensions + contrib/python/wheel library/python/resource ) @@ -23,53 +33,6 @@ PY_SRCS( _distutils_hack/__init__.py _distutils_hack/override.py pkg_resources/__init__.py - pkg_resources/_vendor/__init__.py - pkg_resources/_vendor/backports/__init__.py - pkg_resources/_vendor/backports/tarfile.py - pkg_resources/_vendor/importlib_resources/__init__.py - pkg_resources/_vendor/importlib_resources/_adapters.py - pkg_resources/_vendor/importlib_resources/_common.py - pkg_resources/_vendor/importlib_resources/_compat.py - pkg_resources/_vendor/importlib_resources/_itertools.py - pkg_resources/_vendor/importlib_resources/_legacy.py - pkg_resources/_vendor/importlib_resources/abc.py - pkg_resources/_vendor/importlib_resources/readers.py - pkg_resources/_vendor/importlib_resources/simple.py - pkg_resources/_vendor/jaraco/__init__.py - pkg_resources/_vendor/jaraco/context.py - pkg_resources/_vendor/jaraco/functools/__init__.py - pkg_resources/_vendor/jaraco/functools/__init__.pyi - pkg_resources/_vendor/jaraco/text/__init__.py - pkg_resources/_vendor/more_itertools/__init__.py - pkg_resources/_vendor/more_itertools/__init__.pyi - pkg_resources/_vendor/more_itertools/more.py - pkg_resources/_vendor/more_itertools/more.pyi - pkg_resources/_vendor/more_itertools/recipes.py - pkg_resources/_vendor/more_itertools/recipes.pyi - pkg_resources/_vendor/packaging/__init__.py - pkg_resources/_vendor/packaging/_elffile.py - pkg_resources/_vendor/packaging/_manylinux.py - pkg_resources/_vendor/packaging/_musllinux.py - pkg_resources/_vendor/packaging/_parser.py - pkg_resources/_vendor/packaging/_structures.py - pkg_resources/_vendor/packaging/_tokenizer.py - pkg_resources/_vendor/packaging/markers.py - pkg_resources/_vendor/packaging/metadata.py - pkg_resources/_vendor/packaging/requirements.py - pkg_resources/_vendor/packaging/specifiers.py - pkg_resources/_vendor/packaging/tags.py - pkg_resources/_vendor/packaging/utils.py - pkg_resources/_vendor/packaging/version.py - pkg_resources/_vendor/platformdirs/__init__.py - pkg_resources/_vendor/platformdirs/__main__.py - pkg_resources/_vendor/platformdirs/android.py - pkg_resources/_vendor/platformdirs/api.py - pkg_resources/_vendor/platformdirs/macos.py - pkg_resources/_vendor/platformdirs/unix.py - pkg_resources/_vendor/platformdirs/version.py - pkg_resources/_vendor/platformdirs/windows.py - pkg_resources/_vendor/zipp.py - pkg_resources/extern/__init__.py setuptools/__init__.py setuptools/_core_metadata.py setuptools/_distutils/__init__.py @@ -154,63 +117,6 @@ PY_SRCS( setuptools/_normalization.py setuptools/_path.py setuptools/_reqs.py - setuptools/_vendor/__init__.py - setuptools/_vendor/backports/__init__.py - setuptools/_vendor/backports/tarfile.py - setuptools/_vendor/importlib_metadata/__init__.py - setuptools/_vendor/importlib_metadata/_adapters.py - setuptools/_vendor/importlib_metadata/_collections.py - setuptools/_vendor/importlib_metadata/_compat.py - setuptools/_vendor/importlib_metadata/_functools.py - setuptools/_vendor/importlib_metadata/_itertools.py - setuptools/_vendor/importlib_metadata/_meta.py - setuptools/_vendor/importlib_metadata/_py39compat.py - setuptools/_vendor/importlib_metadata/_text.py - setuptools/_vendor/importlib_resources/__init__.py - setuptools/_vendor/importlib_resources/_adapters.py - setuptools/_vendor/importlib_resources/_common.py - setuptools/_vendor/importlib_resources/_compat.py - setuptools/_vendor/importlib_resources/_itertools.py - setuptools/_vendor/importlib_resources/_legacy.py - setuptools/_vendor/importlib_resources/abc.py - setuptools/_vendor/importlib_resources/readers.py - setuptools/_vendor/importlib_resources/simple.py - setuptools/_vendor/jaraco/__init__.py - setuptools/_vendor/jaraco/context.py - setuptools/_vendor/jaraco/functools/__init__.py - setuptools/_vendor/jaraco/functools/__init__.pyi - setuptools/_vendor/jaraco/text/__init__.py - setuptools/_vendor/more_itertools/__init__.py - setuptools/_vendor/more_itertools/__init__.pyi - setuptools/_vendor/more_itertools/more.py - setuptools/_vendor/more_itertools/more.pyi - setuptools/_vendor/more_itertools/recipes.py - setuptools/_vendor/more_itertools/recipes.pyi - setuptools/_vendor/ordered_set.py - setuptools/_vendor/packaging/__init__.py - setuptools/_vendor/packaging/_elffile.py - setuptools/_vendor/packaging/_manylinux.py - setuptools/_vendor/packaging/_musllinux.py - setuptools/_vendor/packaging/_parser.py - setuptools/_vendor/packaging/_structures.py - setuptools/_vendor/packaging/_tokenizer.py - setuptools/_vendor/packaging/markers.py - setuptools/_vendor/packaging/metadata.py - setuptools/_vendor/packaging/requirements.py - setuptools/_vendor/packaging/specifiers.py - setuptools/_vendor/packaging/tags.py - setuptools/_vendor/packaging/utils.py - setuptools/_vendor/packaging/version.py - setuptools/_vendor/tomli/__init__.py - setuptools/_vendor/tomli/_parser.py - setuptools/_vendor/tomli/_re.py - setuptools/_vendor/tomli/_types.py - setuptools/_vendor/wheel/__init__.py - setuptools/_vendor/wheel/macosx_libfile.py - setuptools/_vendor/wheel/metadata.py - setuptools/_vendor/wheel/util.py - setuptools/_vendor/wheel/wheelfile.py - setuptools/_vendor/zipp.py setuptools/archive_util.py setuptools/build_meta.py setuptools/command/__init__.py @@ -260,7 +166,6 @@ PY_SRCS( setuptools/dist.py setuptools/errors.py setuptools/extension.py - setuptools/extern/__init__.py setuptools/glob.py setuptools/installer.py setuptools/launch.py @@ -283,18 +188,9 @@ RESOURCE_FILES( .dist-info/METADATA .dist-info/entry_points.txt .dist-info/top_level.txt - pkg_resources/_vendor/importlib_resources/py.typed - pkg_resources/_vendor/jaraco/functools/py.typed - pkg_resources/_vendor/more_itertools/py.typed - pkg_resources/_vendor/packaging/py.typed - pkg_resources/_vendor/platformdirs/py.typed + pkg_resources/api_tests.txt + pkg_resources/py.typed setuptools/_distutils/_vendor/packaging/py.typed - setuptools/_vendor/importlib_metadata/py.typed - setuptools/_vendor/importlib_resources/py.typed - setuptools/_vendor/jaraco/functools/py.typed - setuptools/_vendor/more_itertools/py.typed - setuptools/_vendor/packaging/py.typed - setuptools/_vendor/tomli/py.typed setuptools/script.tmpl ) |