diff options
| author | robot-piglet <[email protected]> | 2026-05-01 22:38:31 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2026-05-01 22:59:28 +0300 |
| commit | bef8989f304675728c25192cb1e9011dea0d4b78 (patch) | |
| tree | 25630cae4a823ca3b5b83a4718bfad1271051746 /contrib/python/packaging | |
| parent | 45914ccf9621807951357017eb3bc2da00d09e11 (diff) | |
Intermediate changes
commit_hash:c36c08a925119d3cda68f417748df546e38e27b3
Diffstat (limited to 'contrib/python/packaging')
18 files changed, 2900 insertions, 411 deletions
diff --git a/contrib/python/packaging/py3/.dist-info/METADATA b/contrib/python/packaging/py3/.dist-info/METADATA index 3200e601f97..c1b3d36f8d2 100644 --- a/contrib/python/packaging/py3/.dist-info/METADATA +++ b/contrib/python/packaging/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: packaging -Version: 26.0 +Version: 26.1 Summary: Core utilities for Python packages Author-email: Donald Stufft <[email protected]> Requires-Python: >=3.8 @@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Free Threading :: 4 - Resilient Classifier: Typing :: Typed License-File: LICENSE License-File: LICENSE.APACHE @@ -72,11 +73,11 @@ Discussion If you run into bugs, you can file them in our `issue tracker`_. -You can also join ``#pypa`` on Freenode to ask questions or get involved. - +You can also join discussions on `GitHub Discussions`_ to ask questions or get involved. .. _`documentation`: https://packaging.pypa.io/ .. _`issue tracker`: https://github.com/pypa/packaging/issues +.. _`GitHub Discussions`: https://github.com/pypa/packaging/discussions Code of Conduct diff --git a/contrib/python/packaging/py3/README.rst b/contrib/python/packaging/py3/README.rst index ba3fa462bf1..4b47730e51a 100644 --- a/contrib/python/packaging/py3/README.rst +++ b/contrib/python/packaging/py3/README.rst @@ -43,11 +43,11 @@ Discussion If you run into bugs, you can file them in our `issue tracker`_. -You can also join ``#pypa`` on Freenode to ask questions or get involved. - +You can also join discussions on `GitHub Discussions`_ to ask questions or get involved. .. _`documentation`: https://packaging.pypa.io/ .. _`issue tracker`: https://github.com/pypa/packaging/issues +.. _`GitHub Discussions`: https://github.com/pypa/packaging/discussions Code of Conduct diff --git a/contrib/python/packaging/py3/packaging/__init__.py b/contrib/python/packaging/py3/packaging/__init__.py index 21695a74b51..3c6400263b8 100644 --- a/contrib/python/packaging/py3/packaging/__init__.py +++ b/contrib/python/packaging/py3/packaging/__init__.py @@ -6,7 +6,7 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "26.0" +__version__ = "26.1" __author__ = "Donald Stufft and individual contributors" __email__ = "[email protected]" diff --git a/contrib/python/packaging/py3/packaging/_structures.py b/contrib/python/packaging/py3/packaging/_structures.py deleted file mode 100644 index 225e2eee012..00000000000 --- a/contrib/python/packaging/py3/packaging/_structures.py +++ /dev/null @@ -1,69 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import typing - - -class InfinityType: - __slots__ = () - - def __repr__(self) -> str: - return "Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return False - - def __le__(self, other: object) -> bool: - return False - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return True - - def __ge__(self, other: object) -> bool: - return True - - def __neg__(self: object) -> "NegativeInfinityType": - return NegativeInfinity - - -Infinity = InfinityType() - - -class NegativeInfinityType: - __slots__ = () - - def __repr__(self) -> str: - return "-Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return True - - def __le__(self, other: object) -> bool: - return True - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return False - - def __ge__(self, other: object) -> bool: - return False - - def __neg__(self: object) -> InfinityType: - return Infinity - - -NegativeInfinity = NegativeInfinityType() diff --git a/contrib/python/packaging/py3/packaging/_tokenizer.py b/contrib/python/packaging/py3/packaging/_tokenizer.py index e6d20dd3f56..5ab891ccb8e 100644 --- a/contrib/python/packaging/py3/packaging/_tokenizer.py +++ b/contrib/python/packaging/py3/packaging/_tokenizer.py @@ -75,7 +75,7 @@ DEFAULT_RULES: dict[str, re.Pattern[str]] = { re.VERBOSE, ), "SPECIFIER": re.compile( - Specifier._operator_regex_str + Specifier._version_regex_str, + Specifier._specifier_regex_str, re.VERBOSE | re.IGNORECASE, ), "AT": re.compile(r"\@"), diff --git a/contrib/python/packaging/py3/packaging/dependency_groups.py b/contrib/python/packaging/py3/packaging/dependency_groups.py new file mode 100644 index 00000000000..413e5cb4b40 --- /dev/null +++ b/contrib/python/packaging/py3/packaging/dependency_groups.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence + +from .errors import _ErrorCollector +from .requirements import Requirement + +__all__ = [ + "CyclicDependencyGroup", + "DependencyGroupInclude", + "DependencyGroupResolver", + "DuplicateGroupNames", + "InvalidDependencyGroupObject", + "resolve_dependency_groups", +] + + +def __dir__() -> list[str]: + return __all__ + + +# ----------- +# Error Types +# ----------- + + +class DuplicateGroupNames(ValueError): + """ + The same dependency groups were defined twice, with different non-normalized names. + """ + + +class CyclicDependencyGroup(ValueError): + """ + The dependency group includes form a cycle. + """ + + def __init__(self, requested_group: str, group: str, include_group: str) -> None: + self.requested_group = requested_group + self.group = group + self.include_group = include_group + + if include_group == group: + reason = f"{group} includes itself" + else: + reason = f"{include_group} -> {group}, {group} -> {include_group}" + super().__init__( + "Cyclic dependency group include while resolving " + f"{requested_group}: {reason}" + ) + + +# in the PEP 735 spec, the tables in dependency group lists were described as +# "Dependency Object Specifiers", but the only defined type of object was a +# "Dependency Group Include" -- hence the naming of this error as "Object" +class InvalidDependencyGroupObject(ValueError): + """ + A member of a dependency group was identified as a dict, but was not in a valid + format. + """ + + +# ------------------------ +# Object Model & Interface +# ------------------------ + + +class DependencyGroupInclude: + __slots__ = ("include_group",) + + def __init__(self, include_group: str) -> None: + """ + Initialize a DependencyGroupInclude. + + :param include_group: The name of the group referred to by this include. + """ + self.include_group = include_group + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.include_group!r})" + + +class DependencyGroupResolver: + """ + A resolver for Dependency Group data. + + This class handles caching, name normalization, cycle detection, and other + parsing requirements. There are only two public methods for exploring the data: + ``lookup()`` and ``resolve()``. + + :param dependency_groups: A mapping, as provided via pyproject + ``[dependency-groups]``. + """ + + def __init__( + self, + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], + ) -> None: + errors = _ErrorCollector() + + self.dependency_groups = _normalize_group_names(dependency_groups, errors) + + # a map of group names to parsed data + self._parsed_groups: dict[ + str, tuple[Requirement | DependencyGroupInclude, ...] + ] = {} + # a map of group names to their ancestors, used for cycle detection + self._include_graph_ancestors: dict[str, tuple[str, ...]] = {} + # a cache of completed resolutions to Requirement lists + self._resolve_cache: dict[str, tuple[Requirement, ...]] = {} + + errors.finalize("[dependency-groups] data was invalid") + + def lookup(self, group: str) -> tuple[Requirement | DependencyGroupInclude, ...]: + """ + Lookup a group name, returning the parsed dependency data for that group. + This will not resolve includes. + + :param group: the name of the group to lookup + """ + group = _normalize_name(group) + + with _ErrorCollector().on_exit( + f"[dependency-groups] data for {group!r} was malformed" + ) as errors: + return self._parse_group(group, errors) + + def resolve(self, group: str) -> tuple[Requirement, ...]: + """ + Resolve a dependency group to a list of requirements. + + :param group: the name of the group to resolve + """ + group = _normalize_name(group) + + with _ErrorCollector().on_exit( + f"[dependency-groups] data for {group!r} was malformed" + ) as errors: + return self._resolve(group, group, errors) + + def _resolve( + self, group: str, requested_group: str, errors: _ErrorCollector + ) -> tuple[Requirement, ...]: + """ + This is a helper for cached resolution to strings. It preserves the name of the + group which the user initially requested in order to present a clearer error in + the event that a cycle is detected. + + :param group: The normalized name of the group to resolve. + :param requested_group: The group which was used in the original, user-facing + request. + """ + if group in self._resolve_cache: + return self._resolve_cache[group] + + parsed = self._parse_group(group, errors) + + resolved_group = [] + + for item in parsed: + if isinstance(item, Requirement): + resolved_group.append(item) + elif isinstance(item, DependencyGroupInclude): + include_group = _normalize_name(item.include_group) + + # if a group is cyclic, record the error + # otherwise, follow the include_group reference + # + # this allows us to examine all includes in a group, even in the + # presence of errors + if include_group in self._include_graph_ancestors.get(group, ()): + errors.error( + CyclicDependencyGroup( + requested_group, group, item.include_group + ) + ) + else: + self._include_graph_ancestors[include_group] = ( + *self._include_graph_ancestors.get(group, ()), + group, + ) + resolved_group.extend( + self._resolve(include_group, requested_group, errors) + ) + else: # pragma: no cover + raise NotImplementedError( + f"Invalid dependency group item after parse: {item}" + ) + + # in the event that errors were detected, present the group as empty and do not + # cache the result + # this ensures that repeated access to a cyclic group will raise multiple errors + if errors.errors: + return () + + self._resolve_cache[group] = tuple(resolved_group) + return self._resolve_cache[group] + + def _parse_group( + self, group: str, errors: _ErrorCollector + ) -> tuple[Requirement | DependencyGroupInclude, ...]: + # short circuit -- never do the work twice + if group in self._parsed_groups: + return self._parsed_groups[group] + + if group not in self.dependency_groups: + errors.error(LookupError(f"Dependency group '{group}' not found")) + return () + + raw_group = self.dependency_groups[group] + if isinstance(raw_group, str): + errors.error( + TypeError( + f"Dependency group {group!r} contained a string rather than a list." + ) + ) + return () + + if not isinstance(raw_group, Sequence): + errors.error( + TypeError(f"Dependency group {group!r} is not a sequence type.") + ) + return () + + elements: list[Requirement | DependencyGroupInclude] = [] + for item in raw_group: + if isinstance(item, str): + # packaging.requirements.Requirement parsing ensures that this is a + # valid PEP 508 Dependency Specifier + # raises InvalidRequirement on failure + elements.append(Requirement(item)) + elif isinstance(item, Mapping): + if tuple(item.keys()) != ("include-group",): + errors.error( + InvalidDependencyGroupObject( + f"Invalid dependency group item: {item!r}" + ) + ) + else: + include_group = item["include-group"] + elements.append(DependencyGroupInclude(include_group=include_group)) + else: + errors.error(TypeError(f"Invalid dependency group item: {item!r}")) + + self._parsed_groups[group] = tuple(elements) + return self._parsed_groups[group] + + +# -------------------- +# Functional Interface +# -------------------- + + +def resolve_dependency_groups( + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], /, *groups: str +) -> tuple[str, ...]: + """ + Resolve a dependency group to a tuple of requirements, as strings. + + :param dependency_groups: the parsed contents of the ``[dependency-groups]`` table + from ``pyproject.toml`` + :param groups: the name of the group(s) to resolve + """ + resolver = DependencyGroupResolver(dependency_groups) + return tuple(str(r) for group in groups for r in resolver.resolve(group)) + + +# ---------------- +# internal helpers +# ---------------- + + +_NORMALIZE_PATTERN = re.compile(r"[-_.]+") + + +def _normalize_name(name: str) -> str: + return _NORMALIZE_PATTERN.sub("-", name).lower() + + +def _normalize_group_names( + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], + errors: _ErrorCollector, +) -> dict[str, Sequence[str | Mapping[str, str]]]: + original_names: dict[str, list[str]] = {} + normalized_groups: dict[str, Sequence[str | Mapping[str, str]]] = {} + + for group_name, value in dependency_groups.items(): + normed_group_name = _normalize_name(group_name) + original_names.setdefault(normed_group_name, []).append(group_name) + normalized_groups[normed_group_name] = value + + for normed_name, names in original_names.items(): + if len(names) > 1: + errors.error( + DuplicateGroupNames( + "Duplicate dependency group names: " + f"{normed_name} ({', '.join(names)})" + ) + ) + + return normalized_groups diff --git a/contrib/python/packaging/py3/packaging/direct_url.py b/contrib/python/packaging/py3/packaging/direct_url.py new file mode 100644 index 00000000000..5d1c56ca9d6 --- /dev/null +++ b/contrib/python/packaging/py3/packaging/direct_url.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import dataclasses +import re +import urllib.parse +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Protocol, TypeVar + +if TYPE_CHECKING: # pragma: no cover + import sys + from collections.abc import Collection + + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self + +__all__ = [ + "ArchiveInfo", + "DirInfo", + "DirectUrl", + "DirectUrlValidationError", + "VcsInfo", +] + + +def __dir__() -> list[str]: + return __all__ + + +_T = TypeVar("_T") + + +class _FromMappingProtocol(Protocol): # pragma: no cover + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... + + +_FromMappingProtocolT = TypeVar("_FromMappingProtocolT", bound=_FromMappingProtocol) + + +def _json_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: + return {key: value for key, value in data if value is not None} + + +def _get(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T | None: + """Get a value from the dictionary and verify it's the expected type.""" + if (value := d.get(key)) is None: + return None + if not isinstance(value, expected_type): + raise DirectUrlValidationError( + f"Unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})", + context=key, + ) + return value + + +def _get_required(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T: + """Get a required value from the dictionary and verify it's the expected type.""" + if (value := _get(d, expected_type, key)) is None: + raise _DirectUrlRequiredKeyError(key) + return value + + +def _get_object( + d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str +) -> _FromMappingProtocolT | None: + """Get a dictionary value from the dictionary and convert it to a dataclass.""" + if (value := _get(d, Mapping, key)) is None: # type: ignore[type-abstract] + return None + try: + return target_type._from_dict(value) + except Exception as e: + raise DirectUrlValidationError(e, context=key) from e + + +_PEP610_USER_PASS_ENV_VARS_REGEX = re.compile( + r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$" +) + + +def _strip_auth_from_netloc(netloc: str, safe_user_passwords: Collection[str]) -> str: + if "@" not in netloc: + return netloc + user_pass, netloc_no_user_pass = netloc.split("@", 1) + if user_pass in safe_user_passwords: + return netloc + if _PEP610_USER_PASS_ENV_VARS_REGEX.match(user_pass): + return netloc + return netloc_no_user_pass + + +def _strip_url(url: str, safe_user_passwords: Collection[str]) -> str: + """url with user:password part removed unless it is formed with + environment variables as specified in PEP 610, or it is a safe user:password + such as `git`. + """ + parsed_url = urllib.parse.urlsplit(url) + netloc = _strip_auth_from_netloc(parsed_url.netloc, safe_user_passwords) + return urllib.parse.urlunsplit( + ( + parsed_url.scheme, + netloc, + parsed_url.path, + parsed_url.query, + parsed_url.fragment, + ) + ) + + +class DirectUrlValidationError(Exception): + """Raised when when input data is not spec-compliant.""" + + context: str | None = None + message: str + + def __init__( + self, + cause: str | Exception, + *, + context: str | None = None, + ) -> None: + if isinstance(cause, DirectUrlValidationError): + if cause.context: + self.context = ( + f"{context}.{cause.context}" if context else cause.context + ) + else: + self.context = context # pragma: no cover + self.message = cause.message + else: + self.context = context + self.message = str(cause) + + def __str__(self) -> str: + if self.context: + return f"{self.message} in {self.context!r}" + return self.message + + +class _DirectUrlRequiredKeyError(DirectUrlValidationError): + def __init__(self, key: str) -> None: + super().__init__("Missing required value", context=key) + + [email protected](frozen=True, init=False) +class VcsInfo: + vcs: str + commit_id: str + requested_revision: str | None = None + + def __init__( + self, + *, + vcs: str, + commit_id: str, + requested_revision: str | None = None, + ) -> None: + object.__setattr__(self, "vcs", vcs) + object.__setattr__(self, "commit_id", commit_id) + object.__setattr__(self, "requested_revision", requested_revision) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + # We can't validate vcs value because is not closed. + return cls( + vcs=_get_required(d, str, "vcs"), + requested_revision=_get(d, str, "requested_revision"), + commit_id=_get_required(d, str, "commit_id"), + ) + + [email protected](frozen=True, init=False) +class ArchiveInfo: + hashes: Mapping[str, str] | None = None + + def __init__( + self, + *, + hashes: Mapping[str, str] | None = None, + ) -> None: + object.__setattr__(self, "hashes", hashes) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + hashes = _get(d, Mapping, "hashes") # type: ignore[type-abstract] + if hashes is not None and not all(isinstance(h, str) for h in hashes.values()): + raise DirectUrlValidationError( + "Hash values must be strings", context="hashes" + ) + legacy_hash = _get(d, str, "hash") + if legacy_hash is not None: + if "=" not in legacy_hash: + raise DirectUrlValidationError( + "Invalid hash format (expected '<algorithm>=<hash>')", + context="hash", + ) + hash_algorithm, hash_value = legacy_hash.split("=", 1) + if hashes is None: + # if `hashes` are not present, we can derive it from the legacy `hash` + hashes = {hash_algorithm: hash_value} + else: + # if `hashes` are present, the legacy `hash` must match one of them + if hash_algorithm not in hashes: + raise DirectUrlValidationError( + f"Algorithm {hash_algorithm!r} used in hash field " + f"is not present in hashes field", + context="hashes", + ) + if hashes[hash_algorithm] != hash_value: + raise DirectUrlValidationError( + f"Algorithm {hash_algorithm!r} used in hash field " + f"has different value in hashes field", + context="hash", + ) + return cls(hashes=hashes) + + [email protected](frozen=True, init=False) +class DirInfo: + editable: bool | None = None + + def __init__( + self, + *, + editable: bool | None = None, + ) -> None: + object.__setattr__(self, "editable", editable) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + editable=_get(d, bool, "editable"), + ) + + [email protected](frozen=True, init=False) +class DirectUrl: + """A class representing a direct URL.""" + + url: str + archive_info: ArchiveInfo | None = None + vcs_info: VcsInfo | None = None + dir_info: DirInfo | None = None + subdirectory: str | None = None # XXX Path or str? + + def __init__( + self, + *, + url: str, + archive_info: ArchiveInfo | None = None, + vcs_info: VcsInfo | None = None, + dir_info: DirInfo | None = None, + subdirectory: str | None = None, + ) -> None: + object.__setattr__(self, "url", url) + object.__setattr__(self, "archive_info", archive_info) + object.__setattr__(self, "vcs_info", vcs_info) + object.__setattr__(self, "dir_info", dir_info) + object.__setattr__(self, "subdirectory", subdirectory) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + direct_url = cls( + url=_get_required(d, str, "url"), + archive_info=_get_object(d, ArchiveInfo, "archive_info"), + vcs_info=_get_object(d, VcsInfo, "vcs_info"), + dir_info=_get_object(d, DirInfo, "dir_info"), + subdirectory=_get(d, str, "subdirectory"), + ) + if ( + bool(direct_url.vcs_info) + + bool(direct_url.archive_info) + + bool(direct_url.dir_info) + ) != 1: + raise DirectUrlValidationError( + "Exactly one of vcs_info, archive_info, dir_info must be present" + ) + if direct_url.dir_info is not None and not direct_url.url.startswith("file://"): + raise DirectUrlValidationError( + "URL scheme must be file:// when dir_info is present", + context="url", + ) + # XXX subdirectory must be relative, can we, should we validate that here? + return direct_url + + @classmethod + def from_dict(cls, d: Mapping[str, Any], /) -> Self: + """Create and validate a DirectUrl instance from a JSON dictionary.""" + return cls._from_dict(d) + + def to_dict( + self, + *, + generate_legacy_hash: bool = False, + strip_user_password: bool = True, + safe_user_passwords: Collection[str] = ("git",), + ) -> Mapping[str, Any]: + """Convert the DirectUrl instance to a JSON dictionary. + + :param generate_legacy_hash: If True, include a legacy `hash` field in + `archive_info` for backward compatibility with tools that don't + support the `hashes` field. + :param strip_user_password: If True, strip user:password from the URL + unless it is formed with environment variables as specified in PEP + 610, or it is a safe user:password such as `git`. + :param safe_user_passwords: A collection of user:password strings that + should not be stripped from the URL even if `strip_user_password` is + True. + """ + res = dataclasses.asdict(self, dict_factory=_json_dict_factory) + if generate_legacy_hash and self.archive_info and self.archive_info.hashes: + hash_algorithm, hash_value = next(iter(self.archive_info.hashes.items())) + res["archive_info"]["hash"] = f"{hash_algorithm}={hash_value}" + if strip_user_password: + res["url"] = _strip_url(self.url, safe_user_passwords) + return res + + def validate(self) -> None: + """Validate the DirectUrl instance against the specification. + + Raises :class:`DirectUrlValidationError` if invalid. + """ + self.from_dict(self.to_dict()) diff --git a/contrib/python/packaging/py3/packaging/errors.py b/contrib/python/packaging/py3/packaging/errors.py new file mode 100644 index 00000000000..d1d47cf6c34 --- /dev/null +++ b/contrib/python/packaging/py3/packaging/errors.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import contextlib +import dataclasses +import sys +import typing + +__all__ = ["ExceptionGroup"] + + +def __dir__() -> list[str]: + return __all__ + + +if sys.version_info >= (3, 11): # pragma: no cover + from builtins import ExceptionGroup +else: # pragma: no cover + + class ExceptionGroup(Exception): + """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. + + If :external:exc:`ExceptionGroup` is already defined by Python itself, + that version is used instead. + """ + + message: str + exceptions: list[Exception] + + def __init__(self, message: str, exceptions: list[Exception]) -> None: + self.message = message + self.exceptions = exceptions + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" + + +class _ErrorCollector: + """ + Collect errors into ExceptionGroups. + + Used like this: + + collector = _ErrorCollector() + # Add a single exception + collector.error(ValueError("one")) + + # Supports nesting, including combining ExceptionGroups + with collector.collect(): + raise ValueError("two") + collector.finalize("Found some errors") + + Since making a collector and then calling finalize later is a common pattern, + a convenience method ``on_exit`` is provided. + """ + + errors: list[Exception] = dataclasses.field(default_factory=list, init=False) + + def finalize(self, msg: str) -> None: + """Raise a group exception if there are any errors.""" + if self.errors: + raise ExceptionGroup(msg, self.errors) + + @contextlib.contextmanager + def on_exit(self, msg: str) -> typing.Generator[_ErrorCollector, None, None]: + """ + Calls finalize if no uncollected errors were present. + + Uncollected errors are raised normally. + """ + yield self + self.finalize(msg) + + @contextlib.contextmanager + def collect(self, *err_cls: type[Exception]) -> typing.Generator[None, None, None]: + """ + Context manager to collect errors into the error list. + + Must be inside loops, as only one error can be collected at a time. + """ + error_classes = err_cls or (Exception,) + try: + yield + except ExceptionGroup as error: + self.errors.extend(error.exceptions) + except error_classes as error: + self.errors.append(error) + + def error( + self, + error: Exception, + ) -> None: + """Add an error to the list.""" + self.errors.append(error) diff --git a/contrib/python/packaging/py3/packaging/licenses/__init__.py b/contrib/python/packaging/py3/packaging/licenses/__init__.py index 335b275fa75..36e46ed02ca 100644 --- a/contrib/python/packaging/py3/packaging/licenses/__init__.py +++ b/contrib/python/packaging/py3/packaging/licenses/__init__.py @@ -42,14 +42,25 @@ __all__ = [ "canonicalize_license_expression", ] + +# Simple __dir__ implementation since there are no public submodules +def __dir__() -> list[str]: + return __all__ + + license_ref_allowed = re.compile("^[A-Za-z0-9.-]*$") NormalizedLicenseExpression = NewType("NormalizedLicenseExpression", str) +""" +A :class:`typing.NewType` of :class:`str`, representing a normalized +License-Expression. +""" class InvalidLicenseExpression(ValueError): """Raised when a license-expression string is invalid + >>> from packaging.licenses import canonicalize_license_expression >>> canonicalize_license_expression("invalid") Traceback (most recent call last): ... @@ -60,6 +71,34 @@ class InvalidLicenseExpression(ValueError): def canonicalize_license_expression( raw_license_expression: str, ) -> NormalizedLicenseExpression: + """ + This function takes a valid License-Expression, and returns the normalized + form of it. + + The return type is typed as :class:`NormalizedLicenseExpression`. This + allows type checkers to help require that a string has passed through this + function before use. + + :param str raw_license_expression: The License-Expression to canonicalize. + :raises InvalidLicenseExpression: If the License-Expression is invalid due to an + invalid/unknown license identifier or invalid syntax. + + .. doctest:: + + >>> from packaging.licenses import canonicalize_license_expression + >>> canonicalize_license_expression("mit") + 'MIT' + >>> canonicalize_license_expression("mit and (apache-2.0 or bsd-2-clause)") + 'MIT AND (Apache-2.0 OR BSD-2-Clause)' + >>> canonicalize_license_expression("(mit") + Traceback (most recent call last): + ... + InvalidLicenseExpression: Invalid license expression: '(mit' + >>> canonicalize_license_expression("Use-it-after-midnight") + Traceback (most recent call last): + ... + InvalidLicenseExpression: Unknown license: 'Use-it-after-midnight' + """ if not raw_license_expression: message = f"Invalid license expression: {raw_license_expression!r}" raise InvalidLicenseExpression(message) diff --git a/contrib/python/packaging/py3/packaging/markers.py b/contrib/python/packaging/py3/packaging/markers.py index ca3706fe492..65d7f330b0c 100644 --- a/contrib/python/packaging/py3/packaging/markers.py +++ b/contrib/python/packaging/py3/packaging/markers.py @@ -26,8 +26,22 @@ __all__ = [ "default_environment", ] + +def __dir__() -> list[str]: + return __all__ + + Operator = Callable[[str, Union[str, AbstractSet[str]]], bool] EvaluateContext = Literal["metadata", "lock_file", "requirement"] +"""A ``typing.Literal`` enumerating valid marker evaluation contexts. + +Valid values for the ``context`` passed to :meth:`Marker.evaluate` are: + +* ``"metadata"`` (for core metadata; default) +* ``"lock_file"`` (for lock files) +* ``"requirement"`` (i.e. all other situations) +""" + MARKERS_ALLOWING_SET = {"extras", "dependency_groups"} MARKERS_REQUIRING_VERSION = { "implementation_version", @@ -38,25 +52,32 @@ MARKERS_REQUIRING_VERSION = { class InvalidMarker(ValueError): - """ - An invalid marker was found, users should refer to PEP 508. + """Raised when attempting to create a :class:`Marker` from invalid input. + + This error indicates that the given marker string does not conform to the + :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`. """ class UndefinedComparison(ValueError): - """ - An invalid operation was attempted on a value that doesn't support it. + """Raised when evaluating an unsupported marker comparison. + + This can happen when marker values are compared as versions but do not + conform to the :ref:`specification of version specifiers + <pypug:version-specifiers>`. """ class UndefinedEnvironmentName(ValueError): - """ - A name was attempted to be used that does not exist inside of the - environment. - """ + """Raised when evaluating a marker that references a missing environment key.""" class Environment(TypedDict): + """ + A dictionary that represents a Python environment as captured by + :func:`default_environment`. All fields are required. + """ + implementation_name: str """The implementation's identifier, e.g. ``'cpython'``.""" @@ -263,7 +284,7 @@ def _evaluate_markers( return any(all(item) for item in groups) -def format_full_version(info: sys._version_info) -> str: +def _format_full_version(info: sys._version_info) -> str: version = f"{info.major}.{info.minor}.{info.micro}" kind = info.releaselevel if kind != "final": @@ -272,7 +293,11 @@ def format_full_version(info: sys._version_info) -> str: def default_environment() -> Environment: - iver = format_full_version(sys.implementation.version) + """Return the default marker environment for the current Python process. + + This is the base environment used by :meth:`Marker.evaluate`. + """ + iver = _format_full_version(sys.implementation.version) implementation_name = sys.implementation.name return { "implementation_name": implementation_name, @@ -290,6 +315,17 @@ def default_environment() -> Environment: class Marker: + """Represents a parsed dependency marker expression. + + Marker expressions are parsed according to the + :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`. + + :param marker: The string representation of a marker expression. + :raises InvalidMarker: If ``marker`` cannot be parsed. + """ + + __slots__ = ("_markers",) + def __init__(self, marker: str) -> None: # Note: We create a Marker object without calling this constructor in # packaging.requirements.Requirement. If any additional logic is @@ -320,11 +356,21 @@ class Marker: except ParserSyntaxError as e: raise InvalidMarker(str(e)) from e + @classmethod + def _from_markers(cls, markers: MarkerList) -> Marker: + """Create a Marker instance from a pre-parsed marker tree. + + This avoids re-parsing serialised marker strings when combining markers. + """ + new = cls.__new__(cls) + new._markers = markers + return new + def __str__(self) -> str: return _format_marker(self._markers) def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __hash__(self) -> int: return hash(str(self)) @@ -335,6 +381,16 @@ class Marker: return str(self) == str(other) + def __and__(self, other: Marker) -> Marker: + if not isinstance(other, Marker): + return NotImplemented + return self._from_markers([self._markers, "and", other._markers]) + + def __or__(self, other: Marker) -> Marker: + if not isinstance(other, Marker): + return NotImplemented + return self._from_markers([self._markers, "or", other._markers]) + def evaluate( self, environment: Mapping[str, str | AbstractSet[str]] | None = None, @@ -342,14 +398,23 @@ class Marker: ) -> bool: """Evaluate a marker. - Return the boolean from evaluating the given marker against the - environment. environment is an optional argument to override all or - part of the determined environment. The *context* parameter specifies what - context the markers are being evaluated for, which influences what markers - are considered valid. Acceptable values are "metadata" (for core metadata; - default), "lock_file", and "requirement" (i.e. all other situations). + Return the boolean from evaluating this marker against the environment. + The environment is determined from the current Python process unless + passed in explicitly. + + :param environment: Mapping containing keys and values to override the + detected environment. + :param EvaluateContext context: The context in which the marker is + evaluated, which influences what marker names are considered valid. + Accepted values are ``"metadata"`` (for core metadata; default), + ``"lock_file"``, and ``"requirement"`` (i.e. all other situations). + :raises UndefinedComparison: If the marker uses a comparison on values + that are not valid versions per the :ref:`specification of version + specifiers <pypug:version-specifiers>`. + :raises UndefinedEnvironmentName: If the marker references a value that + is missing from the evaluation environment. + :returns: ``True`` if the marker matches, otherwise ``False``. - The environment is determined from the current Python process. """ current_environment = cast( "dict[str, str | AbstractSet[str]]", default_environment() diff --git a/contrib/python/packaging/py3/packaging/metadata.py b/contrib/python/packaging/py3/packaging/metadata.py index 253f6b1b7eb..b3269a45e9c 100644 --- a/contrib/python/packaging/py3/packaging/metadata.py +++ b/contrib/python/packaging/py3/packaging/metadata.py @@ -1,13 +1,11 @@ from __future__ import annotations -import email.feedparser import email.header import email.message import email.parser import email.policy import keyword import pathlib -import sys import typing from typing import ( Any, @@ -20,6 +18,7 @@ from typing import ( from . import licenses, requirements, specifiers, utils from . import version as version_module +from .errors import ExceptionGroup, _ErrorCollector if typing.TYPE_CHECKING: from .licenses import NormalizedLicenseExpression @@ -27,26 +26,18 @@ if typing.TYPE_CHECKING: T = typing.TypeVar("T") -if sys.version_info >= (3, 11): # pragma: no cover - ExceptionGroup = ExceptionGroup # noqa: F821 -else: # pragma: no cover +__all__ = [ + "InvalidMetadata", + "Metadata", + "RFC822Message", + "RFC822Policy", + "RawMetadata", + "parse_email", +] - class ExceptionGroup(Exception): - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: list[Exception] - - def __init__(self, message: str, exceptions: list[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" +def __dir__() -> list[str]: + return __all__ class InvalidMetadata(ValueError): @@ -76,8 +67,8 @@ class RawMetadata(TypedDict, total=False): Core metadata fields that can be specified multiple times are stored as a list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - + which hold multiple values in a single field are stored as a list. All fields + are considered optional. """ # Metadata 1.0 - PEP 241 @@ -641,7 +632,7 @@ class _Validator(Generic[T]): charset = parameters.get("charset", "UTF-8") if charset != "UTF-8": raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" + f"{{field}} can only specify the UTF-8 charset, not {charset!r}" ) markdown_variants = {"GFM", "CommonMark"} @@ -784,13 +775,11 @@ class Metadata: ins._raw = data.copy() # Mutations occur due to caching enriched values. if validate: - exceptions: list[Exception] = [] - try: + collector = _ErrorCollector() + metadata_version = None + with collector.collect(InvalidMetadata): metadata_version = ins.metadata_version metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None # Make sure to check for the fields that are present, the required # fields (so their absence can be reported). @@ -807,7 +796,7 @@ class Metadata: field_metadata_version = cls.__dict__[key].added except KeyError: exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) + collector.error(exc) continue field_age = _VALID_METADATA_VERSIONS.index( field_metadata_version @@ -819,14 +808,13 @@ class Metadata: f"{field} introduced in metadata version " f"{field_metadata_version}, not {metadata_version}", ) - exceptions.append(exc) + collector.error(exc) continue getattr(ins, key) except InvalidMetadata as exc: - exceptions.append(exc) + collector.error(exc) - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) + collector.finalize("invalid metadata") return ins @@ -840,16 +828,13 @@ class Metadata: raw, unparsed = parse_email(data) if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) + with _ErrorCollector().on_exit("unparsed") as collector: + for unparsed_key in unparsed: + if unparsed_key in _EMAIL_TO_RAW_MAPPING: + message = f"{unparsed_key!r} has invalid data" + else: + message = f"unrecognized field: {unparsed_key!r}" + collector.error(InvalidMetadata(unparsed_key, message)) try: return cls.from_raw(raw, validate=validate) diff --git a/contrib/python/packaging/py3/packaging/pylock.py b/contrib/python/packaging/py3/packaging/pylock.py index a564f15246a..84e25378fc6 100644 --- a/contrib/python/packaging/py3/packaging/pylock.py +++ b/contrib/python/packaging/py3/packaging/pylock.py @@ -12,18 +12,29 @@ from typing import ( Callable, Protocol, TypeVar, + cast, ) +from urllib.parse import urlparse -from .markers import Marker +from .markers import Environment, Marker, default_environment from .specifiers import SpecifierSet -from .utils import NormalizedName, is_normalized_name +from .tags import create_compatible_tags_selector, sys_tags +from .utils import ( + NormalizedName, + is_normalized_name, + parse_sdist_filename, + parse_wheel_filename, +) from .version import Version if TYPE_CHECKING: # pragma: no cover + from collections.abc import Collection, Iterator from pathlib import Path from typing_extensions import Self + from .tags import Tag + _logger = logging.getLogger(__name__) __all__ = [ @@ -39,6 +50,11 @@ __all__ = [ "is_valid_pylock_path", ] + +def __dir__() -> list[str]: + return __all__ + + _T = TypeVar("_T") _T2 = TypeVar("_T2") @@ -222,6 +238,26 @@ def _validate_path_url(path: str | None, url: str | None) -> None: raise PylockValidationError("path or url must be provided") +def _path_name(path: str | None) -> str | None: + if not path: + return None + # If the path is relative it MAY use POSIX-style path separators explicitly + # for portability + if "/" in path: + return path.rsplit("/", 1)[-1] + elif "\\" in path: + return path.rsplit("\\", 1)[-1] + else: + return path + + +def _url_name(url: str | None) -> str | None: + if not url: + return None + url_path = urlparse(url).path + return url_path.rsplit("/", 1)[-1] + + def _validate_hashes(hashes: Mapping[str, Any]) -> Mapping[str, Any]: if not hashes: raise PylockValidationError("At least one hash must be provided") @@ -269,6 +305,10 @@ class PylockUnsupportedVersionError(PylockValidationError): """Raised when encountering an unsupported `lock_version`.""" +class PylockSelectError(Exception): + """Base exception for errors raised by :meth:`Pylock.select`.""" + + @dataclass(frozen=True, init=False) class PackageVcs: type: str @@ -418,6 +458,14 @@ class PackageSdist: _validate_path_url(package_sdist.path, package_sdist.url) return package_sdist + @property + def filename(self) -> str: + """Get the filename of the sdist.""" + filename = self.name or _path_name(self.path) or _url_name(self.url) + if not filename: + raise PylockValidationError("Cannot determine sdist filename") + return filename + @dataclass(frozen=True, init=False) class PackageWheel: @@ -459,6 +507,14 @@ class PackageWheel: _validate_path_url(package_wheel.path, package_wheel.url) return package_wheel + @property + def filename(self) -> str: + """Get the filename of the wheel.""" + filename = self.name or _path_name(self.path) or _url_name(self.url) + if not filename: + raise PylockValidationError("Cannot determine wheel filename") + return filename + @dataclass(frozen=True, init=False) class Package: @@ -538,6 +594,46 @@ class Package: "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" ) + for i, wheel in enumerate(package.wheels or []): + try: + (name, version, _, _) = parse_wheel_filename(wheel.filename) + except Exception as e: + raise PylockValidationError( + f"Invalid wheel filename {wheel.filename!r}", + context=f"wheels[{i}]", + ) from e + if name != package.name: + raise PylockValidationError( + f"Name in {wheel.filename!r} is not consistent with " + f"package name {package.name!r}", + context=f"wheels[{i}]", + ) + if package.version and version != package.version: + raise PylockValidationError( + f"Version in {wheel.filename!r} is not consistent with " + f"package version {str(package.version)!r}", + context=f"wheels[{i}]", + ) + if package.sdist: + try: + name, version = parse_sdist_filename(package.sdist.filename) + except Exception as e: + raise PylockValidationError( + f"Invalid sdist filename {package.sdist.filename!r}", + context="sdist", + ) from e + if name != package.name: + raise PylockValidationError( + f"Name in {package.sdist.filename!r} is not consistent with " + f"package name {package.name!r}", + context="sdist", + ) + if package.version and version != package.version: + raise PylockValidationError( + f"Version in {package.sdist.filename!r} is not consistent with " + f"package version {str(package.version)!r}", + context="sdist", + ) try: for i, attestation_identity in enumerate( # noqa: B007 package.attestation_identities or [] @@ -633,3 +729,177 @@ class Pylock: Raises :class:`PylockValidationError` otherwise.""" self.from_dict(self.to_dict()) + + def select( + self, + *, + environment: Environment | None = None, + tags: Sequence[Tag] | None = None, + extras: Collection[str] | None = None, + dependency_groups: Collection[str] | None = None, + ) -> Iterator[ + tuple[ + Package, + PackageVcs + | PackageDirectory + | PackageArchive + | PackageWheel + | PackageSdist, + ] + ]: + """Select what to install from the lock file. + + The *environment* and *tags* parameters represent the environment being + selected for. If unspecified, ``packaging.markers.default_environment()`` and + ``packaging.tags.sys_tags()`` are used. + + The *extras* parameter represents the extras to install. + + The *dependency_groups* parameter represents the groups to install. If + unspecified, the default groups are used. + + This method must be used on valid Pylock instances (i.e. one obtained + from :meth:`Pylock.from_dict` or if constructed manually, after calling + :meth:`Pylock.validate`). + """ + compatible_tags_selector = create_compatible_tags_selector(tags or sys_tags()) + + # #. Gather the extras and dependency groups to install and set ``extras`` and + # ``dependency_groups`` for marker evaluation, respectively. + # + # #. ``extras`` SHOULD be set to the empty set by default. + # #. ``dependency_groups`` SHOULD be the set created from + # :ref:`pylock-default-groups` by default. + env = cast( + "dict[str, str | frozenset[str]]", + dict( + environment or {}, # Marker.evaluate will fill-up + extras=frozenset(extras or []), + dependency_groups=frozenset( + (self.default_groups or []) + if dependency_groups is None # to allow selecting no group + else dependency_groups + ), + ), + ) + env_python_full_version = ( + environment["python_full_version"] + if environment + else default_environment()["python_full_version"] + ) + + # #. Check if the metadata version specified by :ref:`pylock-lock-version` is + # supported; an error or warning MUST be raised as appropriate. + # Covered by lock.validate() which is a precondition for this method. + + # #. If :ref:`pylock-requires-python` is specified, check that the environment + # being installed for meets the requirement; an error MUST be raised if it is + # not met. + if self.requires_python and not self.requires_python.contains( + env_python_full_version, + ): + raise PylockSelectError( + f"python_full_version {env_python_full_version!r} " + f"in provided environment does not satisfy the Python version " + f"requirement {str(self.requires_python)!r}" + ) + + # #. If :ref:`pylock-environments` is specified, check that at least one of the + # environment marker expressions is satisfied; an error MUST be raised if no + # expression is satisfied. + if self.environments: + for env_marker in self.environments: + if env_marker.evaluate( + cast("dict[str, str]", environment or {}), context="requirement" + ): + break + else: + raise PylockSelectError( + "Provided environment does not satisfy any of the " + "environments specified in the lock file" + ) + + # #. For each package listed in :ref:`pylock-packages`: + selected_packages_by_name: dict[str, tuple[int, Package]] = {} + for package_index, package in enumerate(self.packages): + # #. If :ref:`pylock-packages-marker` is specified, check if it is + # satisfied;if it isn't, skip to the next package. + if package.marker and not package.marker.evaluate(env, context="lock_file"): + continue + + # #. If :ref:`pylock-packages-requires-python` is specified, check if it is + # satisfied; an error MUST be raised if it isn't. + if package.requires_python and not package.requires_python.contains( + env_python_full_version, + ): + raise PylockSelectError( + f"python_full_version {env_python_full_version!r} " + f"in provided environment does not satisfy the Python version " + f"requirement {str(package.requires_python)!r} for package " + f"{package.name!r} at packages[{package_index}]" + ) + + # #. Check that no other conflicting instance of the package has been slated + # to be installed; an error about the ambiguity MUST be raised otherwise. + if package.name in selected_packages_by_name: + raise PylockSelectError( + f"Multiple packages with the name {package.name!r} are " + f"selected at packages[{package_index}] and " + f"packages[{selected_packages_by_name[package.name][0]}]" + ) + + # #. Check that the source of the package is specified appropriately (i.e. + # there are no conflicting sources in the package entry); + # an error MUST be raised if any issues are found. + # Covered by lock.validate() which is a precondition for this method. + + # #. Add the package to the set of packages to install. + selected_packages_by_name[package.name] = (package_index, package) + + # #. For each package to be installed: + for package_index, package in selected_packages_by_name.values(): + # - If :ref:`pylock-packages-vcs` is set: + if package.vcs is not None: + yield package, package.vcs + + # - Else if :ref:`pylock-packages-directory` is set: + elif package.directory is not None: + yield package, package.directory + + # - Else if :ref:`pylock-packages-archive` is set: + elif package.archive is not None: + yield package, package.archive + + # - Else if there are entries for :ref:`pylock-packages-wheels`: + elif package.wheels: + # #. Look for the appropriate wheel file based on + # :ref:`pylock-packages-wheels-name`; if one is not found then move + # on to :ref:`pylock-packages-sdist` or an error MUST be raised about + # a lack of source for the project. + best_wheel = next( + compatible_tags_selector( + (wheel, parse_wheel_filename(wheel.filename)[-1]) + for wheel in package.wheels + ), + None, + ) + if best_wheel: + yield package, best_wheel + elif package.sdist is not None: + yield package, package.sdist + else: + raise PylockSelectError( + f"No wheel found matching the provided tags " + f"for package {package.name!r} " + f"at packages[{package_index}], " + f"and no sdist available as a fallback" + ) + + # - Else if no :ref:`pylock-packages-wheels` file is found or + # :ref:`pylock-packages-sdist` is solely set: + elif package.sdist is not None: + yield package, package.sdist + + else: + # Covered by lock.validate() which is a precondition for this method. + raise NotImplementedError # pragma: no cover diff --git a/contrib/python/packaging/py3/packaging/requirements.py b/contrib/python/packaging/py3/packaging/requirements.py index 3079be69bf8..18640d4386e 100644 --- a/contrib/python/packaging/py3/packaging/requirements.py +++ b/contrib/python/packaging/py3/packaging/requirements.py @@ -11,6 +11,15 @@ from .markers import Marker, _normalize_extra_values from .specifiers import SpecifierSet from .utils import canonicalize_name +__all__ = [ + "InvalidRequirement", + "Requirement", +] + + +def __dir__() -> list[str]: + return __all__ + class InvalidRequirement(ValueError): """ @@ -68,7 +77,7 @@ class Requirement: return "".join(self._iter_parts(self.name)) def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __hash__(self) -> int: return hash(tuple(self._iter_parts(canonicalize_name(self.name)))) diff --git a/contrib/python/packaging/py3/packaging/specifiers.py b/contrib/python/packaging/py3/packaging/specifiers.py index 5d26b0d1ae2..d0aa9a6cd33 100644 --- a/contrib/python/packaging/py3/packaging/specifiers.py +++ b/contrib/python/packaging/py3/packaging/specifiers.py @@ -11,17 +11,254 @@ from __future__ import annotations import abc +import enum +import functools import itertools import re -from typing import Callable, Final, Iterable, Iterator, TypeVar, Union +import typing +from typing import Any, Callable, Final, Iterable, Iterator, Sequence, TypeVar, Union from .utils import canonicalize_version from .version import InvalidVersion, Version +__all__ = [ + "BaseSpecifier", + "InvalidSpecifier", + "Specifier", + "SpecifierSet", +] + + +def __dir__() -> list[str]: + return __all__ + + +T = TypeVar("T") UnparsedVersion = Union[Version, str] UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) CallableOperator = Callable[[Version, str], bool] +# The smallest possible PEP 440 version. No valid version is less than this. +_MIN_VERSION: Final[Version] = Version("0.dev0") + + +def _trim_release(release: tuple[int, ...]) -> tuple[int, ...]: + """Strip trailing zeros from a release tuple for normalized comparison.""" + end = len(release) + while end > 1 and release[end - 1] == 0: + end -= 1 + return release if end == len(release) else release[:end] + + +class _BoundaryKind(enum.Enum): + """Where a boundary marker sits in the version ordering.""" + + AFTER_LOCALS = enum.auto() # after V+local, before V.post0 + AFTER_POSTS = enum.auto() # after V.postN, before next release + + [email protected]_ordering +class _BoundaryVersion: + """A point on the version line between two real PEP 440 versions. + + Some specifier semantics imply boundaries between real versions: + ``<=1.0`` includes ``1.0+local`` and ``>1.0`` excludes + ``1.0.post0``. No real :class:`Version` falls on those boundaries, + so this class creates values that sort between the real versions + on either side. + + Two kinds exist, shown relative to a base version V:: + + V < V+local < AFTER_LOCALS(V) < V.post0 < AFTER_POSTS(V) + + ``AFTER_LOCALS`` sits after V and every V+local, but before + V.post0. Upper bound of ``<=V``, ``==V``, ``!=V``. + + ``AFTER_POSTS`` sits after every V.postN, but before the next + release segment. Lower bound of ``>V`` (final or pre-release V) + to exclude post-releases per PEP 440. + """ + + __slots__ = ("_kind", "_trimmed_release", "version") + + def __init__(self, version: Version, kind: _BoundaryKind) -> None: + self.version = version + self._kind = kind + self._trimmed_release = _trim_release(version.release) + + def _is_family(self, other: Version) -> bool: + """Is ``other`` a version that this boundary sorts above?""" + v = self.version + if not ( + other.epoch == v.epoch + and _trim_release(other.release) == self._trimmed_release + and other.pre == v.pre + ): + return False + if self._kind == _BoundaryKind.AFTER_LOCALS: + # Local family: exact same public version (any local label). + return other.post == v.post and other.dev == v.dev + # Post family: same base + any post-release (or identical). + return other.dev == v.dev or other.post is not None + + def __eq__(self, other: object) -> bool: + if isinstance(other, _BoundaryVersion): + return self.version == other.version and self._kind == other._kind + return NotImplemented + + def __lt__(self, other: _BoundaryVersion | Version) -> bool: + if isinstance(other, _BoundaryVersion): + if self.version != other.version: + return self.version < other.version + return self._kind.value < other._kind.value + return not self._is_family(other) and self.version < other + + def __hash__(self) -> int: + return hash((self.version, self._kind)) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.version!r}, {self._kind.name})" + + [email protected]_ordering +class _LowerBound: + """Lower bound of a version range. + + A version *v* of ``None`` means unbounded below (-inf). + At equal versions, ``[v`` sorts before ``(v`` because an inclusive + bound starts earlier. + """ + + __slots__ = ("inclusive", "version") + + def __init__(self, version: _VersionOrBoundary, inclusive: bool) -> None: + self.version = version + self.inclusive = inclusive + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _LowerBound): + return NotImplemented # pragma: no cover + return self.version == other.version and self.inclusive == other.inclusive + + def __lt__(self, other: _LowerBound) -> bool: + if not isinstance(other, _LowerBound): # pragma: no cover + return NotImplemented + # -inf < anything (except -inf). + if self.version is None: + return other.version is not None + if other.version is None: + return False + if self.version != other.version: + return self.version < other.version + # [v < (v: inclusive starts earlier. + return self.inclusive and not other.inclusive + + def __hash__(self) -> int: + return hash((self.version, self.inclusive)) + + def __repr__(self) -> str: + bracket = "[" if self.inclusive else "(" + return f"<{self.__class__.__name__} {bracket}{self.version!r}>" + + [email protected]_ordering +class _UpperBound: + """Upper bound of a version range. + + A version *v* of ``None`` means unbounded above (+inf). + At equal versions, ``v)`` sorts before ``v]`` because an exclusive + bound ends earlier. + """ + + __slots__ = ("inclusive", "version") + + def __init__(self, version: _VersionOrBoundary, inclusive: bool) -> None: + self.version = version + self.inclusive = inclusive + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _UpperBound): + return NotImplemented # pragma: no cover + return self.version == other.version and self.inclusive == other.inclusive + + def __lt__(self, other: _UpperBound) -> bool: + if not isinstance(other, _UpperBound): # pragma: no cover + return NotImplemented + # Nothing < +inf (except +inf itself). + if self.version is None: + return False + if other.version is None: + return True + if self.version != other.version: + return self.version < other.version + # v) < v]: exclusive ends earlier. + return not self.inclusive and other.inclusive + + def __hash__(self) -> int: + return hash((self.version, self.inclusive)) + + def __repr__(self) -> str: + bracket = "]" if self.inclusive else ")" + return f"<{self.__class__.__name__} {self.version!r}{bracket}>" + + +if typing.TYPE_CHECKING: + _VersionOrBoundary = Union[Version, _BoundaryVersion, None] + + #: A single contiguous version range, represented as a + #: (lower bound, upper bound) pair. + _VersionRange = tuple[_LowerBound, _UpperBound] + +_NEG_INF = _LowerBound(None, False) +_POS_INF = _UpperBound(None, False) +_FULL_RANGE: tuple[_VersionRange] = ((_NEG_INF, _POS_INF),) + + +def _range_is_empty(lower: _LowerBound, upper: _UpperBound) -> bool: + """True when the range defined by *lower* and *upper* contains no versions.""" + if lower.version is None or upper.version is None: + return False + if lower.version == upper.version: + return not (lower.inclusive and upper.inclusive) + return lower.version > upper.version + + +def _intersect_ranges( + left: Sequence[_VersionRange], + right: Sequence[_VersionRange], +) -> list[_VersionRange]: + """Intersect two sorted, non-overlapping range lists (two-pointer merge).""" + result: list[_VersionRange] = [] + left_index = right_index = 0 + while left_index < len(left) and right_index < len(right): + left_lower, left_upper = left[left_index] + right_lower, right_upper = right[right_index] + + lower = max(left_lower, right_lower) + upper = min(left_upper, right_upper) + + if not _range_is_empty(lower, upper): + result.append((lower, upper)) + + # Advance whichever side has the smaller upper bound. + if left_upper < right_upper: + left_index += 1 + else: + right_index += 1 + + return result + + +def _next_prefix_dev0(version: Version) -> Version: + """Smallest version in the next prefix: 1.2 -> 1.3.dev0.""" + release = (*version.release[:-1], version.release[-1] + 1) + return Version.from_parts(epoch=version.epoch, release=release, dev=0) + + +def _base_dev0(version: Version) -> Version: + """The .dev0 of a version's base release: 1.2 -> 1.2.dev0.""" + return Version.from_parts(epoch=version.epoch, release=version.release, dev=0) + def _coerce_version(version: UnparsedVersion) -> Version | None: if not isinstance(version, Version): @@ -33,11 +270,46 @@ def _coerce_version(version: UnparsedVersion) -> Version | None: def _public_version(version: Version) -> Version: + if version.local is None: + return version return version.__replace__(local=None) -def _base_version(version: Version) -> Version: - return version.__replace__(pre=None, post=None, dev=None, local=None) +def _post_base(version: Version) -> Version: + """The version that *version* is a post-release of. + + 1.0.post1 -> 1.0, 1.0a1.post0 -> 1.0a1, 1.0.post0.dev1 -> 1.0. + """ + return version.__replace__(post=None, dev=None, local=None) + + +def _earliest_prerelease(version: Version) -> Version: + """Earliest pre-release of *version*. + + 1.2 -> 1.2.dev0, 1.2.post1 -> 1.2.post1.dev0. + """ + return version.__replace__(dev=0, local=None) + + +def _nearest_non_prerelease( + v: _VersionOrBoundary, +) -> Version | None: + """Smallest non-pre-release version at or above *v*, or None.""" + if v is None: + return None + if isinstance(v, _BoundaryVersion): + inner = v.version + if inner.is_prerelease: + # AFTER_LOCALS(1.0a1) -> nearest non-pre is 1.0 + return inner.__replace__(pre=None, dev=None, local=None) + # AFTER_LOCALS(1.0) -> nearest non-pre is 1.0.post0 + # AFTER_LOCALS(1.0.post0) -> nearest non-pre is 1.0.post1 + k = (inner.post + 1) if inner.post is not None else 0 + return inner.__replace__(post=k, local=None) + if not v.is_prerelease: + return v + # Strip pre/dev to get the final or post-release form. + return v.__replace__(pre=None, dev=None, local=None) class InvalidSpecifier(ValueError): @@ -105,10 +377,29 @@ class BaseSpecifier(metaclass=abc.ABCMeta): Determines if the given item is contained within this specifier. """ + @typing.overload + def filter( + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload + def filter( + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + @abc.abstractmethod def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """ Takes an iterable of items and filters them so that only items which are contained within this specifier are allowed in it. @@ -125,20 +416,23 @@ class Specifier(BaseSpecifier): comma-separated version specifiers (which is what package metadata contains). """ - __slots__ = ("_prereleases", "_spec", "_spec_version") + __slots__ = ( + "_prereleases", + "_ranges", + "_spec", + "_spec_version", + "_wildcard_split", + ) - _operator_regex_str = r""" - (?P<operator>(~=|==|!=|<=|>=|<|>|===)) - """ - _version_regex_str = r""" - (?P<version> + _specifier_regex_str = r""" + (?: (?: # The identity operators allow for an escape hatch that will # do an exact string match of the version you wish to install. # This will not be parsed by PEP 440 and we cannot determine # any semantic meaning from it. This operator is discouraged # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator + === # Only match for the identity operator \s* [^\s;)]* # The arbitrary version can be just about anything, # we match everything except for whitespace, a @@ -150,7 +444,7 @@ class Specifier(BaseSpecifier): # The (non)equality operators allow for wild card and local # versions to be specified so we have to define these two # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals + (?:==|!=) # Only match for equals and not equals \s* v? @@ -162,24 +456,24 @@ class Specifier(BaseSpecifier): (?: \.\* # Wild card syntax of .* | - (?: # pre release + (?a: # pre release [-_\.]? (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? - (?: # post release + (?a: # post release (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + (?a:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?a:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local )? ) | (?: # The compatible operator requires at least two digits in the # release segment. - (?<=~=) # Only match for the compatible operator + (?:~=) # Only match for the compatible operator \s* v? @@ -202,31 +496,28 @@ class Specifier(BaseSpecifier): # (non)equality operators do. Specifically they do not allow # local versions to be specified nor do they allow the prefix # matching wild cards. - (?<!==|!=|~=) # We have special cases for these - # operators so we want to make sure they - # don't match here. + (?:<=|>=|<|>) \s* v? (?:[0-9]+!)? # epoch [0-9]+(?:\.[0-9]+)* # release - (?: # pre release + (?a: # pre release [-_\.]? (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? - (?: # post release + (?a: # post release (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?a:[-_\.]?dev[-_\.]?[0-9]*)? # dev release ) ) """ _regex = re.compile( - r"\s*" + _operator_regex_str + _version_regex_str + r"\s*", - re.VERBOSE | re.IGNORECASE, + r"\s*" + _specifier_regex_str + r"\s*", re.VERBOSE | re.IGNORECASE ) _operators: Final = { @@ -253,14 +544,18 @@ class Specifier(BaseSpecifier): :raises InvalidSpecifier: If the given specifier is invalid (i.e. bad syntax). """ - match = self._regex.fullmatch(spec) - if not match: + if not self._regex.fullmatch(spec): raise InvalidSpecifier(f"Invalid specifier: {spec!r}") - self._spec: tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + spec = spec.strip() + if spec.startswith("==="): + operator, version = spec[:3], spec[3:].strip() + elif spec.startswith(("~=", "==", "!=", "<=", ">=")): + operator, version = spec[:2], spec[2:].strip() + else: + operator, version = spec[:1], spec[1:].strip() + + self._spec: tuple[str, str] = (operator, version) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -268,6 +563,12 @@ class Specifier(BaseSpecifier): # Specifier version cache self._spec_version: tuple[str, Version] | None = None + # Populated on first wildcard (==X.*) comparison + self._wildcard_split: tuple[list[str], int] | None = None + + # Version range cache (populated by _to_ranges) + self._ranges: Sequence[_VersionRange] | None = None + def _get_spec_version(self, version: str) -> Version | None: """One element cache, as only one spec Version is needed per Specifier.""" if self._spec_version is not None and self._spec_version[0] == version: @@ -290,6 +591,105 @@ class Specifier(BaseSpecifier): assert spec_version is not None return spec_version + def _to_ranges(self) -> Sequence[_VersionRange]: + """Convert this specifier to sorted, non-overlapping version ranges. + + Each standard operator maps to one or two ranges. ``===`` is + modeled as full range (actual check done separately). Cached. + """ + if self._ranges is not None: + return self._ranges + + op = self.operator + ver_str = self.version + + if op == "===": + self._ranges = _FULL_RANGE + return _FULL_RANGE + + if ver_str.endswith(".*"): + result = self._wildcard_ranges(op, ver_str) + else: + result = self._standard_ranges(op, ver_str) + + self._ranges = result + return result + + def _wildcard_ranges(self, op: str, ver_str: str) -> list[_VersionRange]: + # ==1.2.* -> [1.2.dev0, 1.3.dev0); !=1.2.* -> complement. + base = self._require_spec_version(ver_str[:-2]) + lower = _base_dev0(base) + upper = _next_prefix_dev0(base) + if op == "==": + return [(_LowerBound(lower, True), _UpperBound(upper, False))] + # != + return [ + (_NEG_INF, _UpperBound(lower, False)), + (_LowerBound(upper, True), _POS_INF), + ] + + def _standard_ranges(self, op: str, ver_str: str) -> list[_VersionRange]: + v = self._require_spec_version(ver_str) + + if op == ">=": + return [(_LowerBound(v, True), _POS_INF)] + + if op == "<=": + return [ + ( + _NEG_INF, + _UpperBound(_BoundaryVersion(v, _BoundaryKind.AFTER_LOCALS), True), + ) + ] + + if op == ">": + if v.dev is not None: + # >V.devN: dev versions have no post-releases, so the + # next real version is V.dev(N+1). + lower_ver = v.__replace__(dev=v.dev + 1, local=None) + return [(_LowerBound(lower_ver, True), _POS_INF)] + if v.post is not None: + # >V.postN: next real version is V.post(N+1).dev0. + lower_ver = v.__replace__(post=v.post + 1, dev=0, local=None) + return [(_LowerBound(lower_ver, True), _POS_INF)] + # >V (final or pre-release): skip V+local and all V.postN. + return [ + ( + _LowerBound(_BoundaryVersion(v, _BoundaryKind.AFTER_POSTS), False), + _POS_INF, + ) + ] + + if op == "<": + # <V excludes prereleases of V when V is not a prerelease. + # V.dev0 is the earliest prerelease of V (final, post, etc.). + bound = v if v.is_prerelease else v.__replace__(dev=0, local=None) + if bound <= _MIN_VERSION: + return [] + return [(_NEG_INF, _UpperBound(bound, False))] + + # ==, !=: local versions of V match when spec has no local segment. + has_local = "+" in ver_str + after_locals = _BoundaryVersion(v, _BoundaryKind.AFTER_LOCALS) + upper = v if has_local else after_locals + + if op == "==": + return [(_LowerBound(v, True), _UpperBound(upper, True))] + + if op == "!=": + return [ + (_NEG_INF, _UpperBound(v, False)), + (_LowerBound(upper, False), _POS_INF), + ] + + if op == "~=": + prefix = v.__replace__(release=v.release[:-1]) + return [ + (_LowerBound(v, True), _UpperBound(_next_prefix_dev0(prefix), False)) + ] + + raise ValueError(f"Unknown operator: {op!r}") # pragma: no cover + @property def prereleases(self) -> bool | None: # If there is an explicit prereleases set for this, then we'll just @@ -300,24 +700,23 @@ class Specifier(BaseSpecifier): # Only the "!=" operator does not imply prereleases when # the version in the specifier is a prerelease. operator, version_str = self._spec - if operator != "!=": - # The == specifier with trailing .* cannot include prereleases - # e.g. "==1.0a1.*" is not valid. - if operator == "==" and version_str.endswith(".*"): - return False + if operator == "!=": + return False - # "===" can have arbitrary string versions, so we cannot parse - # those, we take prereleases as unknown (None) for those. - version = self._get_spec_version(version_str) - if version is None: - return None + # The == specifier with trailing .* cannot include prereleases + # e.g. "==1.0a1.*" is not valid. + if operator == "==" and version_str.endswith(".*"): + return False - # For all other operators, use the check if spec Version - # object implies pre-releases. - if version.is_prerelease: - return True + # "===" can have arbitrary string versions, so we cannot parse + # those, we take prereleases as unknown (None) for those. + version = self._get_spec_version(version_str) + if version is None: + return None - return False + # For all other operators, use the check if spec Version + # object implies pre-releases. + return version.is_prerelease @prereleases.setter def prereleases(self, value: bool | None) -> None: @@ -437,23 +836,35 @@ class Specifier(BaseSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( - prospective, prefix + return (self._compare_greater_than_equal(prospective, spec)) and ( + self._compare_equal(prospective, prefix) ) + def _get_wildcard_split(self, spec: str) -> tuple[list[str], int]: + """Cached split of a wildcard spec into components and numeric length. + + >>> Specifier("==1.*")._get_wildcard_split("1.*") + (['0', '1'], 2) + >>> Specifier("==3.10.*")._get_wildcard_split("3.10.*") + (['0', '3', '10'], 3) + """ + wildcard_split = self._wildcard_split + if wildcard_split is None: + normalized = canonicalize_version(spec[:-2], strip_trailing_zero=False) + split_spec = _version_split(normalized) + wildcard_split = (split_spec, _numeric_prefix_len(split_spec)) + self._wildcard_split = wildcard_split + return wildcard_split + def _compare_equal(self, prospective: Version, spec: str) -> bool: # We need special logic to handle prefix matching if spec.endswith(".*"): + split_spec, spec_numeric_len = self._get_wildcard_split(spec) + # In the case of prefix matching we want to ignore local segment. normalized_prospective = canonicalize_version( _public_version(prospective), strip_trailing_zero=False ) - # Get the normalized version string ignoring the trailing .* - normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) - # Split the spec out by bangs and dots, and pretend that there is - # an implicit dot in between a release segment and a pre-release segment. - split_spec = _version_split(normalized_spec) - # Split the prospective version out by bangs and dots, and pretend # that there is an implicit dot in between a release segment and # a pre-release segment. @@ -461,7 +872,7 @@ class Specifier(BaseSpecifier): # 0-pad the prospective version before shortening it to get the correct # shortened version. - padded_prospective, _ = _pad_version(split_prospective, split_spec) + padded_prospective = _left_pad(split_prospective, spec_numeric_len) # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the @@ -507,14 +918,12 @@ class Specifier(BaseSpecifier): if not prospective < spec: return False - # This special case is here so that, unless the specifier itself - # includes is a pre-release version, that we do not accept pre-release - # versions for the version mentioned in the specifier (e.g. <3.1 should - # not match 3.1.dev0, but should match 3.0.dev0). + # The spec says: "<V MUST NOT allow a pre-release of the specified + # version unless the specified version is itself a pre-release." if ( not spec.is_prerelease and prospective.is_prerelease - and _base_version(prospective) == _base_version(spec) + and prospective >= _earliest_prerelease(spec) ): return False @@ -534,22 +943,20 @@ class Specifier(BaseSpecifier): if not prospective > spec: return False - # This special case is here so that, unless the specifier itself - # includes is a post-release version, that we do not accept - # post-release versions for the version mentioned in the specifier - # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + # The spec says: ">V MUST NOT allow a post-release of the specified + # version unless the specified version is itself a post-release." if ( not spec.is_postrelease and prospective.is_postrelease - and _base_version(prospective) == _base_version(spec) + and _post_base(prospective) == spec ): return False - # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is technically greater than, to match. - if prospective.local is not None and _base_version( - prospective - ) == _base_version(spec): + # Per the spec: ">V MUST NOT match a local version of the specified + # version". A "local version of V" is any version whose public part + # equals V. So >1.0a1 must not match 1.0a1+local, but must still + # match 1.0a2+local. + if prospective.local is not None and _public_version(prospective) == spec: return False # If we've gotten to here, it means that prospective version is both @@ -608,9 +1015,28 @@ class Specifier(BaseSpecifier): return bool(list(self.filter([item], prereleases=prereleases))) + @typing.overload + def filter( + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload + def filter( + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """Filter items in the given iterable, that match the specifier. :param iterable: @@ -620,6 +1046,10 @@ class Specifier(BaseSpecifier): Whether or not to allow prereleases in the returned iterator. If set to ``None`` (the default), it will follow the recommendation from :pep:`440` and match prereleases if there are no other versions. + :param key: + A callable that takes a single argument (an item from the iterable) and + returns a version string or :class:`Version` instance to be used for + filtering. >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) ['1.3'] @@ -631,6 +1061,10 @@ class Specifier(BaseSpecifier): ['1.3', '1.5a1'] >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) ['1.3', '1.5a1'] + >>> list(Specifier(">=1.2.3").filter( + ... [{"ver": "1.2"}, {"ver": "1.3"}], + ... key=lambda x: x["ver"])) + [{'ver': '1.3'}] """ prereleases_versions = [] found_non_prereleases = False @@ -645,14 +1079,22 @@ class Specifier(BaseSpecifier): # Filter versions for version in iterable: - parsed_version = _coerce_version(version) + parsed_version = _coerce_version(version if key is None else key(version)) + match = False if parsed_version is None: # === operator can match arbitrary (non-version) strings if self.operator == "===" and self._compare_arbitrary( version, self.version ): yield version - elif operator_callable(parsed_version, self.version): + elif self.operator == "===": + match = self._compare_arbitrary( + version if key is None else key(version), self.version + ) + else: + match = operator_callable(parsed_version, self.version) + + if match and parsed_version is not None: # If it's not a prerelease or prereleases are allowed, yield it directly if not parsed_version.is_prerelease or include_prereleases: found_non_prereleases = True @@ -674,6 +1116,48 @@ class Specifier(BaseSpecifier): _prefix_regex = re.compile(r"([0-9]+)((?:a|b|c|rc)[0-9]+)") +def _pep440_filter_prereleases( + iterable: Iterable[Any], key: Callable[[Any], UnparsedVersion] | None +) -> Iterator[Any]: + """Filter per PEP 440: exclude prereleases unless no finals exist.""" + # Two lists used: + # * all_nonfinal to preserve order if no finals exist + # * arbitrary_strings for streaming when first final found + all_nonfinal: list[Any] = [] + arbitrary_strings: list[Any] = [] + + found_final = False + for item in iterable: + parsed = _coerce_version(item if key is None else key(item)) + + if parsed is None: + # Arbitrary strings are always included as it is not + # possible to determine if they are prereleases, + # and they have already passed all specifiers. + if found_final: + yield item + else: + arbitrary_strings.append(item) + all_nonfinal.append(item) + continue + + if not parsed.is_prerelease: + # Final release found - flush arbitrary strings, then yield + if not found_final: + yield from arbitrary_strings + found_final = True + yield item + continue + + # Prerelease - buffer if no finals yet, otherwise skip + if not found_final: + all_nonfinal.append(item) + + # No finals found - yield all buffered items + if not found_final: + yield from all_nonfinal + + def _version_split(version: str) -> list[str]: """Split version into components. @@ -713,25 +1197,59 @@ def _is_not_suffix(segment: str) -> bool: ) -def _pad_version(left: list[str], right: list[str]) -> tuple[list[str], list[str]]: - left_split, right_split = [], [] +def _numeric_prefix_len(split: list[str]) -> int: + """Count leading numeric components in a :func:`_version_split` result. + + >>> _numeric_prefix_len(["0", "1", "2", "a1"]) + 3 + """ + count = 0 + for segment in split: + if not segment.isdigit(): + break + count += 1 + return count + - # Get the release segment of our versions - left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) - right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) +def _left_pad(split: list[str], target_numeric_len: int) -> list[str]: + """Pad a :func:`_version_split` result with ``"0"`` segments to reach + ``target_numeric_len`` numeric components. Suffix segments are preserved. - # Get the rest of our versions - left_split.append(left[len(left_split[0]) :]) - right_split.append(right[len(right_split[0]) :]) + >>> _left_pad(["0", "1", "a1"], 4) + ['0', '1', '0', '0', 'a1'] + """ + numeric_len = _numeric_prefix_len(split) + pad_needed = target_numeric_len - numeric_len + if pad_needed <= 0: + return split + return [*split[:numeric_len], *(["0"] * pad_needed), *split[numeric_len:]] - # Insert our padding - left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) - right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain.from_iterable(left_split)), - list(itertools.chain.from_iterable(right_split)), - ) +def _operator_cost(op_entry: tuple[CallableOperator, str, str]) -> int: + """Sort key for Cost Based Ordering of specifier operators in _filter_versions. + + Operators run sequentially on a shrinking candidate set, so operators that + reject the most versions should run first to minimize work for later ones. + + Tier 0: Exact equality (==, ===), likely to narrow candidates to one version + Tier 1: Range checks (>=, <=, >, <), cheap and usually reject a large portion + Tier 2: Wildcard equality (==.*) and compatible release (~=), more expensive + Tier 3: Exact !=, cheap but rarely rejects + Tier 4: Wildcard !=.*, expensive and rarely rejects + """ + _, ver, op = op_entry + if op == "==": + return 0 if not ver.endswith(".*") else 2 + if op in (">=", "<=", ">", "<"): + return 1 + if op == "~=": + return 2 + if op == "!=": + return 3 if not ver.endswith(".*") else 4 + if op == "===": + return 0 + + raise ValueError(f"Unknown operator: {op!r}") # pragma: no cover class SpecifierSet(BaseSpecifier): @@ -741,7 +1259,14 @@ class SpecifierSet(BaseSpecifier): specifiers (``>=3.0,!=3.1``), or no specifier at all. """ - __slots__ = ("_prereleases", "_specs") + __slots__ = ( + "_canonicalized", + "_has_arbitrary", + "_is_unsatisfiable", + "_prereleases", + "_resolved_ops", + "_specs", + ) def __init__( self, @@ -770,17 +1295,33 @@ class SpecifierSet(BaseSpecifier): # strip each item to remove leading/trailing whitespace. split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] - # Make each individual specifier a Specifier and save in a frozen set - # for later. - self._specs = frozenset(map(Specifier, split_specifiers)) + self._specs: tuple[Specifier, ...] = tuple(map(Specifier, split_specifiers)) + # Fast substring check; avoids iterating parsed specs. + self._has_arbitrary = "===" in specifiers else: - # Save the supplied specifiers in a frozen set. - self._specs = frozenset(specifiers) + self._specs = tuple(specifiers) + # Substring check works for both Specifier objects and plain + # strings (setuptools passes lists of strings). + self._has_arbitrary = any("===" in str(s) for s in self._specs) + + self._canonicalized = len(self._specs) <= 1 + self._resolved_ops: list[tuple[CallableOperator, str, str]] | None = None # Store our prereleases value so we can use it later to determine if # we accept prereleases or not. self._prereleases = prereleases + self._is_unsatisfiable: bool | None = None + + def _canonical_specs(self) -> tuple[Specifier, ...]: + """Deduplicate, sort, and cache specs for order-sensitive operations.""" + if not self._canonicalized: + self._specs = tuple(dict.fromkeys(sorted(self._specs, key=str))) + self._canonicalized = True + self._resolved_ops = None + self._is_unsatisfiable = None + return self._specs + @property def prereleases(self) -> bool | None: # If we have been given an explicit prerelease modifier, then we'll @@ -804,6 +1345,7 @@ class SpecifierSet(BaseSpecifier): @prereleases.setter def prereleases(self, value: bool | None) -> None: self._prereleases = value + self._is_unsatisfiable = None def __repr__(self) -> str: """A representation of the specifier set that shows all internal state. @@ -824,7 +1366,7 @@ class SpecifierSet(BaseSpecifier): else "" ) - return f"<SpecifierSet({str(self)!r}{pre})>" + return f"<{self.__class__.__name__}({str(self)!r}{pre})>" def __str__(self) -> str: """A string representation of the specifier set that can be round-tripped. @@ -837,10 +1379,10 @@ class SpecifierSet(BaseSpecifier): >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) '!=1.0.1,>=1.0.0' """ - return ",".join(sorted(str(s) for s in self._specs)) + return ",".join(str(s) for s in self._canonical_specs()) def __hash__(self) -> int: - return hash(self._specs) + return hash(self._canonical_specs()) def __and__(self, other: SpecifierSet | str) -> SpecifierSet: """Return a SpecifierSet which is a combination of the two sets. @@ -858,13 +1400,15 @@ class SpecifierSet(BaseSpecifier): return NotImplemented specifier = SpecifierSet() - specifier._specs = frozenset(self._specs | other._specs) + specifier._specs = self._specs + other._specs + specifier._canonicalized = len(specifier._specs) <= 1 + specifier._has_arbitrary = self._has_arbitrary or other._has_arbitrary + specifier._resolved_ops = None - if self._prereleases is None and other._prereleases is not None: + # Combine prerelease settings: use common or non-None value + if self._prereleases is None or self._prereleases == other._prereleases: specifier._prereleases = other._prereleases - elif ( - self._prereleases is not None and other._prereleases is None - ) or self._prereleases == other._prereleases: + elif other._prereleases is None: specifier._prereleases = self._prereleases else: raise ValueError( @@ -897,7 +1441,7 @@ class SpecifierSet(BaseSpecifier): elif not isinstance(other, SpecifierSet): return NotImplemented - return self._specs == other._specs + return self._canonical_specs() == other._canonical_specs() def __len__(self) -> int: """Returns the number of specifiers in this specifier set.""" @@ -913,6 +1457,113 @@ class SpecifierSet(BaseSpecifier): """ return iter(self._specs) + def _get_ranges(self) -> Sequence[_VersionRange]: + """Intersect all specifiers into a single list of version ranges. + + Returns an empty list when unsatisfiable. ``===`` specs are + modeled as full range; string matching is checked separately + by :meth:`_check_arbitrary_unsatisfiable`. + """ + specs = self._specs + + result: Sequence[_VersionRange] | None = None + for s in specs: + if result is None: + result = s._to_ranges() + else: + result = _intersect_ranges(result, s._to_ranges()) + if not result: + break + + if result is None: # pragma: no cover + raise RuntimeError("_get_ranges called with no specs") + return result + + def is_unsatisfiable(self) -> bool: + """Check whether this specifier set can never be satisfied. + + Returns True if no version can satisfy all specifiers simultaneously. + + >>> SpecifierSet(">=2.0,<1.0").is_unsatisfiable() + True + >>> SpecifierSet(">=1.0,<2.0").is_unsatisfiable() + False + >>> SpecifierSet("").is_unsatisfiable() + False + >>> SpecifierSet("==1.0,!=1.0").is_unsatisfiable() + True + """ + cached = self._is_unsatisfiable + if cached is not None: + return cached + + if not self._specs: + self._is_unsatisfiable = False + return False + + result = not self._get_ranges() + + if not result: + result = self._check_arbitrary_unsatisfiable() + + if not result and self.prereleases is False: + result = self._check_prerelease_only_ranges() + + self._is_unsatisfiable = result + return result + + def _check_prerelease_only_ranges(self) -> bool: + """With prereleases=False, check if every range contains only + pre-release versions (which would be excluded from matching).""" + for lower, upper in self._get_ranges(): + nearest = _nearest_non_prerelease(lower.version) + if nearest is None: + return False + if upper.version is None or nearest < upper.version: + return False + if nearest == upper.version and upper.inclusive: + return False + return True + + def _check_arbitrary_unsatisfiable(self) -> bool: + """Check === (arbitrary equality) specs for unsatisfiability. + + === uses case-insensitive string comparison, so the only candidate + that can match ``===V`` is the literal string V. This method + checks whether that candidate is excluded by other specifiers. + """ + arbitrary = [s for s in self._specs if s.operator == "==="] + if not arbitrary: + return False + + # Multiple === must agree on the same string (case-insensitive). + first = arbitrary[0].version.lower() + if any(s.version.lower() != first for s in arbitrary[1:]): + return True + + # The sole candidate is the === version string. Check whether + # it can satisfy every standard spec. + candidate = _coerce_version(arbitrary[0].version) + + # With prereleases=False, a prerelease candidate is excluded + # by contains() before the === string check even runs. + if ( + self.prereleases is False + and candidate is not None + and candidate.is_prerelease + ): + return True + + standard = [s for s in self._specs if s.operator != "==="] + if not standard: + return False + + if candidate is None: + # Unparsable string cannot satisfy any standard spec. + return True + + return not all(s.contains(candidate) for s in standard) + def __contains__(self, item: UnparsedVersion) -> bool: """Return whether or not the item is contained in this specifier. @@ -971,12 +1622,36 @@ class SpecifierSet(BaseSpecifier): if version is not None and installed and version.is_prerelease: prereleases = True - check_item = item if version is None else version + # When item is a string and === is involved, keep it as-is + # so the comparison isn't done against the normalized form. + if version is None or (self._has_arbitrary and not isinstance(item, Version)): + check_item = item + else: + check_item = version return bool(list(self.filter([check_item], prereleases=prereleases))) + @typing.overload + def filter( + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload + def filter( + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """Filter items in the given iterable, that match the specifiers in this set. :param iterable: @@ -986,6 +1661,10 @@ class SpecifierSet(BaseSpecifier): Whether or not to allow prereleases in the returned iterator. If set to ``None`` (the default), it will follow the recommendation from :pep:`440` and match prereleases if there are no other versions. + :param key: + A callable that takes a single argument (an item from the iterable) and + returns a version string or :class:`Version` instance to be used for + filtering. >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) ['1.3'] @@ -997,6 +1676,10 @@ class SpecifierSet(BaseSpecifier): ['1.3', '1.5a1'] >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) ['1.3', '1.5a1'] + >>> list(SpecifierSet(">=1.2.3").filter( + ... [{"ver": "1.2"}, {"ver": "1.3"}], + ... key=lambda x: x["ver"])) + [{'ver': '1.3'}] An "empty" SpecifierSet will filter items based on the presence of prerelease versions in the set. @@ -1016,53 +1699,91 @@ class SpecifierSet(BaseSpecifier): if prereleases is None and self.prereleases is not None: prereleases = self.prereleases - # If we have any specifiers, then we want to wrap our iterable in the - # filter method for each one, this will act as a logical AND amongst - # each specifier. + # Filter versions that match all specifiers using Cost Based Ordering. if self._specs: # When prereleases is None, we need to let all versions through # the individual filters, then decide about prereleases at the end # based on whether any non-prereleases matched ALL specs. - for spec in self._specs: - iterable = spec.filter( - iterable, prereleases=True if prereleases is None else prereleases + + # Fast path: single specifier, delegate directly. + if len(self._specs) == 1: + filtered = self._specs[0].filter( + iterable, + prereleases=True if prereleases is None else prereleases, + key=key, + ) + else: + filtered = self._filter_versions( + iterable, + key, + prereleases=True if prereleases is None else prereleases, ) if prereleases is not None: - # If we have a forced prereleases value, - # we can immediately return the iterator. - return iter(iterable) - else: - # Handle empty SpecifierSet cases where prereleases is not None. - if prereleases is True: - return iter(iterable) + return filtered - if prereleases is False: - return ( - item - for item in iterable - if (version := _coerce_version(item)) is None + return _pep440_filter_prereleases(filtered, key) + + # Handle Empty SpecifierSet. + if prereleases is True: + return iter(iterable) + + if prereleases is False: + return ( + item + for item in iterable + if ( + (version := _coerce_version(item if key is None else key(item))) + is None or not version.is_prerelease ) + ) + + # PEP 440: exclude prereleases unless no final releases matched + return _pep440_filter_prereleases(iterable, key) - # Finally if prereleases is None, apply PEP 440 logic: - # exclude prereleases unless there are no final releases that matched. - filtered_items: list[UnparsedVersionVar] = [] - found_prereleases: list[UnparsedVersionVar] = [] - found_final_release = False + def _filter_versions( + self, + iterable: Iterable[Any], + key: Callable[[Any], UnparsedVersion] | None, + prereleases: bool | None = None, + ) -> Iterator[Any]: + """Filter versions against all specifiers in a single pass. + + Uses Cost Based Ordering: specifiers are sorted by _operator_cost so + that cheap range operators reject versions early, avoiding expensive + wildcard or compatible operators on versions that would have been + rejected anyway. + """ + # Pre-resolve operators and sort (cached after first call). + if self._resolved_ops is None: + self._resolved_ops = sorted( + ( + (spec._get_operator(spec.operator), spec.version, spec.operator) + for spec in self._specs + ), + key=_operator_cost, + ) + ops = self._resolved_ops + exclude_prereleases = prereleases is False for item in iterable: - parsed_version = _coerce_version(item) - # Arbitrary strings are always included as it is not - # possible to determine if they are prereleases, - # and they have already passed all specifiers. - if parsed_version is None: - filtered_items.append(item) - found_prereleases.append(item) - elif parsed_version.is_prerelease: - found_prereleases.append(item) - else: - filtered_items.append(item) - found_final_release = True + parsed = _coerce_version(item if key is None else key(item)) - return iter(filtered_items if found_final_release else found_prereleases) + if parsed is None: + # Only === can match non-parseable versions. + if all( + op == "===" and str(item).lower() == ver.lower() + for _, ver, op in ops + ): + yield item + elif exclude_prereleases and parsed.is_prerelease: + pass + elif all( + str(item if key is None else key(item)).lower() == ver.lower() + if op == "===" + else op_fn(parsed, ver) + for op_fn, ver, op in ops + ): + # Short-circuits on the first failing operator. + yield item diff --git a/contrib/python/packaging/py3/packaging/tags.py b/contrib/python/packaging/py3/packaging/tags.py index 5ef27c897a4..7f71e3910e5 100644 --- a/contrib/python/packaging/py3/packaging/tags.py +++ b/contrib/python/packaging/py3/packaging/tags.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging +import operator import platform import re import struct @@ -13,20 +14,53 @@ import sys import sysconfig from importlib.machinery import EXTENSION_SUFFIXES from typing import ( + TYPE_CHECKING, Any, Iterable, Iterator, Sequence, Tuple, + TypeVar, cast, ) from . import _manylinux, _musllinux +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from typing import AbstractSet + + +__all__ = [ + "INTERPRETER_SHORT_NAMES", + "AppleVersion", + "PythonVersion", + "Tag", + "UnsortedTagsError", + "android_platforms", + "compatible_tags", + "cpython_tags", + "create_compatible_tags_selector", + "generic_tags", + "interpreter_name", + "interpreter_version", + "ios_platforms", + "mac_platforms", + "parse_tag", + "platform_tags", + "sys_tags", +] + + +def __dir__() -> list[str]: + return __all__ + + logger = logging.getLogger(__name__) PythonVersion = Sequence[int] AppleVersion = Tuple[int, int] +_T = TypeVar("_T") INTERPRETER_SHORT_NAMES: dict[str, str] = { "python": "py", # Generic. @@ -37,7 +71,19 @@ INTERPRETER_SHORT_NAMES: dict[str, str] = { } -_32_BIT_INTERPRETER = struct.calcsize("P") == 4 +# This function can be unit tested without reloading the module +# (Unlike _32_BIT_INTERPRETER) +def _compute_32_bit_interpreter() -> bool: + return struct.calcsize("P") == 4 + + +_32_BIT_INTERPRETER = _compute_32_bit_interpreter() + + +class UnsortedTagsError(ValueError): + """ + Raised when a tag component is not in sorted order per PEP 425. + """ class Tag: @@ -51,6 +97,14 @@ class Tag: __slots__ = ["_abi", "_hash", "_interpreter", "_platform"] def __init__(self, interpreter: str, abi: str, platform: str) -> None: + """ + :param str interpreter: The interpreter name, e.g. ``"py"`` + (see :attr:`INTERPRETER_SHORT_NAMES` for mapping + well-known interpreter names to their short names). + :param str abi: The ABI that a wheel supports, e.g. ``"cp37m"``. + :param str platform: The OS/platform the wheel supports, + e.g. ``"win_amd64"``. + """ self._interpreter = interpreter.lower() self._abi = abi.lower() self._platform = platform.lower() @@ -63,14 +117,25 @@ class Tag: @property def interpreter(self) -> str: + """ + The interpreter name, e.g. ``"py"`` (see + :attr:`INTERPRETER_SHORT_NAMES` for mapping well-known interpreter + names to their short names). + """ return self._interpreter @property def abi(self) -> str: + """ + The supported ABI. + """ return self._abi @property def platform(self) -> str: + """ + The OS/platform. + """ return self._platform def __eq__(self, other: object) -> bool: @@ -101,15 +166,36 @@ class Tag: self._hash = hash((self._interpreter, self._abi, self._platform)) -def parse_tag(tag: str) -> frozenset[Tag]: +def parse_tag(tag: str, *, validate_order: bool = False) -> frozenset[Tag]: """ - Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. + Parses the provided tag (e.g. `py3-none-any`) into a frozenset of + :class:`Tag` instances. Returning a set is required due to the possibility that the tag is a - compressed tag set. + `compressed tag set`_, e.g. ``"py2.py3-none-any"`` which supports both + Python 2 and Python 3. + + If **validate_order** is true, compressed tag set components are checked + to be in sorted order as required by PEP 425. + + :param str tag: The tag to parse, e.g. ``"py3-none-any"``. + :param bool validate_order: Check whether compressed tag set components + are in sorted order. + :raises UnsortedTagsError: If **validate_order** is true and any compressed tag + set component is not in sorted order. + + .. versionadded:: 26.1 + The *validate_order* parameter. """ tags = set() interpreters, abis, platforms = tag.split("-") + if validate_order: + for component in (interpreters, abis, platforms): + parts = component.split(".") + if parts != sorted(parts): + raise UnsortedTagsError( + f"Tag component {component!r} is not in sorted order per PEP 425" + ) for interpreter in interpreters.split("."): for abi in abis.split("."): for platform_ in platforms.split("."): @@ -150,12 +236,25 @@ def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: """ Determine if the Python version supports abi3. - PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) + PEP 384 was first implemented in Python 3.2. The free-threaded builds do not support abi3. """ return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading +def _abi3t_applies(python_version: PythonVersion, threading: bool) -> bool: + """ + Determine if the Python version supports abi3t. + + PEP 803 was first implemented in Python 3.15 but, per PEP 803, this + returns tags going back to Python 3.2 to mirror the abi3 + implementation and leave open the possibility of abi3t wheels + supporting older Python versions. + + """ + return len(python_version) > 1 and tuple(python_version) >= (3, 2) and threading + + def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> list[str]: py_version = tuple(py_version) # To allow for version comparison. abis = [] @@ -197,19 +296,31 @@ def cpython_tags( warn: bool = False, ) -> Iterator[Tag]: """ - Yields the tags for a CPython interpreter. + Yields the tags for the CPython interpreter. + + The specific tags generated are: - The tags consist of: - - cp<python_version>-<abi>-<platform> - - cp<python_version>-abi3-<platform> - - cp<python_version>-none-<platform> - - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. + - ``cp<python_version>-<abi>-<platform>`` + - ``cp<python_version>-<stable_abi>-<platform>`` + - ``cp<python_version>-none-<platform>`` + - ``cp<older version>-<stable_abi>-<platform>`` where "older version" is all older + minor versions down to Python 3.2 (when ``abi3`` was introduced) - If python_version only specifies a major version then user-provided ABIs and - the 'none' ABItag will be used. + If ``python_version`` only provides a major-only version then only + user-provided ABIs via ``abis`` and the ``none`` ABI will be used. - If 'abi3' or 'none' are specified in 'abis' then they will be yielded at - their normal position and not at the beginning. + The ``stable_abi`` will be either ``abi3`` or ``abi3t`` if `abi` is a + GIL-enabled ABI like `"cp315"` or a free-threaded ABI like `"cp315t"`, + respectively. + + :param Sequence python_version: A one- or two-item sequence representing the + targeted Python version. Defaults to + ``sys.version_info[:2]``. + :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs + compatible with the current system. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ if not python_version: python_version = sys.version_info[:2] @@ -233,16 +344,27 @@ def cpython_tags( threading = _is_threaded_cpython(abis) use_abi3 = _abi3_applies(python_version, threading) + use_abi3t = _abi3t_applies(python_version, threading) + if use_abi3: yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) + if use_abi3t: + yield from (Tag(interpreter, "abi3t", platform_) for platform_ in platforms) + yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) - if use_abi3: + if use_abi3 or use_abi3t: for minor_version in range(python_version[1] - 1, 1, -1): for platform_ in platforms: version = _version_nodot((python_version[0], minor_version)) interpreter = f"cp{version}" - yield Tag(interpreter, "abi3", platform_) + if use_abi3: + yield Tag(interpreter, "abi3", platform_) + if use_abi3t: + # Support for abi3t was introduced in Python 3.15, but in + # principle abi3t wheels are possible for older limited API + # versions, so allow things like ("cp37", "abi3t", "platform") + yield Tag(interpreter, "abi3t", platform_) def _generic_abi() -> list[str]: @@ -294,12 +416,25 @@ def generic_tags( warn: bool = False, ) -> Iterator[Tag]: """ - Yields the tags for a generic interpreter. + Yields the tags for an interpreter which requires no specialization. + + This function should be used if one of the other interpreter-specific + functions provided by this module is not appropriate (i.e. not calculating + tags for a CPython interpreter). + + The specific tags generated are: + + - ``<interpreter>-<abi>-<platform>`` - The tags consist of: - - <interpreter>-<abi>-<platform> + The ``"none"`` ABI will be added if it was not explicitly provided. - The "none" ABI will be added if it was not explicitly provided. + :param str interpreter: The name of the interpreter. Defaults to being + calculated. + :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs + compatible with the current system. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ if not interpreter: interp_name = interpreter_name() @@ -335,12 +470,22 @@ def compatible_tags( platforms: Iterable[str] | None = None, ) -> Iterator[Tag]: """ - Yields the sequence of tags that are compatible with a specific version of Python. + Yields the tags for an interpreter compatible with the Python version + specified by ``python_version``. - The tags consist of: - - py*-none-<platform> - - <interpreter>-none-any # ... if `interpreter` is provided. - - py*-none-any + The specific tags generated are: + + - ``py*-none-<platform>`` + - ``<interpreter>-none-any`` if ``interpreter`` is provided + - ``py*-none-any`` + + :param Sequence python_version: A one- or two-item sequence representing the + compatible version of Python. Defaults to + ``sys.version_info[:2]``. + :param str interpreter: The name of the interpreter (if known), e.g. + ``"cp38"``. Defaults to the current interpreter. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. """ if not python_version: python_version = sys.version_info[:2] @@ -400,12 +545,25 @@ def mac_platforms( version: AppleVersion | None = None, arch: str | None = None ) -> Iterator[str]: """ - Yields the platform tags for a macOS system. + Yields the :attr:`~Tag.platform` tags for macOS. The `version` parameter is a two-item tuple specifying the macOS version to generate platform tags for. The `arch` parameter is the CPU architecture to generate platform tags for. Both parameters default to the appropriate value for the current system. + + :param tuple version: A two-item tuple representing the version of macOS. + Defaults to the current system's version. + :param str arch: The CPU architecture. Defaults to the architecture of the + current system, e.g. ``"x86_64"``. + + .. note:: + Equivalent support for the other major platforms is purposefully not + provided: + + - On Windows, platform compatibility is statically specified + - On Linux, code must be run on the system itself to determine + compatibility """ version_str, _, cpu_arch = platform.mac_ver() if version is None: @@ -476,14 +634,19 @@ def ios_platforms( version: AppleVersion | None = None, multiarch: str | None = None ) -> Iterator[str]: """ - Yields the platform tags for an iOS system. - :param version: A two-item tuple specifying the iOS version to generate - platform tags for. Defaults to the current iOS version. - :param multiarch: The CPU architecture+ABI to generate platform tags for - - (the value used by `sys.implementation._multiarch` e.g., - `arm64_iphoneos` or `x84_64_iphonesimulator`). Defaults to the current - multiarch value. + Yields the :attr:`~Tag.platform` tags for iOS. + + :param tuple version: A two-item tuple representing the version of iOS. + Defaults to the current system's version. + :param str multiarch: The CPU architecture+ABI to be used. This should be in + the format by ``sys.implementation._multiarch`` (e.g., + ``arm64_iphoneos`` or ``x86_64_iphonesimulator``). + Defaults to the current system's multiarch value. + + .. note:: + Behavior of this method is undefined if invoked on non-iOS platforms + without providing explicit version and multiarch arguments. """ if version is None: # if iOS is the current platform, ios_ver *must* be defined. However, @@ -585,13 +748,20 @@ def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: yield f"linux_{arch}" +def _emscripten_platforms() -> Iterator[str]: + pyemscripten_abi_version = sysconfig.get_config_var("PYEMSCRIPTEN_ABI_VERSION") + if pyemscripten_abi_version: + yield f"pyemscripten_{pyemscripten_abi_version}_wasm32" + yield from _generic_platforms() + + def _generic_platforms() -> Iterator[str]: yield _normalize_string(sysconfig.get_platform()) def platform_tags() -> Iterator[str]: """ - Provides the platform tags for this installation. + Yields the :attr:`~Tag.platform` tags for the running interpreter. """ if platform.system() == "Darwin": return mac_platforms() @@ -601,6 +771,8 @@ def platform_tags() -> Iterator[str]: return android_platforms() elif platform.system() == "Linux": return _linux_platforms() + elif platform.system() == "Emscripten": + return _emscripten_platforms() else: return _generic_platforms() @@ -611,6 +783,8 @@ def interpreter_name() -> str: Some implementations have a reserved, two-letter abbreviation which will be returned when appropriate. + + This typically acts as the prefix to the :attr:`~Tag.interpreter` tag. """ name = sys.implementation.name return INTERPRETER_SHORT_NAMES.get(name) or name @@ -618,7 +792,11 @@ def interpreter_name() -> str: def interpreter_version(*, warn: bool = False) -> str: """ - Returns the version of the running interpreter. + Returns the running interpreter's version. + + This typically acts as the suffix to the :attr:`~Tag.interpreter` tag. + + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ version = _get_config_var("py_version_nodot", warn=warn) return str(version) if version else _version_nodot(sys.version_info[:2]) @@ -630,10 +808,31 @@ def _version_nodot(version: PythonVersion) -> str: def sys_tags(*, warn: bool = False) -> Iterator[Tag]: """ - Returns the sequence of tag triples for the running interpreter. + Yields the sequence of tag triples that the running interpreter supports. + + The iterable is ordered so that the best-matching tag is first in the + sequence. The exact preferential order to tags is interpreter-specific, but + in general the tag importance is in the order of: + + 1. Interpreter + 2. Platform + 3. ABI - The order of the sequence corresponds to priority order for the - interpreter, from most to least important. + This order is due to the fact that an ABI is inherently tied to the + platform, but platform-specific code is not necessarily tied to the ABI. The + interpreter is the most important tag as it dictates basic support for any + wheel. + + The function returns an iterable in order to allow for the possible + short-circuiting of tag generation if the entire sequence is not necessary + and tag calculation happens to be expensive. + + :param bool warn: Whether warnings should be logged. Defaults to ``False``. + + .. versionchanged:: 21.3 + Added the `pp3-none-any` tag (:issue:`311`). + .. versionchanged:: 27.0 + Added the `abi3t` tag (:issue:`1099`). """ interp_name = interpreter_name() @@ -649,3 +848,49 @@ def sys_tags(*, warn: bool = False) -> Iterator[Tag]: else: interp = None yield from compatible_tags(interpreter=interp) + + +def create_compatible_tags_selector( + tags: Iterable[Tag], +) -> Callable[[Iterable[tuple[_T, AbstractSet[Tag]]]], Iterator[_T]]: + """Create a callable to select things compatible with supported tags. + + This function accepts an ordered sequence of tags, with the preferred + tags first. + + The returned callable accepts an iterable of tuples (thing, set[Tag]), + and returns an iterator of things, with the things with the best + matching tags first. + + Example to select compatible wheel filenames: + + >>> from packaging import tags + >>> from packaging.utils import parse_wheel_filename + >>> selector = tags.create_compatible_tags_selector(tags.sys_tags()) + >>> filenames = ["foo-1.0-py3-none-any.whl", "foo-1.0-py2-none-any.whl"] + >>> list(selector([ + ... (filename, parse_wheel_filename(filename)[-1]) for filename in filenames + ... ])) + ['foo-1.0-py3-none-any.whl'] + + .. versionadded:: 26.1 + """ + tag_ranks: dict[Tag, int] = {} + for rank, tag in enumerate(tags): + tag_ranks.setdefault(tag, rank) # ignore duplicate tags, keep first + supported_tags = tag_ranks.keys() + + def selector( + tagged_things: Iterable[tuple[_T, AbstractSet[Tag]]], + ) -> Iterator[_T]: + ranked_things: list[tuple[_T, int]] = [] + for thing, thing_tags in tagged_things: + supported_thing_tags = thing_tags & supported_tags + if supported_thing_tags: + thing_rank = min(tag_ranks[t] for t in supported_thing_tags) + ranked_things.append((thing, thing_rank)) + return iter( + thing for thing, _ in sorted(ranked_things, key=operator.itemgetter(1)) + ) + + return selector diff --git a/contrib/python/packaging/py3/packaging/utils.py b/contrib/python/packaging/py3/packaging/utils.py index c41c8137f26..cbd3be27c42 100644 --- a/contrib/python/packaging/py3/packaging/utils.py +++ b/contrib/python/packaging/py3/packaging/utils.py @@ -7,11 +7,33 @@ from __future__ import annotations import re from typing import NewType, Tuple, Union, cast -from .tags import Tag, parse_tag +from .tags import Tag, UnsortedTagsError, parse_tag from .version import InvalidVersion, Version, _TrimmedRelease +__all__ = [ + "BuildTag", + "InvalidName", + "InvalidSdistFilename", + "InvalidWheelFilename", + "NormalizedName", + "canonicalize_name", + "canonicalize_version", + "is_normalized_name", + "parse_sdist_filename", + "parse_wheel_filename", +] + + +def __dir__() -> list[str]: + return __all__ + + BuildTag = Union[Tuple[()], Tuple[int, str]] + NormalizedName = NewType("NormalizedName", str) +""" +A :class:`typing.NewType` of :class:`str`, representing a normalized name. +""" class InvalidName(ValueError): @@ -33,13 +55,39 @@ class InvalidSdistFilename(ValueError): # Core metadata spec for `Name` -_validate_regex = re.compile(r"[A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9]", re.IGNORECASE) -_normalized_regex = re.compile(r"[a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9]") +_validate_regex = re.compile( + r"[a-z0-9]|[a-z0-9][a-z0-9._-]*[a-z0-9]", re.IGNORECASE | re.ASCII +) +_normalized_regex = re.compile(r"[a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9]", re.ASCII) # PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") +_build_tag_regex = re.compile(r"(\d+)(.*)", re.ASCII) def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: + """ + This function takes a valid Python package or extra name, and returns the + normalized form of it. + + The return type is typed as :class:`NormalizedName`. This allows type + checkers to help require that a string has passed through this function + before use. + + If **validate** is true, then the function will check if **name** is a valid + distribution name before normalizing. + + :param str name: The name to normalize. + :param bool validate: Check whether the name is a valid distribution name. + :raises InvalidName: If **validate** is true and the name is not an + acceptable distribution name. + + >>> from packaging.utils import canonicalize_name + >>> canonicalize_name("Django") + 'django' + >>> canonicalize_name("oslo.concurrency") + 'oslo-concurrency' + >>> canonicalize_name("requests") + 'requests' + """ if validate and not _validate_regex.fullmatch(name): raise InvalidName(f"name is invalid: {name!r}") # Ensure all ``.`` and ``_`` are ``-`` @@ -53,15 +101,32 @@ def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: def is_normalized_name(name: str) -> bool: + """ + Check if a name is already normalized (i.e. :func:`canonicalize_name` would + roundtrip to the same value). + + :param str name: The name to check. + + >>> from packaging.utils import is_normalized_name + >>> is_normalized_name("requests") + True + >>> is_normalized_name("Django") + False + """ return _normalized_regex.fullmatch(name) is not None def canonicalize_version( version: Version | str, *, strip_trailing_zero: bool = True ) -> str: - """ - Return a canonical form of a version as a string. + """Return a canonical form of a version as a string. + This function takes a string representing a package version (or a + :class:`~packaging.version.Version` instance), and returns the + normalized form of it. By default, it strips trailing zeros from + the release segment. + + >>> from packaging.utils import canonicalize_version >>> canonicalize_version('1.0.1') '1.0.1' @@ -77,6 +142,9 @@ def canonicalize_version( >>> canonicalize_version('foo bar baz') 'foo bar baz' + + >>> canonicalize_version('1.4.0.0.0') + '1.4' """ if isinstance(version, str): try: @@ -88,7 +156,49 @@ def canonicalize_version( def parse_wheel_filename( filename: str, + *, + validate_order: bool = False, ) -> tuple[NormalizedName, Version, BuildTag, frozenset[Tag]]: + """ + This function takes the filename of a wheel file, and parses it, + returning a tuple of name, version, build number, and tags. + + The name part of the tuple is normalized and typed as + :class:`NormalizedName`. The version portion is an instance of + :class:`~packaging.version.Version`. The build number is ``()`` if + there is no build number in the wheel filename, otherwise a + two-item tuple of an integer for the leading digits and + a string for the rest of the build number. The tags portion is a + frozen set of :class:`~packaging.tags.Tag` instances (as the tag + string format allows multiple tags to be combined into a single + string). + + If **validate_order** is true, compressed tag set components are + checked to be in sorted order as required by PEP 425. + + :param str filename: The name of the wheel file. + :param bool validate_order: Check whether compressed tag set components + are in sorted order. + :raises InvalidWheelFilename: If the filename in question + does not follow the :ref:`wheel specification + <pypug:binary-distribution-format>`. + + >>> from packaging.utils import parse_wheel_filename + >>> from packaging.tags import Tag + >>> from packaging.version import Version + >>> name, ver, build, tags = parse_wheel_filename("foo-1.0-py3-none-any.whl") + >>> name + 'foo' + >>> ver == Version('1.0') + True + >>> tags == {Tag("py3", "none", "any")} + True + >>> not build + True + + .. versionadded:: 26.1 + The *validate_order* parameter. + """ if not filename.endswith(".whl"): raise InvalidWheelFilename( f"Invalid wheel filename (extension must be '.whl'): {filename!r}" @@ -125,11 +235,39 @@ def parse_wheel_filename( build = cast("BuildTag", (int(build_match.group(1)), build_match.group(2))) else: build = () - tags = parse_tag(parts[-1]) + tag_str = parts[-1] + try: + tags = parse_tag(tag_str, validate_order=validate_order) + except UnsortedTagsError: + raise InvalidWheelFilename( + f"Invalid wheel filename (compressed tag set components must be in " + f"sorted order per PEP 425): {filename!r}" + ) from None return (name, version, build, tags) def parse_sdist_filename(filename: str) -> tuple[NormalizedName, Version]: + """ + This function takes the filename of a sdist file (as specified + in the `Source distribution format`_ documentation), and parses + it, returning a tuple of the normalized name and version as + represented by an instance of :class:`~packaging.version.Version`. + + :param str filename: The name of the sdist file. + :raises InvalidSdistFilename: If the filename does not end + with an sdist extension (``.zip`` or ``.tar.gz``), or if it does not + contain a dash separating the name and the version of the distribution. + + >>> from packaging.utils import parse_sdist_filename + >>> from packaging.version import Version + >>> name, ver = parse_sdist_filename("foo-1.0.tar.gz") + >>> name + 'foo' + >>> ver == Version('1.0') + True + + .. _Source distribution format: https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-name + """ if filename.endswith(".tar.gz"): file_stem = filename[: -len(".tar.gz")] elif filename.endswith(".zip"): diff --git a/contrib/python/packaging/py3/packaging/version.py b/contrib/python/packaging/py3/packaging/version.py index 1206c462d4f..f872b85df90 100644 --- a/contrib/python/packaging/py3/packaging/version.py +++ b/contrib/python/packaging/py3/packaging/version.py @@ -4,7 +4,7 @@ """ .. testsetup:: - from packaging.version import parse, Version + from packaging.version import parse, normalize_pre, Version, _cmpkey """ from __future__ import annotations @@ -23,8 +23,6 @@ from typing import ( Union, ) -from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType - if typing.TYPE_CHECKING: from typing_extensions import Self, Unpack @@ -37,7 +35,7 @@ else: # pragma: no cover import warnings def _deprecated(message: str) -> object: - def decorator(func: object) -> object: + def decorator(func: Callable[[...], object]) -> object: @functools.wraps(func) def wrapper(*args: object, **kwargs: object) -> object: warnings.warn( @@ -62,22 +60,20 @@ _LETTER_NORMALIZATION = { "r": "post", } -__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "parse"] +__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "normalize_pre", "parse"] + + +def __dir__() -> list[str]: + return __all__ + LocalType = Tuple[Union[int, str], ...] -CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] -CmpLocalType = Union[ - NegativeInfinityType, - Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], -] -CmpKey = Tuple[ - int, - Tuple[int, ...], - CmpPrePostDevType, - CmpPrePostDevType, - CmpPrePostDevType, - CmpLocalType, +CmpLocalType = Tuple[Tuple[int, str], ...] +CmpSuffix = Tuple[int, int, int, int, int, int] +CmpKey = Union[ + Tuple[int, Tuple[int, ...], CmpSuffix], + Tuple[int, Tuple[int, ...], CmpSuffix, CmpLocalType], ] VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] @@ -85,15 +81,38 @@ VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] class _VersionReplace(TypedDict, total=False): epoch: int | None release: tuple[int, ...] | None - pre: tuple[Literal["a", "b", "rc"], int] | None + pre: tuple[str, int] | None post: int | None dev: int | None local: str | None +def normalize_pre(letter: str, /) -> str: + """Normalize the pre-release segment of a version string. + + Returns a lowercase version of the string if not a known pre-release + identifier. + + >>> normalize_pre('alpha') + 'a' + >>> normalize_pre('BETA') + 'b' + >>> normalize_pre('rc') + 'rc' + + :param letter: + + .. versionadded:: 26.1 + """ + letter = letter.lower() + return _LETTER_NORMALIZATION.get(letter, letter) + + def parse(version: str) -> Version: """Parse the given version string. + This is identical to the :class:`Version` constructor. + >>> parse('1.0.dev1') <Version('1.0.dev1')> @@ -173,7 +192,7 @@ class _BaseVersion: # Note that ++ doesn't behave identically on CPython and PyPy, so not using it here _VERSION_PATTERN = r""" v?+ # optional leading v - (?: + (?a: (?:(?P<epoch>[0-9]+)!)?+ # epoch (?P<release>[0-9]+(?:\.[0-9]+)*+) # release segment (?P<pre> # pre-release @@ -199,7 +218,7 @@ _VERSION_PATTERN = r""" (?P<dev_n>[0-9]+)? )?+ ) - (?:\+ + (?a:\+ (?P<local> # local version [a-z0-9]+ (?:[._-][a-z0-9]+)*+ @@ -227,12 +246,21 @@ expressions (for example, matching a version number as part of a file name). The regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` flags set. +.. versionchanged:: 26.0 + + The regex now uses possessive qualifiers on Python 3.11 if they are + supported (CPython 3.11.5+, PyPy 3.11.13+). + :meta hide-value: """ # Validation pattern for local version in replace() -_LOCAL_PATTERN = re.compile(r"[a-z0-9]+(?:[._-][a-z0-9]+)*", re.IGNORECASE) +_LOCAL_PATTERN = re.compile(r"[a-z0-9]+(?:[._-][a-z0-9]+)*", re.IGNORECASE | re.ASCII) + +# Fast path: If a version has only digits and dots then we +# can skip the regex and parse it as a release segment +_SIMPLE_VERSION_INDICATORS = frozenset(".0123456789") def _validate_epoch(value: object, /) -> int: @@ -258,14 +286,12 @@ def _validate_release(value: object, /) -> tuple[int, ...]: def _validate_pre(value: object, /) -> tuple[Literal["a", "b", "rc"], int] | None: if value is None: return value - if ( - isinstance(value, tuple) - and len(value) == 2 - and value[0] in ("a", "b", "rc") - and isinstance(value[1], int) - and value[1] >= 0 - ): - return value + if isinstance(value, tuple) and len(value) == 2: + letter, number = value + letter = normalize_pre(letter) + if letter in {"a", "b", "rc"} and isinstance(number, int) and number >= 0: + # type checkers can't infer the Literal type here on letter + return (letter, number) # type: ignore[return-value] msg = f"pre must be a tuple of ('a'|'b'|'rc', non-negative int), got {value}" raise InvalidVersion(msg) @@ -301,9 +327,9 @@ def _validate_local(value: object, /) -> LocalType | None: class _Version(NamedTuple): epoch: int release: tuple[int, ...] - dev: tuple[str, int] | None - pre: tuple[str, int] | None - post: tuple[str, int] | None + dev: tuple[Literal["dev"], int] | None + pre: tuple[Literal["a", "b", "rc"], int] | None + post: tuple[Literal["post"], int] | None local: LocalType | None @@ -329,20 +355,40 @@ class Version(_BaseVersion): False >>> v1 <= v2 True + + :class:`Version` is immutable; use :meth:`__replace__` to change + part of a version. """ - __slots__ = ("_dev", "_epoch", "_key_cache", "_local", "_post", "_pre", "_release") + __slots__ = ( + "_dev", + "_epoch", + "_hash_cache", + "_key_cache", + "_local", + "_post", + "_pre", + "_release", + ) __match_args__ = ("_str",) + """ + Pattern matching is supported on Python 3.10+. + + .. versionadded:: 26.0 + + :meta hide-value: + """ _regex = re.compile(r"\s*" + VERSION_PATTERN + r"\s*", re.VERBOSE | re.IGNORECASE) _epoch: int _release: tuple[int, ...] - _dev: tuple[str, int] | None - _pre: tuple[str, int] | None - _post: tuple[str, int] | None + _dev: tuple[Literal["dev"], int] | None + _pre: tuple[Literal["a", "b", "rc"], int] | None + _post: tuple[Literal["post"], int] | None _local: LocalType | None + _hash_cache: int | None _key_cache: CmpKey | None def __init__(self, version: str) -> None: @@ -355,23 +401,118 @@ class Version(_BaseVersion): If the ``version`` does not conform to PEP 440 in any way then this exception will be raised. """ + if _SIMPLE_VERSION_INDICATORS.issuperset(version): + try: + self._release = tuple(map(int, version.split("."))) + except ValueError: + # Empty parts (from "1..2", ".1", etc.) are invalid versions. + # Any other ValueError (e.g. int str-digits limit) should + # propagate to the caller. + if "" in version.split("."): + raise InvalidVersion(f"Invalid version: {version!r}") from None + # TODO: remove "no cover" when Python 3.9 is dropped. + raise # pragma: no cover + + self._epoch = 0 + self._pre = None + self._post = None + self._dev = None + self._local = None + self._key_cache = None + self._hash_cache = None + return + # Validate the version and parse it into pieces match = self._regex.fullmatch(version) if not match: raise InvalidVersion(f"Invalid version: {version!r}") self._epoch = int(match.group("epoch")) if match.group("epoch") else 0 self._release = tuple(map(int, match.group("release").split("."))) - self._pre = _parse_letter_version(match.group("pre_l"), match.group("pre_n")) - self._post = _parse_letter_version( + # We can type ignore the assignments below because the regex guarantees + # the correct strings + self._pre = _parse_letter_version(match.group("pre_l"), match.group("pre_n")) # type: ignore[assignment] + self._post = _parse_letter_version( # type: ignore[assignment] match.group("post_l"), match.group("post_n1") or match.group("post_n2") ) - self._dev = _parse_letter_version(match.group("dev_l"), match.group("dev_n")) + self._dev = _parse_letter_version(match.group("dev_l"), match.group("dev_n")) # type: ignore[assignment] self._local = _parse_local_version(match.group("local")) # Key which will be used for sorting self._key_cache = None + self._hash_cache = None + + @classmethod + def from_parts( + cls, + *, + epoch: int = 0, + release: tuple[int, ...], + pre: tuple[str, int] | None = None, + post: int | None = None, + dev: int | None = None, + local: str | None = None, + ) -> Self: + """ + Return a new version composed of the various parts. + + This allows you to build a version without going though a string and + running a regular expression. It normalizes pre-release strings. The + ``release=`` keyword argument is required. + + >>> Version.from_parts(release=(1,2,3)) + <Version('1.2.3')> + >>> Version.from_parts(release=(0,1,0), pre=("b", 1)) + <Version('0.1.0b1')> + + :param epoch: + :param release: This version tuple is required + + .. versionadded:: 26.1 + """ + _epoch = _validate_epoch(epoch) + _release = _validate_release(release) + _pre = _validate_pre(pre) if pre is not None else None + _post = _validate_post(post) if post is not None else None + _dev = _validate_dev(dev) if dev is not None else None + _local = _validate_local(local) if local is not None else None + + new_version = cls.__new__(cls) + new_version._key_cache = None + new_version._hash_cache = None + new_version._epoch = _epoch + new_version._release = _release + new_version._pre = _pre + new_version._post = _post + new_version._dev = _dev + new_version._local = _local + + return new_version def __replace__(self, **kwargs: Unpack[_VersionReplace]) -> Self: + """ + __replace__(*, epoch=..., release=..., pre=..., post=..., dev=..., local=...) + + Return a new version with parts replaced. + + This returns a new version (unless no parts were changed). The + pre-release is normalized. Setting a value to ``None`` clears it. + + >>> v = Version("1.2.3") + >>> v.__replace__(pre=("a", 1)) + <Version('1.2.3a1')> + + :param int | None epoch: + :param tuple[int, ...] | None release: + :param tuple[str, int] | None pre: + :param int | None post: + :param int | None dev: + :param str | None local: + + .. versionadded:: 26.0 + .. versionchanged:: 26.1 + + The pre-release portion is now normalized. + """ epoch = _validate_epoch(kwargs["epoch"]) if "epoch" in kwargs else self._epoch release = ( _validate_release(kwargs["release"]) @@ -395,6 +536,7 @@ class Version(_BaseVersion): new_version = self.__class__.__new__(self.__class__) new_version._key_cache = None + new_version._hash_cache = None new_version._epoch = epoch new_version._release = release new_version._pre = pre @@ -417,6 +559,188 @@ class Version(_BaseVersion): ) return self._key_cache + # __hash__ must be defined when __eq__ is overridden, + # otherwise Python sets __hash__ to None. + def __hash__(self) -> int: + if (cached_hash := self._hash_cache) is not None: + return cached_hash + + if (key := self._key_cache) is None: + self._key_cache = key = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + self._hash_cache = cached_hash = hash(key) + return cached_hash + + # Override comparison methods to use direct _key_cache access + # This is faster than property access, especially before Python 3.12 + def __lt__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache < other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__lt__(other) + + def __le__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache <= other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__le__(other) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache == other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__eq__(other) + + def __ge__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache >= other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__ge__(other) + + def __gt__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache > other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__gt__(other) + + def __ne__(self, other: object) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache != other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__ne__(other) + @property @_deprecated("Version._version is private and will be removed soon") def _version(self) -> _Version: @@ -434,6 +758,7 @@ class Version(_BaseVersion): self._post = value.post self._local = value.local self._key_cache = None + self._hash_cache = None def __repr__(self) -> str: """A representation of the Version that shows all internal state. @@ -441,7 +766,7 @@ class Version(_BaseVersion): >>> Version('1.0.0') <Version('1.0.0')> """ - return f"<Version('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __str__(self) -> str: """A string representation of the version that can be round-tripped. @@ -507,7 +832,7 @@ class Version(_BaseVersion): return self._release @property - def pre(self) -> tuple[str, int] | None: + def pre(self) -> tuple[Literal["a", "b", "rc"], int] | None: """The pre-release segment of the version. >>> print(Version("1.2.3").pre) @@ -561,6 +886,9 @@ class Version(_BaseVersion): def public(self) -> str: """The public portion of the version. + This returns a string. If you want a :class:`Version` again and care + about performance, use ``v.__replace__(local=None)`` instead. + >>> Version("1.2.3").public '1.2.3' >>> Version("1.2.3+abc").public @@ -574,6 +902,10 @@ class Version(_BaseVersion): def base_version(self) -> str: """The "base version" of the version. + This returns a string. If you want a :class:`Version` again and care + about performance, use + ``v.__replace__(pre=None, post=None, dev=None, local=None)`` instead. + >>> Version("1.2.3").base_version '1.2.3' >>> Version("1.2.3+abc").base_version @@ -721,7 +1053,8 @@ _local_version_separators = re.compile(r"[\._-]") def _parse_local_version(local: str | None) -> LocalType | None: """ - Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). + Takes a string like ``"abc.1.twelve"`` and turns it into + ``("abc", 1, "twelve")``. """ if local is not None: return tuple( @@ -731,6 +1064,19 @@ def _parse_local_version(local: str | None) -> LocalType | None: return None +# Sort ranks for pre-release: dev-only < a < b < rc < stable (no pre-release). +_PRE_RANK = {"a": 0, "b": 1, "rc": 2} +_PRE_RANK_DEV_ONLY = -1 # sorts before a(0) +_PRE_RANK_STABLE = 3 # sorts after rc(2) + +# In local version segments, strings sort before ints per PEP 440. +_LOCAL_STR_RANK = -1 # sorts before all non-negative ints + +# Pre-computed suffix for stable releases (no pre, post, or dev segments). +# See _cmpkey() for the suffix layout. +_STABLE_SUFFIX = (_PRE_RANK_STABLE, 0, 0, 0, 1, 0) + + def _cmpkey( epoch: int, release: tuple[int, ...], @@ -739,54 +1085,70 @@ def _cmpkey( dev: tuple[str, int] | None, local: LocalType | None, ) -> CmpKey: - # When we compare a release version, we want to compare it with all of the - # trailing zeros removed. We will use this for our sorting key. + """Build a comparison key for PEP 440 ordering. + + Returns ``(epoch, release, suffix)`` or + ``(epoch, release, suffix, local)`` so that plain tuple + comparison gives the correct order. + + Trailing zeros are stripped from the release so that ``1.0.0 == 1``. + + The suffix is a flat 6-int tuple that encodes pre/post/dev: + ``(pre_rank, pre_n, post_rank, post_n, dev_rank, dev_n)`` + + pre_rank: dev-only=-1, a=0, b=1, rc=2, no-pre=3 + Dev-only releases (no pre or post) get -1 so they sort before + any alpha/beta/rc. Releases without a pre-release tag get 3 + so they sort after rc. + post_rank: no-post=0, post=1 + Releases without a post segment sort before those with one. + dev_rank: dev=0, no-dev=1 + Releases without a dev segment sort after those with one. + + Local segments use ``(n, "")`` for ints and ``(-1, s)`` for strings, + following PEP 440: strings sort before ints, strings compare + lexicographically, ints compare numerically, and shorter segments + sort before longer when prefixes match. Versions without a local + segment sort before those with one (3-tuple < 4-tuple). + + >>> _cmpkey(0, (1, 0, 0), None, None, None, None) + (0, (1,), (3, 0, 0, 0, 1, 0)) + >>> _cmpkey(0, (1,), ("a", 1), None, None, None) + (0, (1,), (0, 1, 0, 0, 1, 0)) + >>> _cmpkey(0, (1,), None, None, None, ("ubuntu", 1)) + (0, (1,), (3, 0, 0, 0, 1, 0), ((-1, 'ubuntu'), (1, ''))) + """ + # Strip trailing zeros: 1.0.0 compares equal to 1. len_release = len(release) i = len_release while i and release[i - 1] == 0: i -= 1 - _release = release if i == len_release else release[:i] + trimmed = release if i == len_release else release[:i] + + # Fast path: stable release with no local segment. + if pre is None and post is None and dev is None and local is None: + return epoch, trimmed, _STABLE_SUFFIX - # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. - # We'll do this by abusing the pre segment, but we _only_ want to do this - # if there is not a pre or a post segment. If we have one of those then - # the normal sorting rules will handle this case correctly. if pre is None and post is None and dev is not None: - _pre: CmpPrePostDevType = NegativeInfinity - # Versions without a pre-release (except as noted above) should sort after - # those with one. + # dev-only (e.g. 1.0.dev1) sorts before all pre-releases. + pre_rank, pre_n = _PRE_RANK_DEV_ONLY, 0 elif pre is None: - _pre = Infinity + pre_rank, pre_n = _PRE_RANK_STABLE, 0 else: - _pre = pre + pre_rank, pre_n = _PRE_RANK[pre[0]], pre[1] - # Versions without a post segment should sort before those with one. - if post is None: - _post: CmpPrePostDevType = NegativeInfinity - - else: - _post = post + post_rank = 0 if post is None else 1 + post_n = 0 if post is None else post[1] - # Versions without a development segment should sort after those with one. - if dev is None: - _dev: CmpPrePostDevType = Infinity + dev_rank = 1 if dev is None else 0 + dev_n = 0 if dev is None else dev[1] - else: - _dev = dev + suffix = (pre_rank, pre_n, post_rank, post_n, dev_rank, dev_n) if local is None: - # Versions without a local segment should sort before those with one. - _local: CmpLocalType = NegativeInfinity - else: - # Versions with a local segment need that segment parsed to implement - # the sorting rules in PEP440. - # - Alpha numeric segments sort before numeric segments - # - Alpha numeric segments sort lexicographically - # - Numeric segments sort numerically - # - Shorter versions sort before longer versions when the prefixes - # match exactly - _local = tuple( - (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local - ) + return epoch, trimmed, suffix - return epoch, _release, _pre, _post, _dev, _local + cmp_local: CmpLocalType = tuple( + (seg, "") if isinstance(seg, int) else (_LOCAL_STR_RANK, seg) for seg in local + ) + return epoch, trimmed, suffix, cmp_local diff --git a/contrib/python/packaging/py3/ya.make b/contrib/python/packaging/py3/ya.make index 6cc32b0d90f..d1ccae20d2c 100644 --- a/contrib/python/packaging/py3/ya.make +++ b/contrib/python/packaging/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(26.0) +VERSION(26.1) LICENSE(BSD-2-Clause AND Apache-2.0) @@ -15,8 +15,10 @@ PY_SRCS( packaging/_manylinux.py packaging/_musllinux.py packaging/_parser.py - packaging/_structures.py packaging/_tokenizer.py + packaging/dependency_groups.py + packaging/direct_url.py + packaging/errors.py packaging/licenses/__init__.py packaging/licenses/_spdx.py packaging/markers.py |
