diff options
| author | robot-piglet <[email protected]> | 2026-05-12 16:44:51 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2026-05-12 17:23:24 +0300 |
| commit | d4e2776cb1e96d45483cdb79d17312206d28e162 (patch) | |
| tree | ae2511a120b89d8d64aa2625b6e908b3796b8aa1 /contrib/python | |
| parent | 3301ad4eaf0ab643b35017edbe57f951907fc916 (diff) | |
Intermediate changes
commit_hash:4a03f8e6cde7c6d79572c9fbd14f8d4c27061968
Diffstat (limited to 'contrib/python')
146 files changed, 11159 insertions, 7925 deletions
diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index bd042f7a870..580a70f4d19 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: hypothesis -Version: 6.152.2 +Version: 6.152.3 Summary: The property-based testing library for Python Author-email: "David R. MacIver and Zac Hatfield-Dodds" <[email protected]> License-Expression: MPL-2.0 diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/providers.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/providers.py index f3b3daef036..be8bbeebf89 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/providers.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/providers.py @@ -1232,7 +1232,12 @@ class URandom(Random): @staticmethod def _urandom(size: int) -> bytes: - with open("/dev/urandom", "rb") as f: + # By default, we buffer more data from /dev/urandom than the actual `size` number + # of bytes we requested. This trips up anyone (and particularly Antithesis) + # hooking /dev/urandom reads to control randomness. + # + # buffering=0 disables this behavior. + with open("/dev/urandom", "rb", buffering=0) as f: return f.read(size) def getrandbits(self, k: int) -> int: diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index d5d6fbf4966..57b18a91d8e 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 152, 2) +__version_info__ = (6, 152, 3) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index bea79f924ab..789baacebd1 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.152.2) +VERSION(6.152.3) LICENSE(MPL-2.0) diff --git a/contrib/python/pip/.dist-info/METADATA b/contrib/python/pip/.dist-info/METADATA index e56d5d1f85d..ca6ba1f2ff2 100644 --- a/contrib/python/pip/.dist-info/METADATA +++ b/contrib/python/pip/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.4 Name: pip -Version: 26.0.1 +Version: 26.1 Summary: The PyPA recommended tool for installing Python packages. Author-email: The pip developers <[email protected]> -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/x-rst License-Expression: MIT Classifier: Development Status :: 5 - Production/Stable @@ -12,7 +12,6 @@ Classifier: Topic :: Software Development :: Build Tools Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 @@ -24,7 +23,6 @@ License-File: AUTHORS.txt License-File: LICENSE.txt License-File: src/pip/_vendor/cachecontrol/LICENSE.txt License-File: src/pip/_vendor/certifi/LICENSE -License-File: src/pip/_vendor/dependency_groups/LICENSE.txt License-File: src/pip/_vendor/distlib/LICENSE.txt License-File: src/pip/_vendor/distro/LICENSE License-File: src/pip/_vendor/idna/LICENSE.md diff --git a/contrib/python/pip/AUTHORS.txt b/contrib/python/pip/AUTHORS.txt index 73b4dc8b3a7..f6291c9bfc8 100644 --- a/contrib/python/pip/AUTHORS.txt +++ b/contrib/python/pip/AUTHORS.txt @@ -242,6 +242,7 @@ Dimitri Papadopoulos Dimitri Papadopoulos Orfanos Dirk Stolle dkjsone +Dmitrii Sutiagin Dmitry Gladkov Dmitry Volodin Domen Kožar @@ -309,7 +310,9 @@ George Margaritis George Song Georgi Valkov Georgy Pchelkin +Gertjan van Zwieten ghost +Giancarlo Cicellyn Comneno Giftlin Rajaiah gizmoguy1 gkdoc @@ -460,6 +463,7 @@ Kevin Burke Kevin Carter Kevin Frommelt Kevin R Patterson +Kevin Turcios Kexuan Sun Kit Randel Klaas van Schelven @@ -591,6 +595,7 @@ Nitesh Sharma Niyas Sait Noah Noah Gorny +Norbert Manthey Nothing-991 Nowell Strite NtaleGrey @@ -816,6 +821,7 @@ Vikram - Google Viktor Szépe Ville Skyttä Vinay Sajip +Vincent Fazio Vincent Philippon Vinicyus Macedo Vipul Kumar @@ -850,6 +856,7 @@ Yu Jian Yuan Jing Vincent Yan Yuki Kobayashi Yusuke Hayashi +Zachary Ware zackzack38 Zearin Zhiping Deng diff --git a/contrib/python/pip/pip/__init__.py b/contrib/python/pip/pip/__init__.py index 978b4129206..b227a4f5b3f 100644 --- a/contrib/python/pip/pip/__init__.py +++ b/contrib/python/pip/pip/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -__version__ = "26.0.1" +__version__ = "26.1" def main(args: list[str] | None = None) -> int: diff --git a/contrib/python/pip/pip/__main__.py b/contrib/python/pip/pip/__main__.py index 5991326115f..094361cd1aa 100644 --- a/contrib/python/pip/pip/__main__.py +++ b/contrib/python/pip/pip/__main__.py @@ -10,7 +10,7 @@ if sys.path[0] in ("", os.getcwd()): # If we are running from a wheel, add the wheel to sys.path # This allows the usage python pip-*.whl/pip install pip-*.whl -if __package__ == "": +if not __spec__ or __spec__.parent == "": # __file__ is pip-*.whl/pip/__main__.py # first dirname call strips of '/__main__.py', second strips off '/pip' # Resulting path is the name of the wheel itself diff --git a/contrib/python/pip/pip/__pip-runner__.py b/contrib/python/pip/pip/__pip-runner__.py index d6be157831a..a1aee819530 100644 --- a/contrib/python/pip/pip/__pip-runner__.py +++ b/contrib/python/pip/pip/__pip-runner__.py @@ -9,7 +9,7 @@ an import statement. import sys # Copied from pyproject.toml -PYTHON_REQUIRES = (3, 9) +PYTHON_REQUIRES = (3, 10) def version_str(version): # type: ignore diff --git a/contrib/python/pip/pip/_internal/cli/base_command.py b/contrib/python/pip/pip/_internal/cli/base_command.py index 499a46f9640..eff71d7b05b 100644 --- a/contrib/python/pip/pip/_internal/cli/base_command.py +++ b/contrib/python/pip/pip/_internal/cli/base_command.py @@ -2,12 +2,14 @@ from __future__ import annotations +import contextlib import logging import logging.config import optparse import os import sys import traceback +from collections.abc import Iterator from optparse import Values from typing import Callable @@ -80,7 +82,8 @@ class Command(CommandContextMixIn): def add_options(self) -> None: pass - def handle_pip_version_check(self, options: Values) -> None: + @contextlib.contextmanager + def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]: """ This is a no-op so that commands by default do not do the pip version check. @@ -88,16 +91,15 @@ class Command(CommandContextMixIn): # Make sure we do the pip version check if the index_group options # are present. assert not hasattr(options, "no_index") + yield def run(self, options: Values, args: list[str]) -> int: raise NotImplementedError def _run_wrapper(self, level_number: int, options: Values, args: list[str]) -> int: def _inner_run() -> int: - try: + with self.pip_version_check(options, args): return self.run(options, args) - finally: - self.handle_pip_version_check(options) if options.debug_mode: rich_traceback.install(show_locals=True) @@ -132,11 +134,18 @@ class Command(CommandContextMixIn): return ERROR except BrokenStdoutLoggingError: - # Bypass our logger and write any remaining messages to - # stderr because stdout no longer works. - print("ERROR: Pipe to stdout was broken", file=sys.stderr) - if level_number <= logging.DEBUG: - traceback.print_exc(file=sys.stderr) + # stdout is broken; write to stderr directly. Use os.write, not + # sys.stderr.write, so a full pipe buffer returns EPIPE instead + # of deadlocking (Windows anonymous pipes are ~4KB). + try: + os.write(2, b"ERROR: Pipe to stdout was broken\n") + if level_number <= logging.DEBUG: + encoding = getattr(sys.stderr, "encoding", None) or "utf-8" + os.write( + 2, traceback.format_exc().encode(encoding, "backslashreplace") + ) + except OSError: + pass return ERROR except KeyboardInterrupt: diff --git a/contrib/python/pip/pip/_internal/cli/cmdoptions.py b/contrib/python/pip/pip/_internal/cli/cmdoptions.py index a269f8614e9..a16130131eb 100644 --- a/contrib/python/pip/pip/_internal/cli/cmdoptions.py +++ b/contrib/python/pip/pip/_internal/cli/cmdoptions.py @@ -14,7 +14,9 @@ from __future__ import annotations import logging import os import pathlib +import re import textwrap +from datetime import datetime, timedelta, timezone from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values from textwrap import dedent @@ -29,6 +31,7 @@ from pip._internal.models.format_control import FormatControl from pip._internal.models.index import PyPI from pip._internal.models.release_control import ReleaseControl from pip._internal.models.target_python import TargetPython +from pip._internal.utils import pylock as pylock_utils from pip._internal.utils.datetime import parse_iso_datetime from pip._internal.utils.hashes import STRONG_HASHES from pip._internal.utils.misc import strtobool @@ -101,6 +104,14 @@ def check_dist_restriction(options: Values, check_target: bool = False) -> None: "installing via '--target' or using '--dry-run'" ) + for filename in options.requirements: + if dist_restriction_set and pylock_utils.is_valid_pylock_filename(filename): + raise CommandError( + "Patform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + f"are not supported when selecting requirements from {filename!r}" + ) + def check_build_constraints(options: Values) -> None: """Function for validating build constraints options. @@ -434,8 +445,9 @@ def _handle_uploaded_prior_to( """ This is an optparse.Option callback for the --uploaded-prior-to option. - Parses an ISO 8601 datetime string. If no timezone is specified in the string, - local timezone is used. + Accepts either an ISO 8601 datetime string (e.g., '2023-01-01T00:00:00Z') + or a strict subset of ISO 8601 durations: PnD where n is a number of days + (e.g., 'P7D' for 7 days ago). Note: This option only works with indexes that provide upload-time metadata as specified in the simple repository API: @@ -444,6 +456,18 @@ def _handle_uploaded_prior_to( if value is None: return None + # Try ISO 8601 duration in PnD format. The leading 'P' disambiguates + # from absolute datetimes. Only whole days are supported; the format may + # be extended to more of the ISO 8601 duration syntax in the future if + # a real need is presented. + match = re.match(r"^P(\d+)D$", value, re.ASCII) + if match: + days = int(match.group(1)) + parser.values.uploaded_prior_to = datetime.now(timezone.utc) - timedelta( + days=days + ) + return + try: uploaded_prior_to = parse_iso_datetime(value) # Use local timezone if no offset is given in the ISO string. @@ -453,8 +477,9 @@ def _handle_uploaded_prior_to( except ValueError as exc: msg = ( f"invalid value: {value!r}: {exc}. " - f"Expected an ISO 8601 datetime string, " - f"e.g '2023-01-01' or '2023-01-01T00:00:00Z'" + f"Expected an ISO 8601 datetime string " + f"(e.g., '2023-01-01' or '2023-01-01T00:00:00Z') " + f"or a duration in days (e.g., 'P3D')" ) raise_option_error(parser, option=option, msg=msg) @@ -463,15 +488,17 @@ def uploaded_prior_to() -> Option: return Option( "--uploaded-prior-to", dest="uploaded_prior_to", - metavar="datetime", + metavar="datetime_or_duration", action="callback", callback=_handle_uploaded_prior_to, type="str", help=( - "Only consider packages uploaded prior to the given date time. " - "Accepts ISO 8601 strings (e.g., '2023-01-01T00:00:00Z'). " - "Uses local timezone if none specified. Only effective when " - "installing from indexes that provide upload-time metadata." + "Only consider packages uploaded prior to the given value. " + "Accepts an ISO 8601 datetime (e.g., '2023-01-01T00:00:00Z', " + "uses local timezone if none specified) or a duration in days " + "(e.g., 'P3D' for packages uploaded at least 3 days ago). " + "Only effective when installing from indexes that provide " + "upload-time metadata." ), ) @@ -524,8 +551,12 @@ def requirements() -> Option: action="append", default=[], metavar="file", - help="Install from the given requirements file. " - "This option can be used multiple times.", + help=( + "Install from the given requirements file. " + "The file or URL can be in pip's requirements.txt format, " + "or pylock.toml format. pylock.toml support is experimental. " + "This option can be used multiple times." + ), ) diff --git a/contrib/python/pip/pip/_internal/cli/index_command.py b/contrib/python/pip/pip/_internal/cli/index_command.py index 0f1ee1a4f82..7962b1572a9 100644 --- a/contrib/python/pip/pip/_internal/cli/index_command.py +++ b/contrib/python/pip/pip/_internal/cli/index_command.py @@ -8,9 +8,10 @@ so commands which don't always hit the network (e.g. list w/o --outdated or from __future__ import annotations +import contextlib import logging import os -import sys +from collections.abc import Iterator from functools import lru_cache from optparse import Values from typing import TYPE_CHECKING @@ -26,16 +27,13 @@ if TYPE_CHECKING: from pip._vendor.packaging.utils import NormalizedName from pip._internal.network.session import PipSession + from pip._internal.self_outdated_check import UpgradePrompt logger = logging.getLogger(__name__) @lru_cache def _create_truststore_ssl_context() -> SSLContext | None: - if sys.version_info < (3, 10): - logger.debug("Disabling truststore because Python version isn't 3.10+") - return None - try: import ssl except ImportError: @@ -139,10 +137,18 @@ class SessionCommandMixin(CommandContextMixIn): return session -def _pip_self_version_check(session: PipSession, options: Values) -> None: - from pip._internal.self_outdated_check import pip_self_version_check as check +def _pip_self_version_check_fetch( + session: PipSession, options: Values +) -> UpgradePrompt | None: + from pip._internal.self_outdated_check import pip_self_version_check_fetch + + return pip_self_version_check_fetch(session, options) + - check(session, options) +def _pip_self_version_check_emit(upgrade_prompt: UpgradePrompt | None) -> None: + from pip._internal.self_outdated_check import pip_self_version_check_emit + + pip_self_version_check_emit(upgrade_prompt) class IndexGroupCommand(Command, SessionCommandMixin): @@ -169,7 +175,8 @@ class IndexGroupCommand(Command, SessionCommandMixin): # No specific setting: exclude prereleases by default return True - def handle_pip_version_check(self, options: Values) -> None: + @contextlib.contextmanager + def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]: """ Do the pip version check if not disabled. @@ -179,17 +186,27 @@ class IndexGroupCommand(Command, SessionCommandMixin): assert hasattr(options, "no_index") if options.disable_pip_version_check or options.no_index: + yield return + upgrade_prompt: UpgradePrompt | None = None try: - # Otherwise, check if we're using the latest version of pip available. session = self._build_session( options, retries=0, timeout=min(5, options.timeout), ) with session: - _pip_self_version_check(session, options) + upgrade_prompt = _pip_self_version_check_fetch(session, options) except Exception: logger.warning("There was an error checking the latest version of pip.") logger.debug("See below for error", exc_info=True) + + try: + yield + finally: + try: + _pip_self_version_check_emit(upgrade_prompt) + except Exception: + logger.warning("There was an error checking the latest version of pip.") + logger.debug("See below for error", exc_info=True) diff --git a/contrib/python/pip/pip/_internal/cli/req_command.py b/contrib/python/pip/pip/_internal/cli/req_command.py index 60b82fb9928..584597f0c9c 100644 --- a/contrib/python/pip/pip/_internal/cli/req_command.py +++ b/contrib/python/pip/pip/_internal/cli/req_command.py @@ -39,6 +39,7 @@ from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, install_req_from_parsed_requirement, + install_req_from_pylock_package, install_req_from_req_string, ) from pip._internal.req.pep723 import PEP723Exception, pep723_metadata @@ -47,6 +48,10 @@ from pip._internal.req.req_file import parse_requirements from pip._internal.req.req_install import InstallRequirement from pip._internal.resolution.base import BaseResolver from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.pylock import ( + is_valid_pylock_filename, + select_from_pylock_path_or_url, +) from pip._internal.utils.temp_dir import ( TempDirectory, TempDirectoryTypeRegistry, @@ -332,6 +337,26 @@ class RequirementCommand(IndexGroupCommand): # NOTE: options.require_hashes may be set if --require-hashes is True for filename in options.requirements: + if is_valid_pylock_filename(filename): + logger.warning( + "Using pylock.toml as a requirements source " + "is an experimental feature. " + "It may be removed/changed in a future release " + "without prior warning." + ) + for package, package_dist in select_from_pylock_path_or_url( + filename, session=session + ): + requirements.append( + install_req_from_pylock_package( + package, + package_dist, + filename, + options.format_control, + user_supplied=True, + ) + ) + continue for parsed_req in parse_requirements( filename, finder=finder, options=options, session=session ): @@ -422,7 +447,7 @@ class RequirementCommand(IndexGroupCommand): options: Values, session: PipSession, target_python: TargetPython | None = None, - ignore_requires_python: bool | None = None, + ignore_requires_python: bool = False, ) -> PackageFinder: """ Create a package finder appropriate to this requirement command. diff --git a/contrib/python/pip/pip/_internal/commands/debug.py b/contrib/python/pip/pip/_internal/commands/debug.py index 0e187e79c28..25bc05d0e8e 100644 --- a/contrib/python/pip/pip/_internal/commands/debug.py +++ b/contrib/python/pip/pip/_internal/commands/debug.py @@ -55,15 +55,8 @@ def get_module_from_module_name(module_name: str) -> ModuleType | None: if module_name == "setuptools": module_name = "pkg_resources" - try: - __import__(f"pip._vendor.{module_name}", globals(), locals(), level=0) - return getattr(pip._vendor, module_name) - except ImportError: - # We allow 'truststore' to fail to import due - # to being unavailable on Python 3.9 and earlier. - if module_name == "truststore" and sys.version_info < (3, 10): - return None - raise + __import__(f"pip._vendor.{module_name}", globals(), locals(), level=0) + return getattr(pip._vendor, module_name) def get_vendor_version_from_module(module_name: str) -> str | None: diff --git a/contrib/python/pip/pip/_internal/commands/index.py b/contrib/python/pip/pip/_internal/commands/index.py index 40b8f580f4b..8c02b3bc603 100644 --- a/contrib/python/pip/pip/_internal/commands/index.py +++ b/contrib/python/pip/pip/_internal/commands/index.py @@ -91,7 +91,7 @@ class IndexCommand(IndexGroupCommand): options: Values, session: PipSession, target_python: TargetPython | None = None, - ignore_requires_python: bool | None = None, + ignore_requires_python: bool = False, ) -> PackageFinder: """ Create a package finder appropriate to the index command. diff --git a/contrib/python/pip/pip/_internal/commands/inspect.py b/contrib/python/pip/pip/_internal/commands/inspect.py index e262012ee4d..15ad165bad5 100644 --- a/contrib/python/pip/pip/_internal/commands/inspect.py +++ b/contrib/python/pip/pip/_internal/commands/inspect.py @@ -71,7 +71,7 @@ class InspectCommand(Command): # report) since it is not recorded in installed metadata. direct_url = dist.direct_url if direct_url is not None: - res["direct_url"] = direct_url.to_dict() + res["direct_url"] = direct_url.to_dict_compat() else: # Emulate direct_url for legacy editable installs. editable_project_location = dist.editable_project_location diff --git a/contrib/python/pip/pip/_internal/commands/install.py b/contrib/python/pip/pip/_internal/commands/install.py index 8c3d86648bc..27c6e7df203 100644 --- a/contrib/python/pip/pip/_internal/commands/install.py +++ b/contrib/python/pip/pip/_internal/commands/install.py @@ -1,14 +1,19 @@ from __future__ import annotations +import contextlib import errno import json import operator import os import shutil import site +import sys +from collections.abc import Iterator from optparse import SUPPRESS_HELP, Values from pathlib import Path +from typing import Any +from pip._vendor.packaging.requirements import InvalidRequirement, Requirement from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.requests.exceptions import InvalidProxyURL from pip._vendor.rich import print_json @@ -43,6 +48,7 @@ from pip._internal.req.req_install import ( InstallRequirement, ) from pip._internal.utils.compat import WINDOWS +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.filesystem import test_writable_dir from pip._internal.utils.logging import getLogger from pip._internal.utils.misc import ( @@ -63,6 +69,76 @@ from pip._internal.wheel_builder import build logger = getLogger(__name__) +_IMPORT_AUDIT_HOOK_INSTALLED = False +_MISSING_MODULES: set[str] = set() + +# Non-stdlib modules pip (or its vendored dependencies) may import lazily +# after installation has started. Importing them eagerly keeps the audit +# hook from misattributing them to a freshly installed distribution. +_EAGER_IMPORTS: tuple[str, ...] = ( + # Used by rich when emitting output to a legacy Windows console. + "pip._vendor.rich._windows_renderer", +) + + +# Imports of standard library modules are always safe: they cannot be +# shadowed by a distribution pip has just installed. +_STDLIB_MODULE_NAMES: frozenset[str] = frozenset(sys.stdlib_module_names) | frozenset( + sys.builtin_module_names +) + + +def _prevent_import_hook(name: str, args: tuple[Any, ...]) -> None: + if name != "import": + return + module = args[0] + if module in _MISSING_MODULES: + raise ImportError(f"No module named {module!r}") + if module.partition(".")[0] in _STDLIB_MODULE_NAMES: + return + deprecated( + reason=f"Unexpected import of {module!r} after pip install started.", + replacement=None, + gone_in="26.3", + issue=13842, + include_source=True, + stacklevel=3, + ) + + +def _eagerly_import_modules() -> None: + """Import modules pip uses lazily so the audit hook ignores them later.""" + for module in _EAGER_IMPORTS: + try: + __import__(module) + except ImportError: + # Record the module as missing so the hook can raise ImportError + # instead of trying to import it again. + _MISSING_MODULES.add(module) + + +def _prevent_further_imports() -> None: + """Install an audit hook that warns on unexpected imports after pip install starts. + + Eagerly pre-imports the known lazy imports first so the hook only fires + on genuinely unexpected modules. + """ + global _IMPORT_AUDIT_HOOK_INSTALLED + if _IMPORT_AUDIT_HOOK_INSTALLED: + return + + _IMPORT_AUDIT_HOOK_INSTALLED = True + sys.addaudithook(_prevent_import_hook) + + +def _arg_refers_to_pip(arg: str) -> bool: + try: + req = Requirement(arg) + except InvalidRequirement: + return False + return canonicalize_name(req.name) == "pip" + + class InstallCommand(RequirementCommand): """ Install packages from: @@ -278,6 +354,17 @@ class InstallCommand(RequirementCommand): ), ) + @contextlib.contextmanager + def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]: + # Skip the self-version check when pip itself is a requirement. The + # running pip may be replaced mid-command, and the upgrade prompt + # is redundant. + if any(_arg_refers_to_pip(arg) for arg in args): + yield + return + with super().pip_version_check(options, args): + yield + @with_cleanup def run(self, options: Values, args: list[str]) -> int: if options.use_user_site and options.target_dir is not None: @@ -459,6 +546,13 @@ class InstallCommand(RequirementCommand): if options.target_dir or options.prefix_path: warn_script_location = False + # Warn on late imports so we don't silently pick up a module + # from a distribution pip is about to install. + try: + _eagerly_import_modules() + finally: + _prevent_further_imports() + installed = install_given_reqs( to_install, root=options.root_path, diff --git a/contrib/python/pip/pip/_internal/commands/list.py b/contrib/python/pip/pip/_internal/commands/list.py index f9bad7e0d28..bc8a5696ba5 100644 --- a/contrib/python/pip/pip/_internal/commands/list.py +++ b/contrib/python/pip/pip/_internal/commands/list.py @@ -1,8 +1,9 @@ from __future__ import annotations +import contextlib import json import logging -from collections.abc import Generator, Sequence +from collections.abc import Generator, Iterator, Sequence from email.parser import Parser from optparse import Values from typing import TYPE_CHECKING, cast @@ -135,9 +136,13 @@ class ListCommand(IndexGroupCommand): self.parser.insert_option_group(0, selection_opts) self.parser.insert_option_group(0, self.cmd_opts) - def handle_pip_version_check(self, options: Values) -> None: - if options.outdated or options.uptodate: - super().handle_pip_version_check(options) + @contextlib.contextmanager + def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]: + if not (options.outdated or options.uptodate): + yield + return + with super().pip_version_check(options, args): + yield def _build_package_finder( self, options: Values, session: PipSession diff --git a/contrib/python/pip/pip/_internal/exceptions.py b/contrib/python/pip/pip/_internal/exceptions.py index 9ddda0e626b..23e1e34cd35 100644 --- a/contrib/python/pip/pip/_internal/exceptions.py +++ b/contrib/python/pip/pip/_internal/exceptions.py @@ -769,7 +769,7 @@ class UninstallMissingRecord(DiagnosticPipError): dep = f"{distribution.raw_name}=={distribution.version}" hint = Text.assemble( "You might be able to recover from this via: ", - (f"pip install --force-reinstall --no-deps {dep}", "green"), + (f"pip install --ignore-installed --no-deps {dep}", "green"), ) else: hint = Text( diff --git a/contrib/python/pip/pip/_internal/index/package_finder.py b/contrib/python/pip/pip/_internal/index/package_finder.py index aa7c2ebd48e..6cd2cad62bd 100644 --- a/contrib/python/pip/pip/_internal/index/package_finder.py +++ b/contrib/python/pip/pip/_internal/index/package_finder.py @@ -19,7 +19,7 @@ from typing import ( from pip._vendor.packaging import specifiers from pip._vendor.packaging.tags import Tag from pip._vendor.packaging.utils import NormalizedName, canonicalize_name -from pip._vendor.packaging.version import InvalidVersion, Version, _BaseVersion +from pip._vendor.packaging.version import InvalidVersion, _BaseVersion from pip._vendor.packaging.version import parse as parse_version from pip._internal.exceptions import ( @@ -137,7 +137,7 @@ class LinkEvaluator: formats: frozenset[str], target_python: TargetPython, allow_yanked: bool, - ignore_requires_python: bool | None = None, + ignore_requires_python: bool = False, uploaded_prior_to: datetime.datetime | None = None, ) -> None: """ @@ -159,8 +159,6 @@ class LinkEvaluator: :param uploaded_prior_to: If set, only allow links uploaded prior to the given datetime. """ - if ignore_requires_python is None: - ignore_requires_python = False self._allow_yanked = allow_yanked self._canonical_name = canonical_name @@ -493,7 +491,6 @@ class CandidateEvaluator: else: allow_prereleases = None specifier = self._specifier - # When using the pkg_resources backend we turn the version object into # a str here because otherwise when we're debundled but setuptools isn't, # Python will see packaging.version.Version and @@ -501,22 +498,17 @@ class CandidateEvaluator: # types. This way we'll use a str as a common data interchange # format. If we stop using the pkg_resources provided specifier # and start using our own, we can drop the cast to str(). - if select_backend().NAME == "pkg_resources": - candidates_and_versions: list[ - tuple[InstallationCandidate, str | Version] - ] = [(c, str(c.version)) for c in candidates] - else: - candidates_and_versions = [(c, c.version) for c in candidates] - versions = set( - specifier.filter( - (v for _, v in candidates_and_versions), - prereleases=allow_prereleases, - ) + applicable_candidates = specifier.filter( + candidates, + prereleases=allow_prereleases, + key=lambda c: ( + str(c.version) + if select_backend().NAME == "pkg_resources" + else c.version + ), ) - - applicable_candidates = [c for c, v in candidates_and_versions if v in versions] filtered_applicable_candidates = filter_unallowed_hashes( - candidates=applicable_candidates, + candidates=list(applicable_candidates), hashes=self._hashes, project_name=self._project_name, ) @@ -633,7 +625,7 @@ class PackageFinder: allow_yanked: bool, format_control: FormatControl | None = None, candidate_prefs: CandidatePreferences | None = None, - ignore_requires_python: bool | None = None, + ignore_requires_python: bool = False, uploaded_prior_to: datetime.datetime | None = None, ) -> None: """ @@ -660,8 +652,10 @@ class PackageFinder: self.format_control = format_control - # These are boring links that have already been logged somehow. - self._logged_links: set[tuple[Link, LinkType, str]] = set() + # Collects the detail strings for links skipped due to Requires-Python + # incompatibility. Used by requires_python_skipped_reasons() to build + # the error message when resolution fails. + self._requires_python_skipped: set[str] = set() # Cache of the result of finding candidates self._all_candidates: dict[str, list[InstallationCandidate]] = {} @@ -772,12 +766,7 @@ class PackageFinder: return self._uploaded_prior_to def requires_python_skipped_reasons(self) -> list[str]: - reasons = { - detail - for _, result, detail in self._logged_links - if result == LinkType.requires_python_mismatch - } - return sorted(reasons) + return sorted(self._requires_python_skipped) def make_link_evaluator(self, project_name: str) -> LinkEvaluator: canonical_name = canonicalize_name(project_name) @@ -810,12 +799,11 @@ class PackageFinder: return no_eggs + eggs def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None: - entry = (link, result, detail) - if entry not in self._logged_links: - # Put the link at the end so the reason is more visible and because - # the link string is usually very long. - logger.debug("Skipping link: %s: %s", detail, link) - self._logged_links.add(entry) + # Put the link at the end so the reason is more visible and because + # the link string is usually very long. + logger.debug("Skipping link: %s: %s", detail, link) + if result == LinkType.requires_python_mismatch: + self._requires_python_skipped.add(detail) def get_install_candidate( self, link_evaluator: LinkEvaluator, link: Link diff --git a/contrib/python/pip/pip/_internal/locations/__init__.py b/contrib/python/pip/pip/_internal/locations/__init__.py index c0b63205fe0..ec15d1dc704 100644 --- a/contrib/python/pip/pip/_internal/locations/__init__.py +++ b/contrib/python/pip/pip/_internal/locations/__init__.py @@ -41,13 +41,11 @@ logger = logging.getLogger(__name__) _PLATLIBDIR: str = getattr(sys, "platlibdir", "lib") -_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10) - def _should_use_sysconfig() -> bool: """This function determines the value of _USE_SYSCONFIG. - By default, pip uses sysconfig on Python 3.10+. + By default, pip uses sysconfig. But Python distributors can override this decision by setting: sysconfig._PIP_USE_SYSCONFIG = True / False Rationale in https://github.com/pypa/pip/issues/10647 @@ -55,7 +53,7 @@ def _should_use_sysconfig() -> bool: This is a function for testability, but should be constant during any one run. """ - return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT)) + return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", True)) _USE_SYSCONFIG = _should_use_sysconfig() @@ -68,10 +66,10 @@ if not _USE_SYSCONFIG: # Be noisy about incompatibilities if this platforms "should" be using # sysconfig, but is explicitly opting out and using distutils instead. -if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG: - _MISMATCH_LEVEL = logging.WARNING -else: +if _USE_SYSCONFIG: _MISMATCH_LEVEL = logging.DEBUG +else: + _MISMATCH_LEVEL = logging.WARNING def _looks_like_bpo_44860() -> bool: @@ -281,7 +279,7 @@ def get_scheme( if k == "platlib" and _looks_like_red_hat_lib(): continue - # On Python 3.9+, sysconfig's posix_user scheme sets platlib against + # sysconfig's posix_user scheme sets platlib against # sys.platlibdir, but distutils's unix_user incorrectly continues # using the same $usersite for both platlib and purelib. This creates a # mismatch when sys.platlibdir is not "lib". diff --git a/contrib/python/pip/pip/_internal/metadata/importlib/_dists.py b/contrib/python/pip/pip/_internal/metadata/importlib/_dists.py index 89364b8b7ab..a8e872d2962 100644 --- a/contrib/python/pip/pip/_internal/metadata/importlib/_dists.py +++ b/contrib/python/pip/pip/_internal/metadata/importlib/_dists.py @@ -207,7 +207,13 @@ class Distribution(BaseDistribution): # a ton of fields that we need, including get() and get_payload(). We # rely on the implementation that the object is actually a Message now, # until upstream can improve the protocol. (python/cpython#94952) - return cast(email.message.Message, self._dist.metadata) + metadata = self._dist.metadata + # From Python 3.15+, importlib.metadata may return None when no + # metadata file (METADATA or PKG-INFO) exists in the distribution + # directory. (python/cpython#132947) + if metadata is None: + return email.message.Message() + return cast(email.message.Message, metadata) def iter_provided_extras(self) -> Iterable[NormalizedName]: return [ diff --git a/contrib/python/pip/pip/_internal/models/candidate.py b/contrib/python/pip/pip/_internal/models/candidate.py index f27f283154a..88ecc99258f 100644 --- a/contrib/python/pip/pip/_internal/models/candidate.py +++ b/contrib/python/pip/pip/_internal/models/candidate.py @@ -6,12 +6,10 @@ from pip._vendor.packaging.version import parse as parse_version from pip._internal.models.link import Link -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class InstallationCandidate: """Represents a potential "candidate" for installation.""" - __slots__ = ["name", "version", "link"] - name: str version: Version link: Link diff --git a/contrib/python/pip/pip/_internal/models/direct_url.py b/contrib/python/pip/pip/_internal/models/direct_url.py index aefc670cd51..a2dfb340abf 100644 --- a/contrib/python/pip/pip/_internal/models/direct_url.py +++ b/contrib/python/pip/pip/_internal/models/direct_url.py @@ -3,225 +3,40 @@ from __future__ import annotations import json -import re -import urllib.parse -from collections.abc import Iterable -from dataclasses import dataclass -from typing import Any, ClassVar, TypeVar, Union +from typing import Any + +from pip._vendor.packaging.direct_url import ( + ArchiveInfo, + DirectUrlValidationError, + DirInfo, + VcsInfo, +) +from pip._vendor.packaging.direct_url import ( + DirectUrl as PackagingDirectUrl, +) __all__ = [ + "ArchiveInfo", + "DirInfo", "DirectUrl", "DirectUrlValidationError", - "DirInfo", - "ArchiveInfo", + "DIRECT_URL_METADATA_NAME", "VcsInfo", ] -T = TypeVar("T") - DIRECT_URL_METADATA_NAME = "direct_url.json" -ENV_VAR_RE = re.compile(r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$") - - -class DirectUrlValidationError(Exception): - pass - - -def _get( - d: dict[str, Any], expected_type: type[T], key: str, default: T | None = None -) -> T | None: - """Get value from dictionary and verify expected type.""" - if key not in d: - return default - value = d[key] - if not isinstance(value, expected_type): - raise DirectUrlValidationError( - f"{value!r} has unexpected type for {key} (expected {expected_type})" - ) - return value - - -def _get_required( - d: dict[str, Any], expected_type: type[T], key: str, default: T | None = None -) -> T: - value = _get(d, expected_type, key, default) - if value is None: - raise DirectUrlValidationError(f"{key} must have a value") - return value - - -def _exactly_one_of(infos: Iterable[InfoType | None]) -> InfoType: - infos = [info for info in infos if info is not None] - if not infos: - raise DirectUrlValidationError( - "missing one of archive_info, dir_info, vcs_info" - ) - if len(infos) > 1: - raise DirectUrlValidationError( - "more than one of archive_info, dir_info, vcs_info" - ) - assert infos[0] is not None - return infos[0] - - -def _filter_none(**kwargs: Any) -> dict[str, Any]: - """Make dict excluding None values.""" - return {k: v for k, v in kwargs.items() if v is not None} - - -@dataclass -class VcsInfo: - name: ClassVar = "vcs_info" - - vcs: str - commit_id: str - requested_revision: str | None = None - - @classmethod - def _from_dict(cls, d: dict[str, Any] | None) -> VcsInfo | None: - if d is None: - return None - return cls( - vcs=_get_required(d, str, "vcs"), - commit_id=_get_required(d, str, "commit_id"), - requested_revision=_get(d, str, "requested_revision"), - ) - - def _to_dict(self) -> dict[str, Any]: - return _filter_none( - vcs=self.vcs, - requested_revision=self.requested_revision, - commit_id=self.commit_id, - ) -class ArchiveInfo: - name = "archive_info" - - def __init__( - self, - hash: str | None = None, - hashes: dict[str, str] | None = None, - ) -> None: - # set hashes before hash, since the hash setter will further populate hashes - self.hashes = hashes - self.hash = hash - - @property - def hash(self) -> str | None: - return self._hash - - @hash.setter - def hash(self, value: str | None) -> None: - if value is not None: - # Auto-populate the hashes key to upgrade to the new format automatically. - # We don't back-populate the legacy hash key from hashes. - try: - hash_name, hash_value = value.split("=", 1) - except ValueError: - raise DirectUrlValidationError( - f"invalid archive_info.hash format: {value!r}" - ) - if self.hashes is None: - self.hashes = {hash_name: hash_value} - elif hash_name not in self.hashes: - self.hashes = self.hashes.copy() - self.hashes[hash_name] = hash_value - self._hash = value - - @classmethod - def _from_dict(cls, d: dict[str, Any] | None) -> ArchiveInfo | None: - if d is None: - return None - return cls(hash=_get(d, str, "hash"), hashes=_get(d, dict, "hashes")) - - def _to_dict(self) -> dict[str, Any]: - return _filter_none(hash=self.hash, hashes=self.hashes) - - -@dataclass -class DirInfo: - name: ClassVar = "dir_info" - - editable: bool = False - - @classmethod - def _from_dict(cls, d: dict[str, Any] | None) -> DirInfo | None: - if d is None: - return None - return cls(editable=_get_required(d, bool, "editable", default=False)) - - def _to_dict(self) -> dict[str, Any]: - return _filter_none(editable=self.editable or None) - - -InfoType = Union[ArchiveInfo, DirInfo, VcsInfo] - - -@dataclass -class DirectUrl: - url: str - info: InfoType - subdirectory: str | None = None - - def _remove_auth_from_netloc(self, netloc: str) -> str: - if "@" not in netloc: - return netloc - user_pass, netloc_no_user_pass = netloc.split("@", 1) - if ( - isinstance(self.info, VcsInfo) - and self.info.vcs == "git" - and user_pass == "git" - ): - return netloc - if ENV_VAR_RE.match(user_pass): - return netloc - return netloc_no_user_pass - - @property - def redacted_url(self) -> str: - """url with user:password part removed unless it is formed with - environment variables as specified in PEP 610, or it is ``git`` - in the case of a git URL. - """ - purl = urllib.parse.urlsplit(self.url) - netloc = self._remove_auth_from_netloc(purl.netloc) - surl = urllib.parse.urlunsplit( - (purl.scheme, netloc, purl.path, purl.query, purl.fragment) - ) - return surl - - def validate(self) -> None: - self.from_dict(self.to_dict()) - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> DirectUrl: - return DirectUrl( - url=_get_required(d, str, "url"), - subdirectory=_get(d, str, "subdirectory"), - info=_exactly_one_of( - [ - ArchiveInfo._from_dict(_get(d, dict, "archive_info")), - DirInfo._from_dict(_get(d, dict, "dir_info")), - VcsInfo._from_dict(_get(d, dict, "vcs_info")), - ] - ), - ) - - def to_dict(self) -> dict[str, Any]: - res = _filter_none( - url=self.redacted_url, - subdirectory=self.subdirectory, - ) - res[self.info.name] = self.info._to_dict() - return res +class DirectUrl(PackagingDirectUrl): + def to_dict_compat(self) -> dict[str, Any]: + return dict(super().to_dict(generate_legacy_hash=True)) @classmethod def from_json(cls, s: str) -> DirectUrl: return cls.from_dict(json.loads(s)) def to_json(self) -> str: - return json.dumps(self.to_dict(), sort_keys=True) + return json.dumps(self.to_dict_compat(), sort_keys=True) def is_local_editable(self) -> bool: - return isinstance(self.info, DirInfo) and self.info.editable + return bool(self.dir_info and self.dir_info.editable) diff --git a/contrib/python/pip/pip/_internal/models/installation_report.py b/contrib/python/pip/pip/_internal/models/installation_report.py index 3e8e9683bed..145e2cf8e57 100644 --- a/contrib/python/pip/pip/_internal/models/installation_report.py +++ b/contrib/python/pip/pip/_internal/models/installation_report.py @@ -19,7 +19,7 @@ class InstallationReport: # be absent when the requirement was installed from the wheel cache # and the cache entry was populated by an older pip version that did not # record origin.json. - "download_info": ireq.download_info.to_dict(), + "download_info": ireq.download_info.to_dict_compat(), # is_direct is true if the requirement was a direct URL reference (which # includes editable requirements), and false if the requirement was # downloaded from a PEP 503 index or --find-links. diff --git a/contrib/python/pip/pip/_internal/models/release_control.py b/contrib/python/pip/pip/_internal/models/release_control.py index f1de068630f..d432fa3e9b4 100644 --- a/contrib/python/pip/pip/_internal/models/release_control.py +++ b/contrib/python/pip/pip/_internal/models/release_control.py @@ -7,8 +7,7 @@ from pip._vendor.packaging.utils import NormalizedName, canonicalize_name from pip._internal.exceptions import CommandError -# TODO: add slots=True when Python 3.9 is dropped -@dataclass +@dataclass(slots=True) class ReleaseControl: """Helper for managing which release types can be installed.""" diff --git a/contrib/python/pip/pip/_internal/models/scheme.py b/contrib/python/pip/pip/_internal/models/scheme.py index 06a9a550e34..5ed705717c2 100644 --- a/contrib/python/pip/pip/_internal/models/scheme.py +++ b/contrib/python/pip/pip/_internal/models/scheme.py @@ -10,14 +10,12 @@ from dataclasses import dataclass SCHEME_KEYS = ["platlib", "purelib", "headers", "scripts", "data"] -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class Scheme: """A Scheme holds paths which are used as the base directories for artifacts associated with a Python package. """ - __slots__ = SCHEME_KEYS - platlib: str purelib: str headers: str diff --git a/contrib/python/pip/pip/_internal/models/search_scope.py b/contrib/python/pip/pip/_internal/models/search_scope.py index 136163ca096..b3aaa18e616 100644 --- a/contrib/python/pip/pip/_internal/models/search_scope.py +++ b/contrib/python/pip/pip/_internal/models/search_scope.py @@ -14,14 +14,12 @@ from pip._internal.utils.misc import normalize_path, redact_auth_from_url logger = logging.getLogger(__name__) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class SearchScope: """ Encapsulates the locations that pip is configured to search. """ - __slots__ = ["find_links", "index_urls", "no_index"] - find_links: list[str] index_urls: list[str] no_index: bool diff --git a/contrib/python/pip/pip/_internal/models/selection_prefs.py b/contrib/python/pip/pip/_internal/models/selection_prefs.py index 04ef63ab543..3473031926f 100644 --- a/contrib/python/pip/pip/_internal/models/selection_prefs.py +++ b/contrib/python/pip/pip/_internal/models/selection_prefs.py @@ -1,56 +1,36 @@ from __future__ import annotations +from dataclasses import dataclass + from pip._internal.models.format_control import FormatControl from pip._internal.models.release_control import ReleaseControl -# TODO: This needs Python 3.10's improved slots support for dataclasses -# to be converted into a dataclass. +@dataclass(slots=True) class SelectionPreferences: """ Encapsulates the candidate selection preferences for downloading and installing files. - """ - __slots__ = [ - "allow_yanked", - "release_control", - "format_control", - "prefer_binary", - "ignore_requires_python", - ] + :param allow_yanked: Whether files marked as yanked (in the sense + of PEP 592) are permitted to be candidates for install. + :param release_control: A ReleaseControl object or None. Used to control + whether pre-releases are allowed for specific packages. + :param format_control: A FormatControl object or None. Used to control + the selection of source packages / binary packages when consulting + the index and links. + :param prefer_binary: Whether to prefer an old, but valid, binary + dist over a new source dist. + :param ignore_requires_python: Whether to ignore incompatible + "Requires-Python" values in links. Defaults to False. + """ # Don't include an allow_yanked default value to make sure each call # site considers whether yanked releases are allowed. This also causes # that decision to be made explicit in the calling code, which helps # people when reading the code. - def __init__( - self, - allow_yanked: bool, - release_control: ReleaseControl | None = None, - format_control: FormatControl | None = None, - prefer_binary: bool = False, - ignore_requires_python: bool | None = None, - ) -> None: - """Create a SelectionPreferences object. - - :param allow_yanked: Whether files marked as yanked (in the sense - of PEP 592) are permitted to be candidates for install. - :param release_control: A ReleaseControl object or None. Used to control - whether pre-releases are allowed for specific packages. - :param format_control: A FormatControl object or None. Used to control - the selection of source packages / binary packages when consulting - the index and links. - :param prefer_binary: Whether to prefer an old, but valid, binary - dist over a new source dist. - :param ignore_requires_python: Whether to ignore incompatible - "Requires-Python" values in links. Defaults to False. - """ - if ignore_requires_python is None: - ignore_requires_python = False - - self.allow_yanked = allow_yanked - self.release_control = release_control - self.format_control = format_control - self.prefer_binary = prefer_binary - self.ignore_requires_python = ignore_requires_python + allow_yanked: bool + release_control: ReleaseControl | None = None + format_control: FormatControl | None = None + prefer_binary: bool = False + ignore_requires_python: bool = False diff --git a/contrib/python/pip/pip/_internal/network/auth.py b/contrib/python/pip/pip/_internal/network/auth.py index 4504f61a74f..1cadc143742 100644 --- a/contrib/python/pip/pip/_internal/network/auth.py +++ b/contrib/python/pip/pip/_internal/network/auth.py @@ -15,7 +15,7 @@ import typing import urllib.parse from abc import ABC, abstractmethod from functools import cache -from os.path import commonprefix +from os.path import commonpath from pathlib import Path from typing import Any, NamedTuple @@ -325,12 +325,14 @@ class MultiDomainBasicAuth(AuthBase): candidates.sort( reverse=True, - key=lambda candidate: commonprefix( - [ - parsed_url.path, - candidate.path, - ] - ).rfind("/"), + key=lambda candidate: len( + commonpath( + [ + parsed_url.path, + candidate.path, + ] + ) + ), ) return urllib.parse.urlunsplit(candidates[0]) diff --git a/contrib/python/pip/pip/_internal/network/download.py b/contrib/python/pip/pip/_internal/network/download.py index 26966423f6e..867452dfda1 100644 --- a/contrib/python/pip/pip/_internal/network/download.py +++ b/contrib/python/pip/pip/_internal/network/download.py @@ -19,7 +19,6 @@ from pip._vendor.urllib3.exceptions import ReadTimeoutError from pip._internal.cli.progress_bars import BarType, get_download_progress_renderer from pip._internal.exceptions import IncompleteDownloadError, NetworkConnectionError -from pip._internal.models.index import PyPI from pip._internal.models.link import Link from pip._internal.network.cache import SafeFileCache, is_from_cache from pip._internal.network.session import CacheControlAdapter, PipSession @@ -51,10 +50,10 @@ def _log_download( total_length: int | None, range_start: int | None = 0, ) -> Iterable[bytes]: - if link.netloc == PyPI.file_storage_domain: - url = link.show_url - else: + if logger.getEffectiveLevel() > logging.INFO: url = link.url_without_fragment + else: + url = link.show_url logged_url = redact_auth_from_url(url) diff --git a/contrib/python/pip/pip/_internal/operations/prepare.py b/contrib/python/pip/pip/_internal/operations/prepare.py index 67f9ee95041..c640d5f120c 100644 --- a/contrib/python/pip/pip/_internal/operations/prepare.py +++ b/contrib/python/pip/pip/_internal/operations/prepare.py @@ -28,7 +28,7 @@ from pip._internal.exceptions import ( ) from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution, get_metadata_distribution -from pip._internal.models.direct_url import ArchiveInfo +from pip._internal.models.direct_url import ArchiveInfo, DirectUrl from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel from pip._internal.network.download import Downloader @@ -586,9 +586,9 @@ class RequirementPreparer: # We need to verify hashes, and we have found the requirement in the cache # of locally built wheels. if ( - isinstance(req.download_info.info, ArchiveInfo) - and req.download_info.info.hashes - and hashes.has_one_of(req.download_info.info.hashes) + req.download_info.archive_info + and req.download_info.archive_info.hashes + and hashes.has_one_of(req.download_info.archive_info.hashes) ): # At this point we know the requirement was built from a hashable source # artifact, and we verified that the cache entry's hash of the original @@ -640,14 +640,18 @@ class RequirementPreparer: # compute it from the downloaded file. # FIXME: https://github.com/pypa/pip/issues/11943 if ( - isinstance(req.download_info.info, ArchiveInfo) - and not req.download_info.info.hashes + req.download_info.archive_info + and not req.download_info.archive_info.hashes and local_file ): hash = hash_file(local_file.path)[0].hexdigest() - # We populate info.hash for backward compatibility. - # This will automatically populate info.hashes. - req.download_info.info.hash = f"sha256={hash}" + # We populate archive_info.hashes. For backward compatibility, + # the legacy hash field will be generated when converting to JSON. + req.download_info = DirectUrl( + url=req.download_info.url, + archive_info=ArchiveInfo(hashes={"sha256": hash}), + subdirectory=req.download_info.subdirectory, + ) # For use in later processing, # preserve the file path on the requirement. diff --git a/contrib/python/pip/pip/_internal/req/constructors.py b/contrib/python/pip/pip/_internal/req/constructors.py index 7fef6b7e0e5..2b9d5ac16fd 100644 --- a/contrib/python/pip/pip/_internal/req/constructors.py +++ b/contrib/python/pip/pip/_internal/req/constructors.py @@ -14,14 +14,16 @@ import copy import logging import os import re -from collections.abc import Collection +from collections.abc import Collection, Mapping from dataclasses import dataclass +from pip._vendor.packaging import pylock from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pip._vendor.packaging.specifiers import Specifier +from pip._vendor.packaging.utils import parse_sdist_filename, parse_wheel_filename from pip._internal.exceptions import InstallationError +from pip._internal.models.format_control import FormatControl from pip._internal.models.index import PyPI, TestPyPI from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel @@ -30,6 +32,13 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.filetypes import is_archive_file from pip._internal.utils.misc import is_installable_dir from pip._internal.utils.packaging import get_requirement +from pip._internal.utils.pylock import ( + package_archive_requirement_url, + package_directory_requirement_url, + package_sdist_requirement_url, + package_vcs_requirement_url, + package_wheel_requirement_url, +) from pip._internal.utils.urls import path_to_url from pip._internal.vcs import is_url, vcs @@ -40,7 +49,10 @@ __all__ = [ ] logger = logging.getLogger(__name__) -operators = Specifier._operators.keys() + +# All standard version specifier operators +# https://packaging.python.org/en/latest/specifications/version-specifiers/#id5 +operators = ("~=", "==", "!=", "<=", ">=", "<", ">", "===") def _strip_extras(path: str) -> tuple[str, str | None]: @@ -566,3 +578,100 @@ def install_req_extend_extras( else None ) return result + + +def _pylock_hashes_to_hash_options(hashes: Mapping[str, str]) -> dict[str, list[str]]: + return {k: [v] for k, v in hashes.items()} + + +def install_req_from_pylock_package( + package: pylock.Package, + package_dist: ( + pylock.PackageVcs + | pylock.PackageArchive + | pylock.PackageDirectory + | pylock.PackageSdist + | pylock.PackageWheel + ), + pylock_path_or_url: str, + format_control: FormatControl, + user_supplied: bool, +) -> InstallRequirement: + pass + # TODO: validate file size + if isinstance(package_dist, pylock.PackageVcs): + return InstallRequirement( + req=Requirement( + f"{package.name} @ " + f"{package_vcs_requirement_url(pylock_path_or_url, package_dist)}" + ), + comes_from=pylock_path_or_url, + user_supplied=user_supplied, + ) + elif isinstance(package_dist, pylock.PackageArchive): + return InstallRequirement( + req=Requirement( + f"{package.name} @ " + f"{package_archive_requirement_url(pylock_path_or_url, package_dist)}" + ), + comes_from=pylock_path_or_url, + hash_options=_pylock_hashes_to_hash_options(package_dist.hashes), + user_supplied=user_supplied, + ) + elif isinstance(package_dist, pylock.PackageDirectory): + req = package_directory_requirement_url(pylock_path_or_url, package_dist) + if package_dist.editable: + return install_req_from_editable( + req, + comes_from=pylock_path_or_url, + user_supplied=user_supplied, + ) + else: + return install_req_from_line( + req, + comes_from=pylock_path_or_url, + user_supplied=user_supplied, + ) + else: + # wheel or sdist + allowed_formats = format_control.get_allowed_formats(package.name) + if ( + isinstance(package_dist, pylock.PackageSdist) + and "source" not in allowed_formats + ): + raise InstallationError( + f"source distributions are not permitted for package {package.name!r} " + f"and there is no compatible wheel for it in {pylock_path_or_url!r}" + ) + if ( + isinstance(package_dist, pylock.PackageWheel) + and "binary" not in allowed_formats + ): + if not package.sdist: + raise InstallationError( + f"binaries are not permitted for package {package.name!r} and " + f"there is no source distribution for it in {pylock_path_or_url!r}" + ) + package_dist = package.sdist + version = package.version + if isinstance(package_dist, pylock.PackageWheel): + if not version: + _, version, _, _ = parse_wheel_filename(package_dist.filename) + requirement_url = package_wheel_requirement_url( + pylock_path_or_url, package_dist + ) + elif isinstance(package_dist, pylock.PackageSdist): + if not version: + _, version = parse_sdist_filename(package_dist.filename) + requirement_url = package_sdist_requirement_url( + pylock_path_or_url, package_dist + ) + ireq = InstallRequirement( + req=Requirement(f"{package.name}=={version}"), + comes_from=pylock_path_or_url, + locked_link=Link(requirement_url), + locked_version=version, + hash_options=_pylock_hashes_to_hash_options(package_dist.hashes), + user_supplied=user_supplied, + ) + return ireq diff --git a/contrib/python/pip/pip/_internal/req/pep723.py b/contrib/python/pip/pip/_internal/req/pep723.py index 805abed7833..2e0c1b970ee 100644 --- a/contrib/python/pip/pip/_internal/req/pep723.py +++ b/contrib/python/pip/pip/_internal/req/pep723.py @@ -14,7 +14,7 @@ class PEP723Exception(ValueError): def pep723_metadata(scriptfile: str) -> dict[str, Any]: - with open(scriptfile) as f: + with open(scriptfile, encoding="utf8") as f: script = f.read() name = "script" diff --git a/contrib/python/pip/pip/_internal/req/req_dependency_group.py b/contrib/python/pip/pip/_internal/req/req_dependency_group.py index 396ac1bb635..eb9d80248af 100644 --- a/contrib/python/pip/pip/_internal/req/req_dependency_group.py +++ b/contrib/python/pip/pip/_internal/req/req_dependency_group.py @@ -1,7 +1,8 @@ from collections.abc import Iterable, Iterator from typing import Any -from pip._vendor.dependency_groups import DependencyGroupResolver +from pip._vendor.packaging.dependency_groups import DependencyGroupResolver +from pip._vendor.packaging.errors import ExceptionGroup from pip._internal.exceptions import InstallationError from pip._internal.utils.compat import tomllib @@ -28,11 +29,13 @@ def _resolve_all_groups( resolver = resolvers[path] try: yield from (str(req) for req in resolver.resolve(groupname)) - except (ValueError, TypeError, LookupError) as e: + except ExceptionGroup as eg: + # Convert ExceptionGroup to a single InstallationError with all messages + messages = [str(e) for e in eg.exceptions] raise InstallationError( f"[dependency-groups] resolution failed for '{groupname}' " - f"from '{path}': {e}" - ) from e + f"from '{path}': {'; '.join(messages)}" + ) from eg def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]: @@ -54,7 +57,15 @@ def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]: "Cannot resolve '--group' option." ) - resolvers[path] = DependencyGroupResolver(raw_dependency_groups) + try: + resolvers[path] = DependencyGroupResolver(raw_dependency_groups) + except ExceptionGroup as eg: + # Handle ExceptionGroup from resolver initialization + messages = [str(e) for e in eg.exceptions] + raise InstallationError( + f"[dependency-groups] data was invalid in {path}: {'; '.join(messages)}" + ) from eg + return resolvers diff --git a/contrib/python/pip/pip/_internal/req/req_file.py b/contrib/python/pip/pip/_internal/req/req_file.py index 9eb58ce665b..80cfdd36abe 100644 --- a/contrib/python/pip/pip/_internal/req/req_file.py +++ b/contrib/python/pip/pip/_internal/req/req_file.py @@ -101,18 +101,8 @@ DEFAULT_ENCODING = "utf-8" logger = logging.getLogger(__name__) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ParsedRequirement: - # TODO: replace this with slots=True when dropping Python 3.9 support. - __slots__ = ( - "requirement", - "is_editable", - "comes_from", - "constraint", - "options", - "line_source", - ) - requirement: str is_editable: bool comes_from: str @@ -121,10 +111,8 @@ class ParsedRequirement: line_source: str | None -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ParsedLine: - __slots__ = ("filename", "lineno", "args", "opts", "constraint") - filename: str lineno: int args: str @@ -412,7 +400,7 @@ class RequirementsFileParser: def _parse_file( self, filename: str, constraint: bool ) -> Generator[ParsedLine, None, None]: - _, content = get_file_content(filename, self._session) + _, content = get_file_content(filename, self._session, constraint=constraint) lines_enum = preprocess(content) @@ -572,7 +560,9 @@ def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines: yield line_number, line -def get_file_content(url: str, session: PipSession) -> tuple[str, str]: +def get_file_content( + url: str, session: PipSession, *, constraint: bool = False +) -> tuple[str, str]: """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. @@ -595,7 +585,8 @@ def get_file_content(url: str, session: PipSession) -> tuple[str, str]: with open(url, "rb") as f: raw_content = f.read() except OSError as exc: - raise InstallationError(f"Could not open requirements file: {exc}") + kind = "constraint" if constraint else "requirements" + raise InstallationError(f"Could not open {kind} file: {exc}") content = _decode_req_file(raw_content, url) diff --git a/contrib/python/pip/pip/_internal/req/req_install.py b/contrib/python/pip/pip/_internal/req/req_install.py index bd4fb0717d6..aad2f0ee41d 100644 --- a/contrib/python/pip/pip/_internal/req/req_install.py +++ b/contrib/python/pip/pip/_internal/req/req_install.py @@ -81,6 +81,8 @@ class InstallRequirement: extras: Collection[str] = (), user_supplied: bool = False, permit_editable_wheels: bool = False, + locked_link: Link | None = None, + locked_version: Version | None = None, ) -> None: assert req is None or isinstance(req, Requirement), req self.req = req @@ -107,6 +109,14 @@ class InstallRequirement: link = Link(req.url) self.link = self.original_link = link + # locked_link is the link from the lock file that must be used. + # A locked link InstallRequirement behaves similarly as a regular requirement + # that would be searched in indexes, except its artifact URL is known + # in advance. Notably, and contrarily to direct URL requirements and direct URL + # constraints, they do not cause the recording of direct_url.json. + self.locked_link = locked_link + self.locked_version = locked_version + # When this InstallRequirement is a wheel obtained from the cache of locally # built wheels, this is the source link corresponding to the cache entry, which # was used to download and build the cached wheel. diff --git a/contrib/python/pip/pip/_internal/req/req_uninstall.py b/contrib/python/pip/pip/_internal/req/req_uninstall.py index 3f3dde2fdd9..adb215c350a 100644 --- a/contrib/python/pip/pip/_internal/req/req_uninstall.py +++ b/contrib/python/pip/pip/_internal/req/req_uninstall.py @@ -5,7 +5,6 @@ import os import sys import sysconfig from collections.abc import Generator, Iterable -from importlib.util import cache_from_source from typing import Any, Callable from pip._internal.exceptions import LegacyDistutilsInstall, UninstallMissingRecord @@ -337,8 +336,14 @@ class UninstallPathSet: # __pycache__ files can show up after 'installed-files.txt' is created, # due to imports + # Add the adjacent __pycache__ directory to the UninstallPathSet when a + # .py file is removed. We do this to avoid the risk of orphaned .pyc + # files created by a different interpreter version than the one running + # pip at the time of package installation and uninstallation or an + # interpreter run at a different optimization level (PYTHONOPTIMIZE). if os.path.splitext(path)[1] == ".py": - self.add(cache_from_source(path)) + pycache = os.path.join(os.path.dirname(path), "__pycache__") + self.add(pycache) def add_pth(self, pth_file: str, entry: str) -> None: pth_file = self._normalize_path_cached(pth_file) diff --git a/contrib/python/pip/pip/_internal/resolution/legacy/resolver.py b/contrib/python/pip/pip/_internal/resolution/legacy/resolver.py index 33a4fdc3bbc..6cc631105f2 100644 --- a/contrib/python/pip/pip/_internal/resolution/legacy/resolver.py +++ b/contrib/python/pip/pip/_internal/resolution/legacy/resolver.py @@ -410,7 +410,7 @@ class Resolver(BaseResolver): If preparer.require_hashes is True, don't use the wheel cache, because cached wheels, always built locally, have different hashes than the files downloaded from the index server and thus throw false hash - mismatches. Furthermore, cached wheels at present have undeterministic + mismatches. Furthermore, cached wheels at present have nondeterministic contents due to file modification times. """ if req.link is None: diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/base.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/base.py index 03877b6c2dd..ececbce9258 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/base.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/base.py @@ -26,16 +26,23 @@ def format_name(project: NormalizedName, extras: frozenset[NormalizedName]) -> s class Constraint: specifier: SpecifierSet hashes: Hashes + hash_options: dict[str, list[str]] links: frozenset[Link] @classmethod def empty(cls) -> Constraint: - return Constraint(SpecifierSet(), Hashes(), frozenset()) + return Constraint(SpecifierSet(), Hashes(), {}, frozenset()) @classmethod def from_ireq(cls, ireq: InstallRequirement) -> Constraint: links = frozenset([ireq.link]) if ireq.link else frozenset() - return Constraint(ireq.specifier, ireq.hashes(trust_internet=False), links) + hash_options = {alg: list(v) for alg, v in ireq.hash_options.items()} + return Constraint( + ireq.specifier, + ireq.hashes(trust_internet=False), + hash_options, + links, + ) def __bool__(self) -> bool: return bool(self.specifier) or bool(self.hashes) or bool(self.links) @@ -45,10 +52,19 @@ class Constraint: return NotImplemented specifier = self.specifier & other.specifier hashes = self.hashes & other.hashes(trust_internet=False) + if not self.hash_options: + hash_options = {alg: list(v) for alg, v in other.hash_options.items()} + elif not other.hash_options: + hash_options = {alg: list(v) for alg, v in self.hash_options.items()} + else: + hash_options = { + alg: [v for v in other.hash_options[alg] if v in self.hash_options[alg]] + for alg in self.hash_options.keys() & other.hash_options.keys() + } links = self.links if other.link: links = links.union([other.link]) - return Constraint(specifier, hashes, links) + return Constraint(specifier, hashes, hash_options, links) def is_satisfied_by(self, candidate: Candidate) -> bool: # Reject if there are any mismatched URL constraints on this package. @@ -59,6 +75,12 @@ class Constraint: # prerelease candidates if the user does not expect them. return self.specifier.contains(candidate.version, prereleases=True) + def format_for_error(self) -> str: + s = str(self.specifier) + if self.links: + s += f" (from {', '.join(str(link) for link in self.links)})" + return s + class Requirement: @property diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py index aa126d4888e..4bfd8296107 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py @@ -58,10 +58,17 @@ def as_base_candidate(candidate: Candidate) -> BaseCandidate | None: def make_install_req_from_link( - link: Link, template: InstallRequirement + link: Link, + template: InstallRequirement, + version: Version | None = None, ) -> InstallRequirement: assert not template.editable, "template is editable" - if template.req: + if version is not None and template.req and template.hash_options: + # When hashes are provided via constraints for an unpinned requirement, + # the resulting install requirement must appear pinned so that the + # hash-checking logic does not reject it as HashUnpinned. + line = f"{template.req.name}=={version}" + elif template.req: line = str(template.req) else: line = link.url @@ -203,7 +210,8 @@ class _InstallRequirementBackedCandidate(Candidate): def format_for_error(self) -> str: return ( f"{self.name} {self.version} " - f"(from {self._link.file_path if self._link.is_file else self._link})" + f"(from {'editable ' if self.is_editable else ''}" + f"{self._link.file_path if self._link.is_file else self._link})" ) def _prepare_distribution(self) -> BaseDistribution: @@ -288,7 +296,7 @@ class LinkCandidate(_InstallRequirementBackedCandidate): if cache_entry is not None: logger.debug("Using cached wheel link: %s", cache_entry.link) link = cache_entry.link - ireq = make_install_req_from_link(link, template) + ireq = make_install_req_from_link(link, template, version=version) assert ireq.link == link if ireq.link.is_wheel and not ireq.link.is_file: wheel = Wheel(ireq.link.filename) diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py index ede3e6b2b94..a74200a71b6 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import copy import functools import logging from collections.abc import Iterable, Iterator, Mapping, Sequence @@ -31,6 +32,7 @@ from pip._internal.exceptions import ( ) from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution, get_default_environment +from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel from pip._internal.operations.prepare import RequirementPreparer @@ -241,6 +243,31 @@ class Factory: return None return self._link_candidate_cache[link] + def _get_locked_installation_candidate( + self, ireqs: Sequence[InstallRequirement], name: str, specifier: SpecifierSet + ) -> InstallationCandidate | None: + locked_ireqs = [ireq for ireq in ireqs if ireq.locked_link] + if not locked_ireqs: + return None + if len(locked_ireqs) > 1: + raise InstallationError( + f"Multiple locks provided for package {name!r} in " + f"{', '.join(str(lir.comes_from) for lir in locked_ireqs)}" + ) + locked_ireq = locked_ireqs[0] + assert locked_ireq.locked_link + assert locked_ireq.locked_version + if not specifier.contains(locked_ireq.locked_version): + raise InstallationError( + f"Locked version {locked_ireq.locked_version!s} " + f"for package {name!r} from {locked_ireq.comes_from!r} " + f"is not compatible with other requirements " + f"for the same package ({specifier!s})" + ) + return InstallationCandidate( + name, str(locked_ireq.locked_version), locked_ireq.locked_link + ) + def _iter_found_candidates( self, ireqs: Sequence[InstallRequirement], @@ -248,6 +275,7 @@ class Factory: hashes: Hashes, prefers_installed: bool, incompatible_ids: set[int], + constraint_hash_options: dict[str, list[str]] | None = None, ) -> Iterable[Candidate]: if not ireqs: return () @@ -258,6 +286,16 @@ class Factory: # Hopefully the Project model can correct this mismatch in the future. template = ireqs[0] assert template.req, "Candidates found on index must be PEP 508" + if ( + constraint_hash_options + and not template.hash_options + and any(constraint_hash_options.values()) + ): + template = copy.copy(template) + template.hash_options = { + k: list(v) for k, v in constraint_hash_options.items() + } + assert template.req # to prevent mypy from being confused by the copy name = canonicalize_name(template.req.name) extras: frozenset[str] = frozenset() @@ -297,12 +335,20 @@ class Factory: return candidate def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]: - result = self._finder.find_best_candidate( - project_name=name, - specifier=specifier, - hashes=hashes, - ) - icans = result.applicable_candidates + if locked_ican := self._get_locked_installation_candidate( + ireqs, name, specifier + ): + # Locked InstallRequirements must behave as if they would have + # been found on an index, except the link is already known, so we don't + # ask the finder for the best candidate in that case. + icans = [locked_ican] + else: + result = self._finder.find_best_candidate( + project_name=name, + specifier=specifier, + hashes=hashes, + ) + icans = result.applicable_candidates # PEP 592: Yanked releases are ignored unless the specifier # explicitly pins a version (via '==' or '===') that can be @@ -376,16 +422,28 @@ class Factory: This creates "fake" InstallRequirement objects that are basically clones of what "should" be the template, but with original_link set to link. """ + extras: frozenset[str] = frozenset() + base_identifier = identifier + with contextlib.suppress(InvalidRequirement): + parsed_requirement = get_requirement(identifier) + if parsed_requirement.name != identifier: + base_identifier = canonicalize_name(parsed_requirement.name) + extras = frozenset(parsed_requirement.extras) + for link in constraint.links: self._fail_if_link_is_unsupported_wheel(link) - candidate = self._make_base_candidate_from_link( + base_candidate = self._make_base_candidate_from_link( link, template=install_req_from_link_and_ireq(link, template), - name=canonicalize_name(identifier), + name=canonicalize_name(base_identifier), version=None, ) - if candidate: - yield candidate + if base_candidate is None: + continue + if extras: + yield self._make_extras_candidate(base_candidate, extras) + else: + yield base_candidate def find_candidates( self, @@ -453,6 +511,7 @@ class Factory: constraint.hashes, prefers_installed, incompat_ids, + constraint.hash_options, ) return ( @@ -706,8 +765,7 @@ class Factory: version_type = "final version" logger.critical( - "Could not find a %s that satisfies the requirement %s " - "(from versions: %s)", + "Could not find a %s that satisfies the requirement %s (from versions: %s)", version_type, req_disp, ", ".join(versions) or "none", @@ -819,8 +877,8 @@ class Factory: msg = msg + "The user requested " msg = msg + req.format_for_error() for key in relevant_constraints: - spec = constraints[key].specifier - msg += f"\n The user requested (constraint) {key}{spec}" + constraint_text = f"{key}{constraints[key].format_for_error()}" + msg += f"\n The user requested (constraint) {constraint_text}" # Check for causes that had no candidates causes = set() diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/provider.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/provider.py index 994748dba4f..6641d912490 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/provider.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/provider.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +from collections import defaultdict from collections.abc import Iterable, Iterator, Mapping, Sequence from functools import cache from typing import ( @@ -27,6 +28,8 @@ if TYPE_CHECKING: else: _ProviderBase = AbstractProvider +_CONFLICT_PRIORITY_THRESHOLD = 5 + # Notes on the relationship between the provider, the factory, and the # candidate and requirement classes. # @@ -99,6 +102,8 @@ class PipProvider(_ProviderBase): self._ignore_dependencies = ignore_dependencies self._upgrade_strategy = upgrade_strategy self._user_requested = user_requested + self._conflict_counts: defaultdict[str, int] = defaultdict(int) + self._conflict_promoted: set[str] = set() @property def constraints(self) -> dict[str, Constraint]: @@ -130,29 +135,42 @@ class PipProvider(_ProviderBase): Further, the current backtrack causes likely need to be resolved before other requirements as a resolution can't be found while there is a conflict. + * Identifiers that repeatedly appear as not-yet-pinned in conflicts + get promoted so they are resolved earlier. This lets their + constraints take effect before other packages pick a version. """ backtrack_identifiers = set() for info in backtrack_causes: - backtrack_identifiers.add(info.requirement.name) + names = [info.requirement.name] if info.parent is not None: - backtrack_identifiers.add(info.parent.name) + names.append(info.parent.name) + for name in names: + backtrack_identifiers.add(name) + if name not in resolutions: + self._conflict_counts[name] += 1 + if self._conflict_counts[name] >= _CONFLICT_PRIORITY_THRESHOLD: + self._conflict_promoted.add(name) current_backtrack_causes = [] + promoted = [] for identifier in identifiers: - # Requires-Python has only one candidate and the check is basically - # free, so we always do it first to avoid needless work if it fails. - # This skips calling get_preference() for all other identifiers. if identifier == REQUIRES_PYTHON_IDENTIFIER: return [identifier] - # Check if this identifier is a backtrack cause if identifier in backtrack_identifiers: current_backtrack_causes.append(identifier) continue + if identifier in self._conflict_promoted: + promoted.append(identifier) + continue + if current_backtrack_causes: return current_backtrack_causes + if promoted: + return promoted + return identifiers def get_preference( @@ -223,7 +241,10 @@ class PipProvider(_ProviderBase): unfree = bool(operators) requested_order = self._user_requested.get(identifier, math.inf) + conflict_promoted = identifier in self._conflict_promoted + return ( + not conflict_promoted, not direct, not pinned, not upper_bounded, diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/reporter.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/reporter.py index 6ba9bbd706e..de0deb26b43 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/reporter.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/reporter.py @@ -64,7 +64,7 @@ class PipReporter(BaseReporter[Requirement, Candidate, str]): name = candidate.name constraint = self._constraints.get(name) if constraint and constraint.specifier: - constraint_text = f"{name}{constraint.specifier}" + constraint_text = f"{name}{constraint.format_for_error()}" msg += f"\n The user requested (constraint) {constraint_text}" logger.debug(msg) diff --git a/contrib/python/pip/pip/_internal/self_outdated_check.py b/contrib/python/pip/pip/_internal/self_outdated_check.py index e131ec8f84b..4b5248ed3de 100644 --- a/contrib/python/pip/pip/_internal/self_outdated_check.py +++ b/contrib/python/pip/pip/_internal/self_outdated_check.py @@ -1,7 +1,6 @@ from __future__ import annotations import datetime -import functools import hashlib import json import logging @@ -9,7 +8,6 @@ import optparse import os.path import sys from dataclasses import dataclass -from typing import Callable from pip._vendor.packaging.version import Version from pip._vendor.packaging.version import parse as parse_version @@ -156,16 +154,6 @@ class UpgradePrompt: ) -def was_installed_by_pip(pkg: str) -> bool: - """Checks whether pkg was installed by pip - - This is used not to display the upgrade message when pip is in fact - installed by system package manager, such as dnf on Fedora. - """ - dist = get_default_environment().get_distribution(pkg) - return dist is not None and "pip" == dist.installer - - def _get_current_remote_pip_version( session: PipSession, options: optparse.Values ) -> str | None: @@ -194,28 +182,15 @@ def _get_current_remote_pip_version( return str(best_candidate.version) -def _self_version_check_logic( - *, - state: SelfCheckState, - current_time: datetime.datetime, - local_version: Version, - get_remote_version: Callable[[], str | None], +def _compute_upgrade_prompt( + local_version: Version, remote_version_str: str, installed_by_pip: bool ) -> UpgradePrompt | None: - remote_version_str = state.get(current_time) - if remote_version_str is None: - remote_version_str = get_remote_version() - if remote_version_str is None: - logger.debug("No remote pip version found") - return None - state.set(remote_version_str, current_time) - remote_version = parse_version(remote_version_str) logger.debug("Remote version of pip: %s", remote_version) logger.debug("Local version of pip: %s", local_version) + logger.debug("Was pip installed by pip? %s", installed_by_pip) - pip_installed_by_pip = was_installed_by_pip("pip") - logger.debug("Was pip installed by pip? %s", pip_installed_by_pip) - if not pip_installed_by_pip: + if not installed_by_pip: return None # Only suggest upgrade if pip is installed by pip. local_version_is_older = ( @@ -228,28 +203,44 @@ def _self_version_check_logic( return None -def pip_self_version_check(session: PipSession, options: optparse.Values) -> None: - """Check for an update for pip. +def pip_self_version_check_fetch( + session: PipSession, options: optparse.Values +) -> UpgradePrompt | None: + """Compute the pip upgrade prompt, if any, before the command runs. Limit the frequency of checks to once per week. State is stored either in the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix of the pip script path. + + Pair with :func:`pip_self_version_check_emit`, which displays the prompt + after the command body runs. """ installed_dist = get_default_environment().get_distribution("pip") if not installed_dist: - return + return None try: check_externally_managed() except ExternallyManagedEnvironment: - return + return None + + state = SelfCheckState(cache_dir=options.cache_dir) + current_time = datetime.datetime.now(datetime.timezone.utc) + remote_version_str = state.get(current_time) + if remote_version_str is None: + remote_version_str = _get_current_remote_pip_version(session, options) + if remote_version_str is None: + logger.debug("No remote pip version found") + return None + state.set(remote_version_str, current_time) - upgrade_prompt = _self_version_check_logic( - state=SelfCheckState(cache_dir=options.cache_dir), - current_time=datetime.datetime.now(datetime.timezone.utc), + return _compute_upgrade_prompt( local_version=installed_dist.version, - get_remote_version=functools.partial( - _get_current_remote_pip_version, session, options - ), + remote_version_str=remote_version_str, + installed_by_pip=installed_dist.installer == "pip", ) + + +def pip_self_version_check_emit(upgrade_prompt: UpgradePrompt | None) -> None: + """Emit the upgrade prompt captured by :func:`pip_self_version_check_fetch`.""" if upgrade_prompt is not None: logger.warning("%s", upgrade_prompt, extra={"rich": True}) diff --git a/contrib/python/pip/pip/_internal/utils/deprecation.py b/contrib/python/pip/pip/_internal/utils/deprecation.py index 96e7783feb3..d586da7590e 100644 --- a/contrib/python/pip/pip/_internal/utils/deprecation.py +++ b/contrib/python/pip/pip/_internal/utils/deprecation.py @@ -16,7 +16,7 @@ DEPRECATION_MSG_PREFIX = "DEPRECATION: " class PipDeprecationWarning(Warning): - pass + include_source: bool = False _original_showwarning: Any = None @@ -38,7 +38,10 @@ def _showwarning( # We use a specially named logger which will handle all of the # deprecation messages for pip. logger = logging.getLogger("pip._internal.deprecations") - logger.warning(message) + if isinstance(message, PipDeprecationWarning) and message.include_source: + logger.warning("%s (%s:%s)", message, filename, lineno) + else: + logger.warning(message) else: _original_showwarning(message, category, filename, lineno, file, line) @@ -61,6 +64,8 @@ def deprecated( gone_in: str | None, feature_flag: str | None = None, issue: int | None = None, + stacklevel: int = 2, + include_source: bool = False, ) -> None: """Helper to deprecate existing functionality. @@ -80,6 +85,12 @@ def deprecated( issue: Issue number on the tracker that would serve as a useful place for users to find related discussion and provide feedback. + stacklevel: + How many frames up the call stack to attribute the warning to. + Defaults to 2 (the caller of deprecated()). + include_source: + If True, include the source filename and line number in the warning + output. Useful when the warning originates from external code. """ # Determine whether or not the feature is already gone in this version. @@ -123,4 +134,6 @@ def deprecated( if is_gone: raise PipDeprecationWarning(message) - warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) + warning = PipDeprecationWarning(message) + warning.include_source = include_source + warnings.warn(warning, stacklevel=stacklevel) diff --git a/contrib/python/pip/pip/_internal/utils/direct_url_helpers.py b/contrib/python/pip/pip/_internal/utils/direct_url_helpers.py index 3cbc1e76344..f2d6e9037d5 100644 --- a/contrib/python/pip/pip/_internal/utils/direct_url_helpers.py +++ b/contrib/python/pip/pip/_internal/utils/direct_url_helpers.py @@ -11,16 +11,20 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s direct_url.validate() # if invalid, this is a pip bug requirement = name + " @ " fragments = [] - if isinstance(direct_url.info, VcsInfo): + if direct_url.vcs_info: requirement += ( - f"{direct_url.info.vcs}+{direct_url.url}@{direct_url.info.commit_id}" + f"{direct_url.vcs_info.vcs}+{direct_url.url}" + f"@{direct_url.vcs_info.commit_id}" ) - elif isinstance(direct_url.info, ArchiveInfo): + elif direct_url.archive_info: requirement += direct_url.url - if direct_url.info.hash: - fragments.append(direct_url.info.hash) + if direct_url.archive_info.hashes: + hash_algorithm, hash_value = next( + iter(direct_url.archive_info.hashes.items()) + ) + fragments.append(f"{hash_algorithm}={hash_value}") else: - assert isinstance(direct_url.info, DirInfo) + assert direct_url.dir_info requirement += direct_url.url if direct_url.subdirectory: fragments.append("subdirectory=" + direct_url.subdirectory) @@ -32,7 +36,7 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s def direct_url_for_editable(source_dir: str) -> DirectUrl: return DirectUrl( url=path_to_url(source_dir), - info=DirInfo(editable=True), + dir_info=DirInfo(editable=True), ) @@ -62,7 +66,7 @@ def direct_url_from_link( commit_id = vcs_backend.get_revision(source_dir) return DirectUrl( url=url, - info=VcsInfo( + vcs_info=VcsInfo( vcs=vcs_backend.name, commit_id=commit_id, requested_revision=requested_revision, @@ -72,16 +76,17 @@ def direct_url_from_link( elif link.is_existing_dir(): return DirectUrl( url=link.url_without_fragment, - info=DirInfo(), + dir_info=DirInfo(), subdirectory=link.subdirectory_fragment, ) else: - hash = None - hash_name = link.hash_name - if hash_name: - hash = f"{hash_name}={link.hash}" + if link.hash_name: + assert link.hash + hashes = {link.hash_name: link.hash} + else: + hashes = None return DirectUrl( url=link.url_without_fragment, - info=ArchiveInfo(hash=hash), + archive_info=ArchiveInfo(hashes=hashes), subdirectory=link.subdirectory_fragment, ) diff --git a/contrib/python/pip/pip/_internal/utils/filesystem.py b/contrib/python/pip/pip/_internal/utils/filesystem.py index 52eb78f4207..e7c2ffcdef5 100644 --- a/contrib/python/pip/pip/_internal/utils/filesystem.py +++ b/contrib/python/pip/pip/_internal/utils/filesystem.py @@ -172,7 +172,7 @@ def _subdirs_without_generic( predicate under it.""" directories = [] - excluded = set() + excluded: set[Path] = set() for root_str, _, filenames in os.walk(Path(path).resolve()): root = Path(root_str) @@ -180,9 +180,7 @@ def _subdirs_without_generic( # This directory should be excluded, so exclude it and all of its # parent directories. # The last item in root.parents is ".", so we ignore it. - # - # Wrapping this in `list()` is only needed for Python 3.9. - excluded.update(list(root.parents)[:-1]) + excluded.update(root.parents[:-1]) excluded.add(root) directories.append(root) diff --git a/contrib/python/pip/pip/_internal/utils/hashes.py b/contrib/python/pip/pip/_internal/utils/hashes.py index 3d8c125ada3..43f14cf2a3b 100644 --- a/contrib/python/pip/pip/_internal/utils/hashes.py +++ b/contrib/python/pip/pip/_internal/utils/hashes.py @@ -8,9 +8,9 @@ from pip._internal.exceptions import HashMismatch, HashMissing, InstallationErro from pip._internal.utils.misc import read_chunks if TYPE_CHECKING: + from collections.abc import Mapping from hashlib import _Hash - # The recommended hash algo of the moment. Change this whenever the state of # the art changes; it won't hurt backward compatibility. FAVORITE_HASH = "sha256" @@ -104,7 +104,7 @@ class Hashes: with open(path, "rb") as file: return self.check_against_file(file) - def has_one_of(self, hashes: dict[str, str]) -> bool: + def has_one_of(self, hashes: Mapping[str, str]) -> bool: """Return whether any of the given hashes are allowed.""" for hash_name, hex_digest in hashes.items(): if self.is_hash_allowed(hash_name, hex_digest): diff --git a/contrib/python/pip/pip/_internal/utils/misc.py b/contrib/python/pip/pip/_internal/utils/misc.py index 66e521bbc1b..aa2729b7c2e 100644 --- a/contrib/python/pip/pip/_internal/utils/misc.py +++ b/contrib/python/pip/pip/_internal/utils/misc.py @@ -548,7 +548,7 @@ class HiddenText: if type(self) is type(other): # The string being used for redaction doesn't also have to match, # just the raw, original string. - return self.secret == cast(HiddenText, other).secret + return self.secret == other.secret return NotImplemented # Disable hashing, since we have a custom __eq__ and don't need hash-ability diff --git a/contrib/python/pip/pip/_internal/utils/pylock.py b/contrib/python/pip/pip/_internal/utils/pylock.py index 0f77835226d..ff91a75ab8b 100644 --- a/contrib/python/pip/pip/_internal/utils/pylock.py +++ b/contrib/python/pip/pip/_internal/utils/pylock.py @@ -1,5 +1,11 @@ -from collections.abc import Iterable +from __future__ import annotations + +import os +import re +from collections.abc import Iterable, Iterator from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import urljoin, urlsplit from pip._vendor.packaging.pylock import ( Package, @@ -9,13 +15,18 @@ from pip._vendor.packaging.pylock import ( PackageVcs, PackageWheel, Pylock, + is_valid_pylock_path, ) from pip._vendor.packaging.version import Version -from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo +from pip._internal.exceptions import InstallationError from pip._internal.models.link import Link -from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.urls import url_to_path +from pip._internal.utils.compat import tomllib +from pip._internal.utils.urls import path_to_url, url_to_path + +if TYPE_CHECKING: + from pip._internal.network.session import PipSession + from pip._internal.req.req_install import InstallRequirement def _pylock_package_from_install_requirement( @@ -32,16 +43,16 @@ def _pylock_package_from_install_requirement( package_sdist = None package_wheels = None if ireq.is_direct: - if isinstance(download_info.info, VcsInfo): + if download_info.vcs_info: package_vcs = PackageVcs( - type=download_info.info.vcs, + type=download_info.vcs_info.vcs, url=download_info.url, path=None, - requested_revision=download_info.info.requested_revision, - commit_id=download_info.info.commit_id, + requested_revision=download_info.vcs_info.requested_revision, + commit_id=download_info.vcs_info.commit_id, subdirectory=download_info.subdirectory, ) - elif isinstance(download_info.info, DirInfo): + elif download_info.dir_info: package_directory = PackageDirectory( path=( Path(url_to_path(download_info.url)) @@ -50,17 +61,19 @@ def _pylock_package_from_install_requirement( .as_posix() ), editable=( - download_info.info.editable if download_info.info.editable else None + download_info.dir_info.editable + if download_info.dir_info.editable + else None ), subdirectory=download_info.subdirectory, ) - elif isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: + elif download_info.archive_info: + if not download_info.archive_info.hashes: raise NotImplementedError() package_archive = PackageArchive( url=download_info.url, path=None, - hashes=download_info.info.hashes, + hashes=download_info.archive_info.hashes, subdirectory=download_info.subdirectory, ) else: @@ -68,8 +81,8 @@ def _pylock_package_from_install_requirement( raise NotImplementedError() else: package_version = dist.version - if isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: + if download_info.archive_info: + if not download_info.archive_info.hashes: raise NotImplementedError() link = Link(download_info.url) if link.is_wheel: @@ -77,14 +90,14 @@ def _pylock_package_from_install_requirement( PackageWheel( name=link.filename, url=download_info.url, - hashes=download_info.info.hashes, + hashes=download_info.archive_info.hashes, ) ] else: package_sdist = PackageSdist( name=link.filename, url=download_info.url, - hashes=download_info.info.hashes, + hashes=download_info.archive_info.hashes, ) else: # should never happen @@ -114,3 +127,157 @@ def pylock_from_install_requirements( key=lambda p: p.name, ), ) + + +_SCHEME_RE = re.compile("^(http|https|file)://", re.IGNORECASE) + + +def _is_url(s: str) -> bool: + return bool(_SCHEME_RE.match(s)) + + +def is_valid_pylock_filename(filename: str) -> bool: + if _is_url(filename): + path = Path(urlsplit(filename).path.rpartition("/")[-1]) + else: + path = Path(filename) + return is_valid_pylock_path(path) + + +def _package_dist_url( + pylock_path_or_url: str, path: str | None, url: str | None +) -> str: + """Compute an url from a Pylock package path and url. + + Give priority to path over url. If path is relative, + compute an url using the pylock file location as base. + """ + if path is not None: + if not os.path.isabs(path): + # relative path, join to pylock location + if _is_url(pylock_path_or_url): + return urljoin(pylock_path_or_url, path) + else: + return path_to_url( + os.path.join(os.path.dirname(pylock_path_or_url), path) + ) + else: + # absolute path, reject if pylock comes from a URL + if _is_url(pylock_path_or_url): + raise InstallationError( + f"Absolute paths are not supported in pylock files obtained " + f"from a URL: {path!r} in {pylock_path_or_url!r}" + ) + return path_to_url(path) + else: + assert url is not None # guaranteed by packaging.pylock validation + return url + + +def package_vcs_requirement_url( + pylock_path_or_url: str, package_vcs: PackageVcs +) -> str: + dist_url = _package_dist_url(pylock_path_or_url, package_vcs.path, package_vcs.url) + url = f"{package_vcs.type}+{dist_url}@{package_vcs.commit_id}" + if package_vcs.subdirectory: + if "#" in url: + raise InstallationError( + f"Package URL {url!r} cannot contain fragments in combination " + f"with subdirectory field (in {pylock_path_or_url!r})" + ) + url += "#subdirectory=" + package_vcs.subdirectory + return url + + +def package_archive_requirement_url( + pylock_path_or_url: str, package_archive: PackageArchive +) -> str: + url = _package_dist_url( + pylock_path_or_url, package_archive.path, package_archive.url + ) + if package_archive.subdirectory: + if "#" in url: + raise InstallationError( + f"Package URL {url!r} cannot contain fragments in combination " + f"with subdirectory field (in {pylock_path_or_url!r})" + ) + url += "#subdirectory=" + package_archive.subdirectory + return url + + +def package_directory_requirement_url( + pylock_path_or_url: str, package_directory: PackageDirectory +) -> str: + if _is_url(pylock_path_or_url) and not pylock_path_or_url.startswith("file://"): + raise InstallationError( + f"Directory entries are not supported in remote pylock.toml " + f"{pylock_path_or_url!r}" + ) + url = _package_dist_url(pylock_path_or_url, package_directory.path, None) + assert url.startswith("file://") + if not url.endswith("/"): + url += "/" + if package_directory.subdirectory: + url += package_directory.subdirectory + if not url.endswith("/"): + url += "/" + return url + + +def package_sdist_requirement_url( + pylock_path_or_url: str, package_sdist: PackageSdist +) -> str: + return _package_dist_url(pylock_path_or_url, package_sdist.path, package_sdist.url) + + +def package_wheel_requirement_url( + pylock_path_or_url: str, package_wheel: PackageWheel +) -> str: + return _package_dist_url(pylock_path_or_url, package_wheel.path, package_wheel.url) + + +def _get_pylock_path_or_url_content(path_or_url: str, session: PipSession) -> str: + # TODO: refactor - this is similar to req_file.get_file_content + scheme = urlsplit(path_or_url).scheme + # Pip has special support for file:// URLs (LocalFSAdapter). + if scheme in ["http", "https", "file"]: + # Delay importing heavy network modules until absolutely necessary. + from pip._internal.network.utils import raise_for_status + + resp = session.get(path_or_url) + raise_for_status(resp) + return resp.text + + # Assume this is a bare path. + return Path(path_or_url).read_text(encoding="utf-8") + + +def select_from_pylock_path_or_url( + pylock_path_or_url: str, + session: PipSession, +) -> Iterator[ + tuple[ + Package, + PackageVcs | PackageDirectory | PackageArchive | PackageWheel | PackageSdist, + ] +]: + try: + pylock_content = _get_pylock_path_or_url_content(pylock_path_or_url, session) + except Exception as exc: + raise InstallationError( + f"Error reading pylock file {pylock_path_or_url!r}: {exc}" + ) from exc + + try: + lock = Pylock.from_dict(tomllib.loads(pylock_content)) + except Exception as exc: + raise InstallationError( + f"Invalid pylock file {pylock_path_or_url!r}: {exc}" + ) from exc + + try: + yield from lock.select() + except Exception as exc: + raise InstallationError( + f"Cannot select requirements from pylock file {pylock_path_or_url!r}: {exc}" + ) from exc diff --git a/contrib/python/pip/pip/_internal/utils/unpacking.py b/contrib/python/pip/pip/_internal/utils/unpacking.py index b3f52e85e68..879b40c37ec 100644 --- a/contrib/python/pip/pip/_internal/utils/unpacking.py +++ b/contrib/python/pip/pip/_internal/utils/unpacking.py @@ -336,27 +336,46 @@ def unpack_file( location: str, content_type: str | None = None, ) -> None: + """Unpack ``filename`` into ``location``. + + Archive format is chosen in order of decreasing reliability: + ``content_type``, then filename extension, then magic signature + (unambiguous matches only). + """ filename = os.path.realpath(filename) - if ( - content_type == "application/zip" - or filename.lower().endswith(ZIP_EXTENSIONS) - or zipfile.is_zipfile(filename) - ): - unzip_file(filename, location, flatten=not filename.endswith(".whl")) - elif ( - content_type == "application/x-gzip" - or tarfile.is_tarfile(filename) - or filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS) - ): + zip_flatten = not filename.endswith(".whl") + + def _unzip() -> None: + unzip_file(filename, location, flatten=zip_flatten) + + def _untar() -> None: untar_file(filename, location) - else: - # FIXME: handle? - # FIXME: magic signatures? - logger.critical( - "Cannot unpack file %s (downloaded from %s, content-type: %s); " - "cannot detect archive format", - filename, - location, - content_type, - ) - raise InstallationError(f"Cannot determine archive format of {location}") + + if content_type == "application/zip": + return _unzip() + if content_type == "application/x-gzip": + return _untar() + + if filename.lower().endswith(ZIP_EXTENSIONS): + return _unzip() + if filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS): + return _untar() + + # avoid ambiguous case where both signature checks return True + is_zipfile = zipfile.is_zipfile(filename) + is_tarfile = tarfile.is_tarfile(filename) + if is_zipfile and not is_tarfile: + return _unzip() + if is_tarfile and not is_zipfile: + return _untar() + if is_zipfile and is_tarfile: + logger.error("Ambiguous file signature in %s.", filename) + + logger.critical( + "Cannot unpack file %s (downloaded from %s, content-type: %s); " + "cannot detect archive format", + filename, + location, + content_type, + ) + raise InstallationError(f"Cannot determine archive format of {location}") diff --git a/contrib/python/pip/pip/_vendor/README.rst b/contrib/python/pip/pip/_vendor/README.rst index a925e8cc6aa..7429a52f46f 100644 --- a/contrib/python/pip/pip/_vendor/README.rst +++ b/contrib/python/pip/pip/_vendor/README.rst @@ -118,12 +118,10 @@ Tool configuration is done via ``pyproject.toml``. To update the vendored library versions, we have a session defined in ``nox``. The command to upgrade everything is:: - nox -s vendoring -- --upgrade-all --skip urllib3 --skip setuptools + nox -s vendoring -- --upgrade-all --skip setuptools -At the time of writing (April 2025) we do not upgrade ``urllib3`` because the -next version is a major upgrade and will be handled as an independent PR. We also -do not upgrade ``setuptools``, because we only rely on ``pkg_resources``, and -tracking every ``setuptools`` change is unnecessary for our needs. +We do not upgrade ``setuptools``, because we only rely on ``pkg_resources``, +and tracking every ``setuptools`` change is unnecessary for our needs. Managing Local Patches diff --git a/contrib/python/pip/pip/_vendor/certifi/__init__.py b/contrib/python/pip/pip/_vendor/certifi/__init__.py index 090fd58487b..16c0c7c268f 100644 --- a/contrib/python/pip/pip/_vendor/certifi/__init__.py +++ b/contrib/python/pip/pip/_vendor/certifi/__init__.py @@ -1,4 +1,4 @@ from .core import contents, where __all__ = ["contents", "where"] -__version__ = "2026.01.04" +__version__ = "2026.02.25" diff --git a/contrib/python/pip/pip/_vendor/certifi/cacert.pem b/contrib/python/pip/pip/_vendor/certifi/cacert.pem index 132db0df1b8..5ec1afe02d4 100644 --- a/contrib/python/pip/pip/_vendor/certifi/cacert.pem +++ b/contrib/python/pip/pip/_vendor/certifi/cacert.pem @@ -4429,7 +4429,7 @@ YmH5LVerVrkR3SW+ak5KGoJr3M/TvEqzPNcum9v4KGm8ay3sMaE641c= # Issuer: CN=OISTE Server Root RSA G1 O=OISTE Foundation # Subject: CN=OISTE Server Root RSA G1 O=OISTE Foundation -# Label: " OISTE Server Root RSA G1" +# Label: "OISTE Server Root RSA G1" # Serial: 113845518112613905024960613408179309848 # MD5 Fingerprint: 23:a7:9e:d4:70:b8:b9:14:57:41:8a:7e:44:59:e2:68 # SHA1 Fingerprint: f7:00:34:25:94:88:68:31:e4:34:87:3f:70:fe:86:b3:86:9f:f0:6e @@ -4466,3 +4466,29 @@ J8tRd/iWkx7P8nd9H0aTolkelUTFLXVksNb54Dxp6gS1HAviRkRNQzuXSXERvSS2 wq1yVAb+axj5d9spLFKebXd7Yv0PTY6YMjAwcRLWJTXjn/hvnLXrahut6hDTlhZy BiElxky8j3C7DOReIoMt0r7+hVu05L0= -----END CERTIFICATE----- + +# Issuer: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. +# Subject: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. +# Label: "e-Szigno TLS Root CA 2023" +# Serial: 71934828665710877219916191754 +# MD5 Fingerprint: 6a:e9:99:74:a5:da:5e:f1:d9:2e:f2:c8:d1:86:8b:71 +# SHA1 Fingerprint: 6f:9a:d5:d5:df:e8:2c:eb:be:37:07:ee:4f:4f:52:58:29:41:d1:fe +# SHA256 Fingerprint: b4:91:41:50:2d:00:66:3d:74:0f:2e:7e:c3:40:c5:28:00:96:26:66:12:1a:36:d0:9c:f7:dd:2b:90:38:4f:b4 +-----BEGIN CERTIFICATE----- +MIICzzCCAjGgAwIBAgINAOhvGHvWOWuYSkmYCjAKBggqhkjOPQQDBDB1MQswCQYD +VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 +ZC4xFzAVBgNVBGEMDlZBVEhVLTIzNTg0NDk3MSIwIAYDVQQDDBllLVN6aWdubyBU +TFMgUm9vdCBDQSAyMDIzMB4XDTIzMDcxNzE0MDAwMFoXDTM4MDcxNzE0MDAwMFow +dTELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRYwFAYDVQQKDA1NaWNy +b3NlYyBMdGQuMRcwFQYDVQRhDA5WQVRIVS0yMzU4NDQ5NzEiMCAGA1UEAwwZZS1T +emlnbm8gVExTIFJvb3QgQ0EgMjAyMzCBmzAQBgcqhkjOPQIBBgUrgQQAIwOBhgAE +AGgP36J8PKp0iGEKjcJMpQEiFNT3YHdCnAo4YKGMZz6zY+n6kbCLS+Y53wLCMAFS +AL/fjO1ZrTJlqwlZULUZwmgcAOAFX9pQJhzDrAQixTpN7+lXWDajwRlTEArRzT/v +SzUaQ49CE0y5LBqcvjC2xN7cS53kpDzLLtmt3999Cd8ukv+ho2MwYTAPBgNVHRMB +Af8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUWYQCYlpGePVd3I8K +ECgj3NXW+0UwHwYDVR0jBBgwFoAUWYQCYlpGePVd3I8KECgj3NXW+0UwCgYIKoZI +zj0EAwQDgYsAMIGHAkIBLdqu9S54tma4n7Zwf2Z0z+yOfP7AAXmazlIC58PRDHpt +y7Ve7hekm9sEdu4pKeiv+62sUvTXK9Z3hBC9xdIoaDQCQTV2WnXzkoYI9bIeCvZl +C9p2x1L/Cx6AcCIwwzPbGO2E14vs7dOoY4G1VnxHx1YwlGhza9IuqbnZLBwpvQy6 +uWWL +-----END CERTIFICATE----- diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/LICENSE.txt b/contrib/python/pip/pip/_vendor/dependency_groups/LICENSE.txt deleted file mode 100644 index b9723b85ed7..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/LICENSE.txt +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2024-present Stephen Rosen <[email protected]> - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/__init__.py b/contrib/python/pip/pip/_vendor/dependency_groups/__init__.py deleted file mode 100644 index 9fec2029949..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._implementation import ( - CyclicDependencyError, - DependencyGroupInclude, - DependencyGroupResolver, - resolve, -) - -__all__ = ( - "CyclicDependencyError", - "DependencyGroupInclude", - "DependencyGroupResolver", - "resolve", -) diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/__main__.py b/contrib/python/pip/pip/_vendor/dependency_groups/__main__.py deleted file mode 100644 index 48ebb0d41cf..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/__main__.py +++ /dev/null @@ -1,65 +0,0 @@ -import argparse -import sys - -from ._implementation import resolve -from ._toml_compat import tomllib - - -def main() -> None: - if tomllib is None: - print( - "Usage error: dependency-groups CLI requires tomli or Python 3.11+", - file=sys.stderr, - ) - raise SystemExit(2) - - parser = argparse.ArgumentParser( - description=( - "A dependency-groups CLI. Prints out a resolved group, newline-delimited." - ) - ) - parser.add_argument( - "GROUP_NAME", nargs="*", help="The dependency group(s) to resolve." - ) - parser.add_argument( - "-f", - "--pyproject-file", - default="pyproject.toml", - help="The pyproject.toml file. Defaults to trying in the current directory.", - ) - parser.add_argument( - "-o", - "--output", - help="An output file. Defaults to stdout.", - ) - parser.add_argument( - "-l", - "--list", - action="store_true", - help="List the available dependency groups", - ) - args = parser.parse_args() - - with open(args.pyproject_file, "rb") as fp: - pyproject = tomllib.load(fp) - - dependency_groups_raw = pyproject.get("dependency-groups", {}) - - if args.list: - print(*dependency_groups_raw.keys()) - return - if not args.GROUP_NAME: - print("A GROUP_NAME is required", file=sys.stderr) - raise SystemExit(3) - - content = "\n".join(resolve(dependency_groups_raw, *args.GROUP_NAME)) - - if args.output is None or args.output == "-": - print(content) - else: - with open(args.output, "w", encoding="utf-8") as fp: - print(content, file=fp) - - -if __name__ == "__main__": - main() diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/_implementation.py b/contrib/python/pip/pip/_vendor/dependency_groups/_implementation.py deleted file mode 100644 index 64e314a6328..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/_implementation.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import annotations - -import dataclasses -import re -from collections.abc import Mapping - -from pip._vendor.packaging.requirements import Requirement - - -def _normalize_name(name: str) -> str: - return re.sub(r"[-_.]+", "-", name).lower() - - -def _normalize_group_names( - dependency_groups: Mapping[str, str | Mapping[str, str]], -) -> Mapping[str, str | Mapping[str, str]]: - original_names: dict[str, list[str]] = {} - normalized_groups = {} - - for group_name, value in dependency_groups.items(): - normed_group_name = _normalize_name(group_name) - original_names.setdefault(normed_group_name, []).append(group_name) - normalized_groups[normed_group_name] = value - - errors = [] - for normed_name, names in original_names.items(): - if len(names) > 1: - errors.append(f"{normed_name} ({', '.join(names)})") - if errors: - raise ValueError(f"Duplicate dependency group names: {', '.join(errors)}") - - return normalized_groups - - -class DependencyGroupInclude: - include_group: str - - -class CyclicDependencyError(ValueError): - """ - An error representing the detection of a cycle. - """ - - def __init__(self, requested_group: str, group: str, include_group: str) -> None: - self.requested_group = requested_group - self.group = group - self.include_group = include_group - - if include_group == group: - reason = f"{group} includes itself" - else: - reason = f"{include_group} -> {group}, {group} -> {include_group}" - super().__init__( - "Cyclic dependency group include while resolving " - f"{requested_group}: {reason}" - ) - - -class DependencyGroupResolver: - """ - A resolver for Dependency Group data. - - This class handles caching, name normalization, cycle detection, and other - parsing requirements. There are only two public methods for exploring the data: - ``lookup()`` and ``resolve()``. - - :param dependency_groups: A mapping, as provided via pyproject - ``[dependency-groups]``. - """ - - def __init__( - self, - dependency_groups: Mapping[str, str | Mapping[str, str]], - ) -> None: - if not isinstance(dependency_groups, Mapping): - raise TypeError("Dependency Groups table is not a mapping") - self.dependency_groups = _normalize_group_names(dependency_groups) - # a map of group names to parsed data - self._parsed_groups: dict[ - str, tuple[Requirement | DependencyGroupInclude, ...] - ] = {} - # a map of group names to their ancestors, used for cycle detection - self._include_graph_ancestors: dict[str, tuple[str, ...]] = {} - # a cache of completed resolutions to Requirement lists - self._resolve_cache: dict[str, tuple[Requirement, ...]] = {} - - def lookup(self, group: str) -> tuple[Requirement | DependencyGroupInclude, ...]: - """ - Lookup a group name, returning the parsed dependency data for that group. - This will not resolve includes. - - :param group: the name of the group to lookup - - :raises ValueError: if the data does not appear to be valid dependency group - data - :raises TypeError: if the data is not a string - :raises LookupError: if group name is absent - :raises packaging.requirements.InvalidRequirement: if a specifier is not valid - """ - if not isinstance(group, str): - raise TypeError("Dependency group name is not a str") - group = _normalize_name(group) - return self._parse_group(group) - - def resolve(self, group: str) -> tuple[Requirement, ...]: - """ - Resolve a dependency group to a list of requirements. - - :param group: the name of the group to resolve - - :raises TypeError: if the inputs appear to be the wrong types - :raises ValueError: if the data does not appear to be valid dependency group - data - :raises LookupError: if group name is absent - :raises packaging.requirements.InvalidRequirement: if a specifier is not valid - """ - if not isinstance(group, str): - raise TypeError("Dependency group name is not a str") - group = _normalize_name(group) - return self._resolve(group, group) - - def _parse_group( - self, group: str - ) -> tuple[Requirement | DependencyGroupInclude, ...]: - # short circuit -- never do the work twice - if group in self._parsed_groups: - return self._parsed_groups[group] - - if group not in self.dependency_groups: - raise LookupError(f"Dependency group '{group}' not found") - - raw_group = self.dependency_groups[group] - if not isinstance(raw_group, list): - raise TypeError(f"Dependency group '{group}' is not a list") - - elements: list[Requirement | DependencyGroupInclude] = [] - for item in raw_group: - if isinstance(item, str): - # packaging.requirements.Requirement parsing ensures that this is a - # valid PEP 508 Dependency Specifier - # raises InvalidRequirement on failure - elements.append(Requirement(item)) - elif isinstance(item, dict): - if tuple(item.keys()) != ("include-group",): - raise ValueError(f"Invalid dependency group item: {item}") - - include_group = next(iter(item.values())) - elements.append(DependencyGroupInclude(include_group=include_group)) - else: - raise ValueError(f"Invalid dependency group item: {item}") - - self._parsed_groups[group] = tuple(elements) - return self._parsed_groups[group] - - def _resolve(self, group: str, requested_group: str) -> tuple[Requirement, ...]: - """ - This is a helper for cached resolution to strings. - - :param group: The name of the group to resolve. - :param requested_group: The group which was used in the original, user-facing - request. - """ - if group in self._resolve_cache: - return self._resolve_cache[group] - - parsed = self._parse_group(group) - - resolved_group = [] - for item in parsed: - if isinstance(item, Requirement): - resolved_group.append(item) - elif isinstance(item, DependencyGroupInclude): - include_group = _normalize_name(item.include_group) - if include_group in self._include_graph_ancestors.get(group, ()): - raise CyclicDependencyError( - requested_group, group, item.include_group - ) - self._include_graph_ancestors[include_group] = ( - *self._include_graph_ancestors.get(group, ()), - group, - ) - resolved_group.extend(self._resolve(include_group, requested_group)) - else: # unreachable - raise NotImplementedError( - f"Invalid dependency group item after parse: {item}" - ) - - self._resolve_cache[group] = tuple(resolved_group) - return self._resolve_cache[group] - - -def resolve( - dependency_groups: Mapping[str, str | Mapping[str, str]], /, *groups: str -) -> tuple[str, ...]: - """ - Resolve a dependency group to a tuple of requirements, as strings. - - :param dependency_groups: the parsed contents of the ``[dependency-groups]`` table - from ``pyproject.toml`` - :param groups: the name of the group(s) to resolve - - :raises TypeError: if the inputs appear to be the wrong types - :raises ValueError: if the data does not appear to be valid dependency group data - :raises LookupError: if group name is absent - :raises packaging.requirements.InvalidRequirement: if a specifier is not valid - """ - resolver = DependencyGroupResolver(dependency_groups) - return tuple(str(r) for group in groups for r in resolver.resolve(group)) diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/_lint_dependency_groups.py b/contrib/python/pip/pip/_vendor/dependency_groups/_lint_dependency_groups.py deleted file mode 100644 index 09454bdc280..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/_lint_dependency_groups.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import argparse -import sys - -from ._implementation import DependencyGroupResolver -from ._toml_compat import tomllib - - -def main(*, argv: list[str] | None = None) -> None: - if tomllib is None: - print( - "Usage error: dependency-groups CLI requires tomli or Python 3.11+", - file=sys.stderr, - ) - raise SystemExit(2) - - parser = argparse.ArgumentParser( - description=( - "Lint Dependency Groups for validity. " - "This will eagerly load and check all of your Dependency Groups." - ) - ) - parser.add_argument( - "-f", - "--pyproject-file", - default="pyproject.toml", - help="The pyproject.toml file. Defaults to trying in the current directory.", - ) - args = parser.parse_args(argv if argv is not None else sys.argv[1:]) - - with open(args.pyproject_file, "rb") as fp: - pyproject = tomllib.load(fp) - dependency_groups_raw = pyproject.get("dependency-groups", {}) - - errors: list[str] = [] - try: - resolver = DependencyGroupResolver(dependency_groups_raw) - except (ValueError, TypeError) as e: - errors.append(f"{type(e).__name__}: {e}") - else: - for groupname in resolver.dependency_groups: - try: - resolver.resolve(groupname) - except (LookupError, ValueError, TypeError) as e: - errors.append(f"{type(e).__name__}: {e}") - - if errors: - print("errors encountered while examining dependency groups:") - for msg in errors: - print(f" {msg}") - sys.exit(1) - else: - print("ok") - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/_pip_wrapper.py b/contrib/python/pip/pip/_vendor/dependency_groups/_pip_wrapper.py deleted file mode 100644 index f86d8961ba2..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/_pip_wrapper.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import argparse -import subprocess -import sys - -from ._implementation import DependencyGroupResolver -from ._toml_compat import tomllib - - -def _invoke_pip(deps: list[str]) -> None: - subprocess.check_call([sys.executable, "-m", "pip", "install", *deps]) - - -def main(*, argv: list[str] | None = None) -> None: - if tomllib is None: - print( - "Usage error: dependency-groups CLI requires tomli or Python 3.11+", - file=sys.stderr, - ) - raise SystemExit(2) - - parser = argparse.ArgumentParser(description="Install Dependency Groups.") - parser.add_argument( - "DEPENDENCY_GROUP", nargs="+", help="The dependency groups to install." - ) - parser.add_argument( - "-f", - "--pyproject-file", - default="pyproject.toml", - help="The pyproject.toml file. Defaults to trying in the current directory.", - ) - args = parser.parse_args(argv if argv is not None else sys.argv[1:]) - - with open(args.pyproject_file, "rb") as fp: - pyproject = tomllib.load(fp) - dependency_groups_raw = pyproject.get("dependency-groups", {}) - - errors: list[str] = [] - resolved: list[str] = [] - try: - resolver = DependencyGroupResolver(dependency_groups_raw) - except (ValueError, TypeError) as e: - errors.append(f"{type(e).__name__}: {e}") - else: - for groupname in args.DEPENDENCY_GROUP: - try: - resolved.extend(str(r) for r in resolver.resolve(groupname)) - except (LookupError, ValueError, TypeError) as e: - errors.append(f"{type(e).__name__}: {e}") - - if errors: - print("errors encountered while examining dependency groups:") - for msg in errors: - print(f" {msg}") - sys.exit(1) - - _invoke_pip(resolved) - - -if __name__ == "__main__": - main() diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/_toml_compat.py b/contrib/python/pip/pip/_vendor/dependency_groups/_toml_compat.py deleted file mode 100644 index 8d6f921c2a5..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/_toml_compat.py +++ /dev/null @@ -1,9 +0,0 @@ -try: - import tomllib -except ImportError: - try: - from pip._vendor import tomli as tomllib # type: ignore[no-redef, unused-ignore] - except ModuleNotFoundError: # pragma: no cover - tomllib = None # type: ignore[assignment, unused-ignore] - -__all__ = ("tomllib",) diff --git a/contrib/python/pip/pip/_vendor/dependency_groups/py.typed b/contrib/python/pip/pip/_vendor/dependency_groups/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/pip/pip/_vendor/dependency_groups/py.typed +++ /dev/null diff --git a/contrib/python/pip/pip/_vendor/packaging/__init__.py b/contrib/python/pip/pip/_vendor/packaging/__init__.py index 21695a74b51..a6bdf59cd71 100644 --- a/contrib/python/pip/pip/_vendor/packaging/__init__.py +++ b/contrib/python/pip/pip/_vendor/packaging/__init__.py @@ -6,7 +6,7 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "26.0" +__version__ = "26.2" __author__ = "Donald Stufft and individual contributors" __email__ = "[email protected]" diff --git a/contrib/python/pip/pip/_vendor/packaging/_parser.py b/contrib/python/pip/pip/_vendor/packaging/_parser.py index f6c1f5cd226..d320269e153 100644 --- a/contrib/python/pip/pip/_vendor/packaging/_parser.py +++ b/contrib/python/pip/pip/_vendor/packaging/_parser.py @@ -27,6 +27,34 @@ class Node: def serialize(self) -> str: raise NotImplementedError + def __getstate__(self) -> str: + # Return just the value string for compactness and stability. + return self.value + + def _restore_value(self, value: object) -> None: + if not isinstance(value, str): + raise TypeError( + f"Cannot restore {self.__class__.__name__} value from {value!r}" + ) + self.value = value + + def __setstate__(self, state: object) -> None: + if isinstance(state, str): + # New format (26.2+): just the value string. + self._restore_value(state) + return + if isinstance(state, tuple) and len(state) == 2: + # Old format (packaging <= 26.0, __slots__): (None, {slot: value}). + _, slot_dict = state + if isinstance(slot_dict, dict) and "value" in slot_dict: + self._restore_value(slot_dict["value"]) + return + if isinstance(state, dict) and "value" in state: + # Old format (packaging <= 25.0, no __slots__): plain __dict__. + self._restore_value(state["value"]) + return + raise TypeError(f"Cannot restore {self.__class__.__name__} from {state!r}") + class Variable(Node): __slots__ = () diff --git a/contrib/python/pip/pip/_vendor/packaging/_structures.py b/contrib/python/pip/pip/_vendor/packaging/_structures.py index 225e2eee012..4306784d9a2 100644 --- a/contrib/python/pip/pip/_vendor/packaging/_structures.py +++ b/contrib/python/pip/pip/_vendor/packaging/_structures.py @@ -2,68 +2,32 @@ # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. -import typing +"""Backward-compatibility shim for unpickling Version objects serialized before +packaging 26.1. + +Old pickles reference ``packaging._structures.InfinityType`` and +``packaging._structures.NegativeInfinityType``. This module provides minimal +stand-in classes so that ``pickle.loads()`` can resolve those references. +The deserialized objects are not used for comparisons — ``Version.__setstate__`` +discards the stale ``_key`` cache and recomputes it from the core version fields. +""" + +from __future__ import annotations class InfinityType: - __slots__ = () + """Stand-in for the removed ``InfinityType`` used in old comparison keys.""" def __repr__(self) -> str: return "Infinity" - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return False - - def __le__(self, other: object) -> bool: - return False - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return True - - def __ge__(self, other: object) -> bool: - return True - - def __neg__(self: object) -> "NegativeInfinityType": - return NegativeInfinity - - -Infinity = InfinityType() - class NegativeInfinityType: - __slots__ = () + """Stand-in for the removed ``NegativeInfinityType`` used in old comparison keys.""" def __repr__(self) -> str: return "-Infinity" - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return True - - def __le__(self, other: object) -> bool: - return True - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return False - - def __ge__(self, other: object) -> bool: - return False - - def __neg__(self: object) -> InfinityType: - return Infinity - +Infinity = InfinityType() NegativeInfinity = NegativeInfinityType() diff --git a/contrib/python/pip/pip/_vendor/packaging/_tokenizer.py b/contrib/python/pip/pip/_vendor/packaging/_tokenizer.py index e6d20dd3f56..5ab891ccb8e 100644 --- a/contrib/python/pip/pip/_vendor/packaging/_tokenizer.py +++ b/contrib/python/pip/pip/_vendor/packaging/_tokenizer.py @@ -75,7 +75,7 @@ DEFAULT_RULES: dict[str, re.Pattern[str]] = { re.VERBOSE, ), "SPECIFIER": re.compile( - Specifier._operator_regex_str + Specifier._version_regex_str, + Specifier._specifier_regex_str, re.VERBOSE | re.IGNORECASE, ), "AT": re.compile(r"\@"), diff --git a/contrib/python/pip/pip/_vendor/packaging/dependency_groups.py b/contrib/python/pip/pip/_vendor/packaging/dependency_groups.py new file mode 100644 index 00000000000..413e5cb4b40 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/packaging/dependency_groups.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence + +from .errors import _ErrorCollector +from .requirements import Requirement + +__all__ = [ + "CyclicDependencyGroup", + "DependencyGroupInclude", + "DependencyGroupResolver", + "DuplicateGroupNames", + "InvalidDependencyGroupObject", + "resolve_dependency_groups", +] + + +def __dir__() -> list[str]: + return __all__ + + +# ----------- +# Error Types +# ----------- + + +class DuplicateGroupNames(ValueError): + """ + The same dependency groups were defined twice, with different non-normalized names. + """ + + +class CyclicDependencyGroup(ValueError): + """ + The dependency group includes form a cycle. + """ + + def __init__(self, requested_group: str, group: str, include_group: str) -> None: + self.requested_group = requested_group + self.group = group + self.include_group = include_group + + if include_group == group: + reason = f"{group} includes itself" + else: + reason = f"{include_group} -> {group}, {group} -> {include_group}" + super().__init__( + "Cyclic dependency group include while resolving " + f"{requested_group}: {reason}" + ) + + +# in the PEP 735 spec, the tables in dependency group lists were described as +# "Dependency Object Specifiers", but the only defined type of object was a +# "Dependency Group Include" -- hence the naming of this error as "Object" +class InvalidDependencyGroupObject(ValueError): + """ + A member of a dependency group was identified as a dict, but was not in a valid + format. + """ + + +# ------------------------ +# Object Model & Interface +# ------------------------ + + +class DependencyGroupInclude: + __slots__ = ("include_group",) + + def __init__(self, include_group: str) -> None: + """ + Initialize a DependencyGroupInclude. + + :param include_group: The name of the group referred to by this include. + """ + self.include_group = include_group + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.include_group!r})" + + +class DependencyGroupResolver: + """ + A resolver for Dependency Group data. + + This class handles caching, name normalization, cycle detection, and other + parsing requirements. There are only two public methods for exploring the data: + ``lookup()`` and ``resolve()``. + + :param dependency_groups: A mapping, as provided via pyproject + ``[dependency-groups]``. + """ + + def __init__( + self, + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], + ) -> None: + errors = _ErrorCollector() + + self.dependency_groups = _normalize_group_names(dependency_groups, errors) + + # a map of group names to parsed data + self._parsed_groups: dict[ + str, tuple[Requirement | DependencyGroupInclude, ...] + ] = {} + # a map of group names to their ancestors, used for cycle detection + self._include_graph_ancestors: dict[str, tuple[str, ...]] = {} + # a cache of completed resolutions to Requirement lists + self._resolve_cache: dict[str, tuple[Requirement, ...]] = {} + + errors.finalize("[dependency-groups] data was invalid") + + def lookup(self, group: str) -> tuple[Requirement | DependencyGroupInclude, ...]: + """ + Lookup a group name, returning the parsed dependency data for that group. + This will not resolve includes. + + :param group: the name of the group to lookup + """ + group = _normalize_name(group) + + with _ErrorCollector().on_exit( + f"[dependency-groups] data for {group!r} was malformed" + ) as errors: + return self._parse_group(group, errors) + + def resolve(self, group: str) -> tuple[Requirement, ...]: + """ + Resolve a dependency group to a list of requirements. + + :param group: the name of the group to resolve + """ + group = _normalize_name(group) + + with _ErrorCollector().on_exit( + f"[dependency-groups] data for {group!r} was malformed" + ) as errors: + return self._resolve(group, group, errors) + + def _resolve( + self, group: str, requested_group: str, errors: _ErrorCollector + ) -> tuple[Requirement, ...]: + """ + This is a helper for cached resolution to strings. It preserves the name of the + group which the user initially requested in order to present a clearer error in + the event that a cycle is detected. + + :param group: The normalized name of the group to resolve. + :param requested_group: The group which was used in the original, user-facing + request. + """ + if group in self._resolve_cache: + return self._resolve_cache[group] + + parsed = self._parse_group(group, errors) + + resolved_group = [] + + for item in parsed: + if isinstance(item, Requirement): + resolved_group.append(item) + elif isinstance(item, DependencyGroupInclude): + include_group = _normalize_name(item.include_group) + + # if a group is cyclic, record the error + # otherwise, follow the include_group reference + # + # this allows us to examine all includes in a group, even in the + # presence of errors + if include_group in self._include_graph_ancestors.get(group, ()): + errors.error( + CyclicDependencyGroup( + requested_group, group, item.include_group + ) + ) + else: + self._include_graph_ancestors[include_group] = ( + *self._include_graph_ancestors.get(group, ()), + group, + ) + resolved_group.extend( + self._resolve(include_group, requested_group, errors) + ) + else: # pragma: no cover + raise NotImplementedError( + f"Invalid dependency group item after parse: {item}" + ) + + # in the event that errors were detected, present the group as empty and do not + # cache the result + # this ensures that repeated access to a cyclic group will raise multiple errors + if errors.errors: + return () + + self._resolve_cache[group] = tuple(resolved_group) + return self._resolve_cache[group] + + def _parse_group( + self, group: str, errors: _ErrorCollector + ) -> tuple[Requirement | DependencyGroupInclude, ...]: + # short circuit -- never do the work twice + if group in self._parsed_groups: + return self._parsed_groups[group] + + if group not in self.dependency_groups: + errors.error(LookupError(f"Dependency group '{group}' not found")) + return () + + raw_group = self.dependency_groups[group] + if isinstance(raw_group, str): + errors.error( + TypeError( + f"Dependency group {group!r} contained a string rather than a list." + ) + ) + return () + + if not isinstance(raw_group, Sequence): + errors.error( + TypeError(f"Dependency group {group!r} is not a sequence type.") + ) + return () + + elements: list[Requirement | DependencyGroupInclude] = [] + for item in raw_group: + if isinstance(item, str): + # packaging.requirements.Requirement parsing ensures that this is a + # valid PEP 508 Dependency Specifier + # raises InvalidRequirement on failure + elements.append(Requirement(item)) + elif isinstance(item, Mapping): + if tuple(item.keys()) != ("include-group",): + errors.error( + InvalidDependencyGroupObject( + f"Invalid dependency group item: {item!r}" + ) + ) + else: + include_group = item["include-group"] + elements.append(DependencyGroupInclude(include_group=include_group)) + else: + errors.error(TypeError(f"Invalid dependency group item: {item!r}")) + + self._parsed_groups[group] = tuple(elements) + return self._parsed_groups[group] + + +# -------------------- +# Functional Interface +# -------------------- + + +def resolve_dependency_groups( + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], /, *groups: str +) -> tuple[str, ...]: + """ + Resolve a dependency group to a tuple of requirements, as strings. + + :param dependency_groups: the parsed contents of the ``[dependency-groups]`` table + from ``pyproject.toml`` + :param groups: the name of the group(s) to resolve + """ + resolver = DependencyGroupResolver(dependency_groups) + return tuple(str(r) for group in groups for r in resolver.resolve(group)) + + +# ---------------- +# internal helpers +# ---------------- + + +_NORMALIZE_PATTERN = re.compile(r"[-_.]+") + + +def _normalize_name(name: str) -> str: + return _NORMALIZE_PATTERN.sub("-", name).lower() + + +def _normalize_group_names( + dependency_groups: Mapping[str, Sequence[str | Mapping[str, str]]], + errors: _ErrorCollector, +) -> dict[str, Sequence[str | Mapping[str, str]]]: + original_names: dict[str, list[str]] = {} + normalized_groups: dict[str, Sequence[str | Mapping[str, str]]] = {} + + for group_name, value in dependency_groups.items(): + normed_group_name = _normalize_name(group_name) + original_names.setdefault(normed_group_name, []).append(group_name) + normalized_groups[normed_group_name] = value + + for normed_name, names in original_names.items(): + if len(names) > 1: + errors.error( + DuplicateGroupNames( + "Duplicate dependency group names: " + f"{normed_name} ({', '.join(names)})" + ) + ) + + return normalized_groups diff --git a/contrib/python/pip/pip/_vendor/packaging/direct_url.py b/contrib/python/pip/pip/_vendor/packaging/direct_url.py new file mode 100644 index 00000000000..5d1c56ca9d6 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/packaging/direct_url.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import dataclasses +import re +import urllib.parse +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Protocol, TypeVar + +if TYPE_CHECKING: # pragma: no cover + import sys + from collections.abc import Collection + + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self + +__all__ = [ + "ArchiveInfo", + "DirInfo", + "DirectUrl", + "DirectUrlValidationError", + "VcsInfo", +] + + +def __dir__() -> list[str]: + return __all__ + + +_T = TypeVar("_T") + + +class _FromMappingProtocol(Protocol): # pragma: no cover + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... + + +_FromMappingProtocolT = TypeVar("_FromMappingProtocolT", bound=_FromMappingProtocol) + + +def _json_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: + return {key: value for key, value in data if value is not None} + + +def _get(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T | None: + """Get a value from the dictionary and verify it's the expected type.""" + if (value := d.get(key)) is None: + return None + if not isinstance(value, expected_type): + raise DirectUrlValidationError( + f"Unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})", + context=key, + ) + return value + + +def _get_required(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T: + """Get a required value from the dictionary and verify it's the expected type.""" + if (value := _get(d, expected_type, key)) is None: + raise _DirectUrlRequiredKeyError(key) + return value + + +def _get_object( + d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str +) -> _FromMappingProtocolT | None: + """Get a dictionary value from the dictionary and convert it to a dataclass.""" + if (value := _get(d, Mapping, key)) is None: # type: ignore[type-abstract] + return None + try: + return target_type._from_dict(value) + except Exception as e: + raise DirectUrlValidationError(e, context=key) from e + + +_PEP610_USER_PASS_ENV_VARS_REGEX = re.compile( + r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$" +) + + +def _strip_auth_from_netloc(netloc: str, safe_user_passwords: Collection[str]) -> str: + if "@" not in netloc: + return netloc + user_pass, netloc_no_user_pass = netloc.split("@", 1) + if user_pass in safe_user_passwords: + return netloc + if _PEP610_USER_PASS_ENV_VARS_REGEX.match(user_pass): + return netloc + return netloc_no_user_pass + + +def _strip_url(url: str, safe_user_passwords: Collection[str]) -> str: + """url with user:password part removed unless it is formed with + environment variables as specified in PEP 610, or it is a safe user:password + such as `git`. + """ + parsed_url = urllib.parse.urlsplit(url) + netloc = _strip_auth_from_netloc(parsed_url.netloc, safe_user_passwords) + return urllib.parse.urlunsplit( + ( + parsed_url.scheme, + netloc, + parsed_url.path, + parsed_url.query, + parsed_url.fragment, + ) + ) + + +class DirectUrlValidationError(Exception): + """Raised when when input data is not spec-compliant.""" + + context: str | None = None + message: str + + def __init__( + self, + cause: str | Exception, + *, + context: str | None = None, + ) -> None: + if isinstance(cause, DirectUrlValidationError): + if cause.context: + self.context = ( + f"{context}.{cause.context}" if context else cause.context + ) + else: + self.context = context # pragma: no cover + self.message = cause.message + else: + self.context = context + self.message = str(cause) + + def __str__(self) -> str: + if self.context: + return f"{self.message} in {self.context!r}" + return self.message + + +class _DirectUrlRequiredKeyError(DirectUrlValidationError): + def __init__(self, key: str) -> None: + super().__init__("Missing required value", context=key) + + [email protected](frozen=True, init=False) +class VcsInfo: + vcs: str + commit_id: str + requested_revision: str | None = None + + def __init__( + self, + *, + vcs: str, + commit_id: str, + requested_revision: str | None = None, + ) -> None: + object.__setattr__(self, "vcs", vcs) + object.__setattr__(self, "commit_id", commit_id) + object.__setattr__(self, "requested_revision", requested_revision) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + # We can't validate vcs value because is not closed. + return cls( + vcs=_get_required(d, str, "vcs"), + requested_revision=_get(d, str, "requested_revision"), + commit_id=_get_required(d, str, "commit_id"), + ) + + [email protected](frozen=True, init=False) +class ArchiveInfo: + hashes: Mapping[str, str] | None = None + + def __init__( + self, + *, + hashes: Mapping[str, str] | None = None, + ) -> None: + object.__setattr__(self, "hashes", hashes) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + hashes = _get(d, Mapping, "hashes") # type: ignore[type-abstract] + if hashes is not None and not all(isinstance(h, str) for h in hashes.values()): + raise DirectUrlValidationError( + "Hash values must be strings", context="hashes" + ) + legacy_hash = _get(d, str, "hash") + if legacy_hash is not None: + if "=" not in legacy_hash: + raise DirectUrlValidationError( + "Invalid hash format (expected '<algorithm>=<hash>')", + context="hash", + ) + hash_algorithm, hash_value = legacy_hash.split("=", 1) + if hashes is None: + # if `hashes` are not present, we can derive it from the legacy `hash` + hashes = {hash_algorithm: hash_value} + else: + # if `hashes` are present, the legacy `hash` must match one of them + if hash_algorithm not in hashes: + raise DirectUrlValidationError( + f"Algorithm {hash_algorithm!r} used in hash field " + f"is not present in hashes field", + context="hashes", + ) + if hashes[hash_algorithm] != hash_value: + raise DirectUrlValidationError( + f"Algorithm {hash_algorithm!r} used in hash field " + f"has different value in hashes field", + context="hash", + ) + return cls(hashes=hashes) + + [email protected](frozen=True, init=False) +class DirInfo: + editable: bool | None = None + + def __init__( + self, + *, + editable: bool | None = None, + ) -> None: + object.__setattr__(self, "editable", editable) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + editable=_get(d, bool, "editable"), + ) + + [email protected](frozen=True, init=False) +class DirectUrl: + """A class representing a direct URL.""" + + url: str + archive_info: ArchiveInfo | None = None + vcs_info: VcsInfo | None = None + dir_info: DirInfo | None = None + subdirectory: str | None = None # XXX Path or str? + + def __init__( + self, + *, + url: str, + archive_info: ArchiveInfo | None = None, + vcs_info: VcsInfo | None = None, + dir_info: DirInfo | None = None, + subdirectory: str | None = None, + ) -> None: + object.__setattr__(self, "url", url) + object.__setattr__(self, "archive_info", archive_info) + object.__setattr__(self, "vcs_info", vcs_info) + object.__setattr__(self, "dir_info", dir_info) + object.__setattr__(self, "subdirectory", subdirectory) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + direct_url = cls( + url=_get_required(d, str, "url"), + archive_info=_get_object(d, ArchiveInfo, "archive_info"), + vcs_info=_get_object(d, VcsInfo, "vcs_info"), + dir_info=_get_object(d, DirInfo, "dir_info"), + subdirectory=_get(d, str, "subdirectory"), + ) + if ( + bool(direct_url.vcs_info) + + bool(direct_url.archive_info) + + bool(direct_url.dir_info) + ) != 1: + raise DirectUrlValidationError( + "Exactly one of vcs_info, archive_info, dir_info must be present" + ) + if direct_url.dir_info is not None and not direct_url.url.startswith("file://"): + raise DirectUrlValidationError( + "URL scheme must be file:// when dir_info is present", + context="url", + ) + # XXX subdirectory must be relative, can we, should we validate that here? + return direct_url + + @classmethod + def from_dict(cls, d: Mapping[str, Any], /) -> Self: + """Create and validate a DirectUrl instance from a JSON dictionary.""" + return cls._from_dict(d) + + def to_dict( + self, + *, + generate_legacy_hash: bool = False, + strip_user_password: bool = True, + safe_user_passwords: Collection[str] = ("git",), + ) -> Mapping[str, Any]: + """Convert the DirectUrl instance to a JSON dictionary. + + :param generate_legacy_hash: If True, include a legacy `hash` field in + `archive_info` for backward compatibility with tools that don't + support the `hashes` field. + :param strip_user_password: If True, strip user:password from the URL + unless it is formed with environment variables as specified in PEP + 610, or it is a safe user:password such as `git`. + :param safe_user_passwords: A collection of user:password strings that + should not be stripped from the URL even if `strip_user_password` is + True. + """ + res = dataclasses.asdict(self, dict_factory=_json_dict_factory) + if generate_legacy_hash and self.archive_info and self.archive_info.hashes: + hash_algorithm, hash_value = next(iter(self.archive_info.hashes.items())) + res["archive_info"]["hash"] = f"{hash_algorithm}={hash_value}" + if strip_user_password: + res["url"] = _strip_url(self.url, safe_user_passwords) + return res + + def validate(self) -> None: + """Validate the DirectUrl instance against the specification. + + Raises :class:`DirectUrlValidationError` if invalid. + """ + self.from_dict(self.to_dict()) diff --git a/contrib/python/pip/pip/_vendor/packaging/errors.py b/contrib/python/pip/pip/_vendor/packaging/errors.py new file mode 100644 index 00000000000..d1d47cf6c34 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/packaging/errors.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import contextlib +import dataclasses +import sys +import typing + +__all__ = ["ExceptionGroup"] + + +def __dir__() -> list[str]: + return __all__ + + +if sys.version_info >= (3, 11): # pragma: no cover + from builtins import ExceptionGroup +else: # pragma: no cover + + class ExceptionGroup(Exception): + """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. + + If :external:exc:`ExceptionGroup` is already defined by Python itself, + that version is used instead. + """ + + message: str + exceptions: list[Exception] + + def __init__(self, message: str, exceptions: list[Exception]) -> None: + self.message = message + self.exceptions = exceptions + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" + + +class _ErrorCollector: + """ + Collect errors into ExceptionGroups. + + Used like this: + + collector = _ErrorCollector() + # Add a single exception + collector.error(ValueError("one")) + + # Supports nesting, including combining ExceptionGroups + with collector.collect(): + raise ValueError("two") + collector.finalize("Found some errors") + + Since making a collector and then calling finalize later is a common pattern, + a convenience method ``on_exit`` is provided. + """ + + errors: list[Exception] = dataclasses.field(default_factory=list, init=False) + + def finalize(self, msg: str) -> None: + """Raise a group exception if there are any errors.""" + if self.errors: + raise ExceptionGroup(msg, self.errors) + + @contextlib.contextmanager + def on_exit(self, msg: str) -> typing.Generator[_ErrorCollector, None, None]: + """ + Calls finalize if no uncollected errors were present. + + Uncollected errors are raised normally. + """ + yield self + self.finalize(msg) + + @contextlib.contextmanager + def collect(self, *err_cls: type[Exception]) -> typing.Generator[None, None, None]: + """ + Context manager to collect errors into the error list. + + Must be inside loops, as only one error can be collected at a time. + """ + error_classes = err_cls or (Exception,) + try: + yield + except ExceptionGroup as error: + self.errors.extend(error.exceptions) + except error_classes as error: + self.errors.append(error) + + def error( + self, + error: Exception, + ) -> None: + """Add an error to the list.""" + self.errors.append(error) diff --git a/contrib/python/pip/pip/_vendor/packaging/licenses/__init__.py b/contrib/python/pip/pip/_vendor/packaging/licenses/__init__.py index 335b275fa75..36e46ed02ca 100644 --- a/contrib/python/pip/pip/_vendor/packaging/licenses/__init__.py +++ b/contrib/python/pip/pip/_vendor/packaging/licenses/__init__.py @@ -42,14 +42,25 @@ __all__ = [ "canonicalize_license_expression", ] + +# Simple __dir__ implementation since there are no public submodules +def __dir__() -> list[str]: + return __all__ + + license_ref_allowed = re.compile("^[A-Za-z0-9.-]*$") NormalizedLicenseExpression = NewType("NormalizedLicenseExpression", str) +""" +A :class:`typing.NewType` of :class:`str`, representing a normalized +License-Expression. +""" class InvalidLicenseExpression(ValueError): """Raised when a license-expression string is invalid + >>> from packaging.licenses import canonicalize_license_expression >>> canonicalize_license_expression("invalid") Traceback (most recent call last): ... @@ -60,6 +71,34 @@ class InvalidLicenseExpression(ValueError): def canonicalize_license_expression( raw_license_expression: str, ) -> NormalizedLicenseExpression: + """ + This function takes a valid License-Expression, and returns the normalized + form of it. + + The return type is typed as :class:`NormalizedLicenseExpression`. This + allows type checkers to help require that a string has passed through this + function before use. + + :param str raw_license_expression: The License-Expression to canonicalize. + :raises InvalidLicenseExpression: If the License-Expression is invalid due to an + invalid/unknown license identifier or invalid syntax. + + .. doctest:: + + >>> from packaging.licenses import canonicalize_license_expression + >>> canonicalize_license_expression("mit") + 'MIT' + >>> canonicalize_license_expression("mit and (apache-2.0 or bsd-2-clause)") + 'MIT AND (Apache-2.0 OR BSD-2-Clause)' + >>> canonicalize_license_expression("(mit") + Traceback (most recent call last): + ... + InvalidLicenseExpression: Invalid license expression: '(mit' + >>> canonicalize_license_expression("Use-it-after-midnight") + Traceback (most recent call last): + ... + InvalidLicenseExpression: Unknown license: 'Use-it-after-midnight' + """ if not raw_license_expression: message = f"Invalid license expression: {raw_license_expression!r}" raise InvalidLicenseExpression(message) diff --git a/contrib/python/pip/pip/_vendor/packaging/markers.py b/contrib/python/pip/pip/_vendor/packaging/markers.py index ca3706fe492..62846452f94 100644 --- a/contrib/python/pip/pip/_vendor/packaging/markers.py +++ b/contrib/python/pip/pip/_vendor/packaging/markers.py @@ -26,8 +26,22 @@ __all__ = [ "default_environment", ] + +def __dir__() -> list[str]: + return __all__ + + Operator = Callable[[str, Union[str, AbstractSet[str]]], bool] EvaluateContext = Literal["metadata", "lock_file", "requirement"] +"""A ``typing.Literal`` enumerating valid marker evaluation contexts. + +Valid values for the ``context`` passed to :meth:`Marker.evaluate` are: + +* ``"metadata"`` (for core metadata; default) +* ``"lock_file"`` (for lock files) +* ``"requirement"`` (i.e. all other situations) +""" + MARKERS_ALLOWING_SET = {"extras", "dependency_groups"} MARKERS_REQUIRING_VERSION = { "implementation_version", @@ -38,25 +52,32 @@ MARKERS_REQUIRING_VERSION = { class InvalidMarker(ValueError): - """ - An invalid marker was found, users should refer to PEP 508. + """Raised when attempting to create a :class:`Marker` from invalid input. + + This error indicates that the given marker string does not conform to the + :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`. """ class UndefinedComparison(ValueError): - """ - An invalid operation was attempted on a value that doesn't support it. + """Raised when evaluating an unsupported marker comparison. + + This can happen when marker values are compared as versions but do not + conform to the :ref:`specification of version specifiers + <pypug:version-specifiers>`. """ class UndefinedEnvironmentName(ValueError): - """ - A name was attempted to be used that does not exist inside of the - environment. - """ + """Raised when evaluating a marker that references a missing environment key.""" class Environment(TypedDict): + """ + A dictionary that represents a Python environment as captured by + :func:`default_environment`. All fields are required. + """ + implementation_name: str """The implementation's identifier, e.g. ``'cpython'``.""" @@ -263,7 +284,7 @@ def _evaluate_markers( return any(all(item) for item in groups) -def format_full_version(info: sys._version_info) -> str: +def _format_full_version(info: sys._version_info) -> str: version = f"{info.major}.{info.minor}.{info.micro}" kind = info.releaselevel if kind != "final": @@ -272,7 +293,11 @@ def format_full_version(info: sys._version_info) -> str: def default_environment() -> Environment: - iver = format_full_version(sys.implementation.version) + """Return the default marker environment for the current Python process. + + This is the base environment used by :meth:`Marker.evaluate`. + """ + iver = _format_full_version(sys.implementation.version) implementation_name = sys.implementation.name return { "implementation_name": implementation_name, @@ -290,6 +315,27 @@ def default_environment() -> Environment: class Marker: + """Represents a parsed dependency marker expression. + + Marker expressions are parsed according to the + :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`. + + :param marker: The string representation of a marker expression. + :raises InvalidMarker: If ``marker`` cannot be parsed. + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with packaging 26.2+ can + be unpickled with future releases. Backward compatibility with pickles + from pip._vendor.packaging < 26.2 is supported but may be removed in a future + release. + """ + + __slots__ = ("_markers",) + def __init__(self, marker: str) -> None: # Note: We create a Marker object without calling this constructor in # packaging.requirements.Requirement. If any additional logic is @@ -320,11 +366,21 @@ class Marker: except ParserSyntaxError as e: raise InvalidMarker(str(e)) from e + @classmethod + def _from_markers(cls, markers: MarkerList) -> Marker: + """Create a Marker instance from a pre-parsed marker tree. + + This avoids re-parsing serialised marker strings when combining markers. + """ + new = cls.__new__(cls) + new._markers = markers + return new + def __str__(self) -> str: return _format_marker(self._markers) def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __hash__(self) -> int: return hash(str(self)) @@ -335,6 +391,45 @@ class Marker: return str(self) == str(other) + def __getstate__(self) -> str: + # Return the marker expression string for compactness and stability. + # Internal Node objects are excluded; the string is re-parsed on load. + return str(self) + + def __setstate__(self, state: object) -> None: + if isinstance(state, str): + # New format (26.2+): just the marker expression string. + try: + self._markers = _normalize_extra_values(_parse_marker(state)) + except ParserSyntaxError as exc: + raise TypeError(f"Cannot restore Marker from {state!r}") from exc + return + if isinstance(state, dict) and "_markers" in state: + # Old format (packaging <= 26.1, no __slots__): plain __dict__. + markers = state["_markers"] + if isinstance(markers, list): + self._markers = markers + return + if isinstance(state, tuple) and len(state) == 2: + # Old format (packaging <= 26.1, __slots__): (None, {slot: value}). + _, slot_dict = state + if isinstance(slot_dict, dict) and "_markers" in slot_dict: + markers = slot_dict["_markers"] + if isinstance(markers, list): + self._markers = markers + return + raise TypeError(f"Cannot restore Marker from {state!r}") + + def __and__(self, other: Marker) -> Marker: + if not isinstance(other, Marker): + return NotImplemented + return self._from_markers([self._markers, "and", other._markers]) + + def __or__(self, other: Marker) -> Marker: + if not isinstance(other, Marker): + return NotImplemented + return self._from_markers([self._markers, "or", other._markers]) + def evaluate( self, environment: Mapping[str, str | AbstractSet[str]] | None = None, @@ -342,14 +437,23 @@ class Marker: ) -> bool: """Evaluate a marker. - Return the boolean from evaluating the given marker against the - environment. environment is an optional argument to override all or - part of the determined environment. The *context* parameter specifies what - context the markers are being evaluated for, which influences what markers - are considered valid. Acceptable values are "metadata" (for core metadata; - default), "lock_file", and "requirement" (i.e. all other situations). + Return the boolean from evaluating this marker against the environment. + The environment is determined from the current Python process unless + passed in explicitly. + + :param environment: Mapping containing keys and values to override the + detected environment. + :param EvaluateContext context: The context in which the marker is + evaluated, which influences what marker names are considered valid. + Accepted values are ``"metadata"`` (for core metadata; default), + ``"lock_file"``, and ``"requirement"`` (i.e. all other situations). + :raises UndefinedComparison: If the marker uses a comparison on values + that are not valid versions per the :ref:`specification of version + specifiers <pypug:version-specifiers>`. + :raises UndefinedEnvironmentName: If the marker references a value that + is missing from the evaluation environment. + :returns: ``True`` if the marker matches, otherwise ``False``. - The environment is determined from the current Python process. """ current_environment = cast( "dict[str, str | AbstractSet[str]]", default_environment() diff --git a/contrib/python/pip/pip/_vendor/packaging/metadata.py b/contrib/python/pip/pip/_vendor/packaging/metadata.py index 253f6b1b7eb..dccb627dea3 100644 --- a/contrib/python/pip/pip/_vendor/packaging/metadata.py +++ b/contrib/python/pip/pip/_vendor/packaging/metadata.py @@ -1,13 +1,11 @@ from __future__ import annotations -import email.feedparser import email.header import email.message import email.parser import email.policy import keyword import pathlib -import sys import typing from typing import ( Any, @@ -20,6 +18,7 @@ from typing import ( from . import licenses, requirements, specifiers, utils from . import version as version_module +from .errors import ExceptionGroup, _ErrorCollector if typing.TYPE_CHECKING: from .licenses import NormalizedLicenseExpression @@ -27,26 +26,19 @@ if typing.TYPE_CHECKING: T = typing.TypeVar("T") -if sys.version_info >= (3, 11): # pragma: no cover - ExceptionGroup = ExceptionGroup # noqa: F821 -else: # pragma: no cover +__all__ = [ + "ExceptionGroup", # Keep this for a bit (makes mypy happy w/ 26.0 compat) + "InvalidMetadata", + "Metadata", + "RFC822Message", + "RFC822Policy", + "RawMetadata", + "parse_email", +] - class ExceptionGroup(Exception): - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: list[Exception] - - def __init__(self, message: str, exceptions: list[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" +def __dir__() -> list[str]: + return __all__ class InvalidMetadata(ValueError): @@ -76,8 +68,8 @@ class RawMetadata(TypedDict, total=False): Core metadata fields that can be specified multiple times are stored as a list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - + which hold multiple values in a single field are stored as a list. All fields + are considered optional. """ # Metadata 1.0 - PEP 241 @@ -641,7 +633,7 @@ class _Validator(Generic[T]): charset = parameters.get("charset", "UTF-8") if charset != "UTF-8": raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" + f"{{field}} can only specify the UTF-8 charset, not {charset!r}" ) markdown_variants = {"GFM", "CommonMark"} @@ -784,13 +776,11 @@ class Metadata: ins._raw = data.copy() # Mutations occur due to caching enriched values. if validate: - exceptions: list[Exception] = [] - try: + collector = _ErrorCollector() + metadata_version = None + with collector.collect(InvalidMetadata): metadata_version = ins.metadata_version metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None # Make sure to check for the fields that are present, the required # fields (so their absence can be reported). @@ -807,7 +797,7 @@ class Metadata: field_metadata_version = cls.__dict__[key].added except KeyError: exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) + collector.error(exc) continue field_age = _VALID_METADATA_VERSIONS.index( field_metadata_version @@ -819,14 +809,13 @@ class Metadata: f"{field} introduced in metadata version " f"{field_metadata_version}, not {metadata_version}", ) - exceptions.append(exc) + collector.error(exc) continue getattr(ins, key) except InvalidMetadata as exc: - exceptions.append(exc) + collector.error(exc) - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) + collector.finalize("invalid metadata") return ins @@ -840,16 +829,13 @@ class Metadata: raw, unparsed = parse_email(data) if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) + with _ErrorCollector().on_exit("unparsed") as collector: + for unparsed_key in unparsed: + if unparsed_key in _EMAIL_TO_RAW_MAPPING: + message = f"{unparsed_key!r} has invalid data" + else: + message = f"unrecognized field: {unparsed_key!r}" + collector.error(InvalidMetadata(unparsed_key, message)) try: return cls.from_raw(raw, validate=validate) diff --git a/contrib/python/pip/pip/_vendor/packaging/pylock.py b/contrib/python/pip/pip/_vendor/packaging/pylock.py index a564f15246a..84e25378fc6 100644 --- a/contrib/python/pip/pip/_vendor/packaging/pylock.py +++ b/contrib/python/pip/pip/_vendor/packaging/pylock.py @@ -12,18 +12,29 @@ from typing import ( Callable, Protocol, TypeVar, + cast, ) +from urllib.parse import urlparse -from .markers import Marker +from .markers import Environment, Marker, default_environment from .specifiers import SpecifierSet -from .utils import NormalizedName, is_normalized_name +from .tags import create_compatible_tags_selector, sys_tags +from .utils import ( + NormalizedName, + is_normalized_name, + parse_sdist_filename, + parse_wheel_filename, +) from .version import Version if TYPE_CHECKING: # pragma: no cover + from collections.abc import Collection, Iterator from pathlib import Path from typing_extensions import Self + from .tags import Tag + _logger = logging.getLogger(__name__) __all__ = [ @@ -39,6 +50,11 @@ __all__ = [ "is_valid_pylock_path", ] + +def __dir__() -> list[str]: + return __all__ + + _T = TypeVar("_T") _T2 = TypeVar("_T2") @@ -222,6 +238,26 @@ def _validate_path_url(path: str | None, url: str | None) -> None: raise PylockValidationError("path or url must be provided") +def _path_name(path: str | None) -> str | None: + if not path: + return None + # If the path is relative it MAY use POSIX-style path separators explicitly + # for portability + if "/" in path: + return path.rsplit("/", 1)[-1] + elif "\\" in path: + return path.rsplit("\\", 1)[-1] + else: + return path + + +def _url_name(url: str | None) -> str | None: + if not url: + return None + url_path = urlparse(url).path + return url_path.rsplit("/", 1)[-1] + + def _validate_hashes(hashes: Mapping[str, Any]) -> Mapping[str, Any]: if not hashes: raise PylockValidationError("At least one hash must be provided") @@ -269,6 +305,10 @@ class PylockUnsupportedVersionError(PylockValidationError): """Raised when encountering an unsupported `lock_version`.""" +class PylockSelectError(Exception): + """Base exception for errors raised by :meth:`Pylock.select`.""" + + @dataclass(frozen=True, init=False) class PackageVcs: type: str @@ -418,6 +458,14 @@ class PackageSdist: _validate_path_url(package_sdist.path, package_sdist.url) return package_sdist + @property + def filename(self) -> str: + """Get the filename of the sdist.""" + filename = self.name or _path_name(self.path) or _url_name(self.url) + if not filename: + raise PylockValidationError("Cannot determine sdist filename") + return filename + @dataclass(frozen=True, init=False) class PackageWheel: @@ -459,6 +507,14 @@ class PackageWheel: _validate_path_url(package_wheel.path, package_wheel.url) return package_wheel + @property + def filename(self) -> str: + """Get the filename of the wheel.""" + filename = self.name or _path_name(self.path) or _url_name(self.url) + if not filename: + raise PylockValidationError("Cannot determine wheel filename") + return filename + @dataclass(frozen=True, init=False) class Package: @@ -538,6 +594,46 @@ class Package: "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" ) + for i, wheel in enumerate(package.wheels or []): + try: + (name, version, _, _) = parse_wheel_filename(wheel.filename) + except Exception as e: + raise PylockValidationError( + f"Invalid wheel filename {wheel.filename!r}", + context=f"wheels[{i}]", + ) from e + if name != package.name: + raise PylockValidationError( + f"Name in {wheel.filename!r} is not consistent with " + f"package name {package.name!r}", + context=f"wheels[{i}]", + ) + if package.version and version != package.version: + raise PylockValidationError( + f"Version in {wheel.filename!r} is not consistent with " + f"package version {str(package.version)!r}", + context=f"wheels[{i}]", + ) + if package.sdist: + try: + name, version = parse_sdist_filename(package.sdist.filename) + except Exception as e: + raise PylockValidationError( + f"Invalid sdist filename {package.sdist.filename!r}", + context="sdist", + ) from e + if name != package.name: + raise PylockValidationError( + f"Name in {package.sdist.filename!r} is not consistent with " + f"package name {package.name!r}", + context="sdist", + ) + if package.version and version != package.version: + raise PylockValidationError( + f"Version in {package.sdist.filename!r} is not consistent with " + f"package version {str(package.version)!r}", + context="sdist", + ) try: for i, attestation_identity in enumerate( # noqa: B007 package.attestation_identities or [] @@ -633,3 +729,177 @@ class Pylock: Raises :class:`PylockValidationError` otherwise.""" self.from_dict(self.to_dict()) + + def select( + self, + *, + environment: Environment | None = None, + tags: Sequence[Tag] | None = None, + extras: Collection[str] | None = None, + dependency_groups: Collection[str] | None = None, + ) -> Iterator[ + tuple[ + Package, + PackageVcs + | PackageDirectory + | PackageArchive + | PackageWheel + | PackageSdist, + ] + ]: + """Select what to install from the lock file. + + The *environment* and *tags* parameters represent the environment being + selected for. If unspecified, ``packaging.markers.default_environment()`` and + ``packaging.tags.sys_tags()`` are used. + + The *extras* parameter represents the extras to install. + + The *dependency_groups* parameter represents the groups to install. If + unspecified, the default groups are used. + + This method must be used on valid Pylock instances (i.e. one obtained + from :meth:`Pylock.from_dict` or if constructed manually, after calling + :meth:`Pylock.validate`). + """ + compatible_tags_selector = create_compatible_tags_selector(tags or sys_tags()) + + # #. Gather the extras and dependency groups to install and set ``extras`` and + # ``dependency_groups`` for marker evaluation, respectively. + # + # #. ``extras`` SHOULD be set to the empty set by default. + # #. ``dependency_groups`` SHOULD be the set created from + # :ref:`pylock-default-groups` by default. + env = cast( + "dict[str, str | frozenset[str]]", + dict( + environment or {}, # Marker.evaluate will fill-up + extras=frozenset(extras or []), + dependency_groups=frozenset( + (self.default_groups or []) + if dependency_groups is None # to allow selecting no group + else dependency_groups + ), + ), + ) + env_python_full_version = ( + environment["python_full_version"] + if environment + else default_environment()["python_full_version"] + ) + + # #. Check if the metadata version specified by :ref:`pylock-lock-version` is + # supported; an error or warning MUST be raised as appropriate. + # Covered by lock.validate() which is a precondition for this method. + + # #. If :ref:`pylock-requires-python` is specified, check that the environment + # being installed for meets the requirement; an error MUST be raised if it is + # not met. + if self.requires_python and not self.requires_python.contains( + env_python_full_version, + ): + raise PylockSelectError( + f"python_full_version {env_python_full_version!r} " + f"in provided environment does not satisfy the Python version " + f"requirement {str(self.requires_python)!r}" + ) + + # #. If :ref:`pylock-environments` is specified, check that at least one of the + # environment marker expressions is satisfied; an error MUST be raised if no + # expression is satisfied. + if self.environments: + for env_marker in self.environments: + if env_marker.evaluate( + cast("dict[str, str]", environment or {}), context="requirement" + ): + break + else: + raise PylockSelectError( + "Provided environment does not satisfy any of the " + "environments specified in the lock file" + ) + + # #. For each package listed in :ref:`pylock-packages`: + selected_packages_by_name: dict[str, tuple[int, Package]] = {} + for package_index, package in enumerate(self.packages): + # #. If :ref:`pylock-packages-marker` is specified, check if it is + # satisfied;if it isn't, skip to the next package. + if package.marker and not package.marker.evaluate(env, context="lock_file"): + continue + + # #. If :ref:`pylock-packages-requires-python` is specified, check if it is + # satisfied; an error MUST be raised if it isn't. + if package.requires_python and not package.requires_python.contains( + env_python_full_version, + ): + raise PylockSelectError( + f"python_full_version {env_python_full_version!r} " + f"in provided environment does not satisfy the Python version " + f"requirement {str(package.requires_python)!r} for package " + f"{package.name!r} at packages[{package_index}]" + ) + + # #. Check that no other conflicting instance of the package has been slated + # to be installed; an error about the ambiguity MUST be raised otherwise. + if package.name in selected_packages_by_name: + raise PylockSelectError( + f"Multiple packages with the name {package.name!r} are " + f"selected at packages[{package_index}] and " + f"packages[{selected_packages_by_name[package.name][0]}]" + ) + + # #. Check that the source of the package is specified appropriately (i.e. + # there are no conflicting sources in the package entry); + # an error MUST be raised if any issues are found. + # Covered by lock.validate() which is a precondition for this method. + + # #. Add the package to the set of packages to install. + selected_packages_by_name[package.name] = (package_index, package) + + # #. For each package to be installed: + for package_index, package in selected_packages_by_name.values(): + # - If :ref:`pylock-packages-vcs` is set: + if package.vcs is not None: + yield package, package.vcs + + # - Else if :ref:`pylock-packages-directory` is set: + elif package.directory is not None: + yield package, package.directory + + # - Else if :ref:`pylock-packages-archive` is set: + elif package.archive is not None: + yield package, package.archive + + # - Else if there are entries for :ref:`pylock-packages-wheels`: + elif package.wheels: + # #. Look for the appropriate wheel file based on + # :ref:`pylock-packages-wheels-name`; if one is not found then move + # on to :ref:`pylock-packages-sdist` or an error MUST be raised about + # a lack of source for the project. + best_wheel = next( + compatible_tags_selector( + (wheel, parse_wheel_filename(wheel.filename)[-1]) + for wheel in package.wheels + ), + None, + ) + if best_wheel: + yield package, best_wheel + elif package.sdist is not None: + yield package, package.sdist + else: + raise PylockSelectError( + f"No wheel found matching the provided tags " + f"for package {package.name!r} " + f"at packages[{package_index}], " + f"and no sdist available as a fallback" + ) + + # - Else if no :ref:`pylock-packages-wheels` file is found or + # :ref:`pylock-packages-sdist` is solely set: + elif package.sdist is not None: + yield package, package.sdist + + else: + # Covered by lock.validate() which is a precondition for this method. + raise NotImplementedError # pragma: no cover diff --git a/contrib/python/pip/pip/_vendor/packaging/requirements.py b/contrib/python/pip/pip/_vendor/packaging/requirements.py index 3079be69bf8..d5c786d161e 100644 --- a/contrib/python/pip/pip/_vendor/packaging/requirements.py +++ b/contrib/python/pip/pip/_vendor/packaging/requirements.py @@ -11,6 +11,15 @@ from .markers import Marker, _normalize_extra_values from .specifiers import SpecifierSet from .utils import canonicalize_name +__all__ = [ + "InvalidRequirement", + "Requirement", +] + + +def __dir__() -> list[str]: + return __all__ + class InvalidRequirement(ValueError): """ @@ -24,6 +33,16 @@ class Requirement: Parse a given requirement string into its parts, such as name, specifier, URL, and extras. Raises InvalidRequirement on a badly-formed requirement string. + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with packaging 26.2+ can + be unpickled with future releases. Backward compatibility with pickles + from pip._vendor.packaging < 26.2 is supported but may be removed in a future + release. """ # TODO: Can we test whether something is contained within a requirement? @@ -64,11 +83,35 @@ class Requirement: if self.marker: yield f"; {self.marker}" + def __getstate__(self) -> str: + # Return the requirement string for compactness and stability. + # Re-parsed on load to reconstruct all fields. + return str(self) + + def __setstate__(self, state: object) -> None: + if isinstance(state, str): + # New format (26.2+): just the requirement string. + try: + tmp = Requirement(state) + except InvalidRequirement as exc: + raise TypeError(f"Cannot restore Requirement from {state!r}") from exc + self.name = tmp.name + self.url = tmp.url + self.extras = tmp.extras + self.specifier = tmp.specifier + self.marker = tmp.marker + return + if isinstance(state, dict): + # Old format (packaging <= 26.1, no __slots__): plain __dict__. + self.__dict__.update(state) + return + raise TypeError(f"Cannot restore Requirement from {state!r}") + def __str__(self) -> str: return "".join(self._iter_parts(self.name)) def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __hash__(self) -> int: return hash(tuple(self._iter_parts(canonicalize_name(self.name)))) diff --git a/contrib/python/pip/pip/_vendor/packaging/specifiers.py b/contrib/python/pip/pip/_vendor/packaging/specifiers.py index e4b9c442e40..83e0015026c 100644 --- a/contrib/python/pip/pip/_vendor/packaging/specifiers.py +++ b/contrib/python/pip/pip/_vendor/packaging/specifiers.py @@ -11,17 +11,283 @@ from __future__ import annotations import abc +import enum +import functools import itertools import re -from typing import Callable, Final, Iterable, Iterator, TypeVar, Union +import sys +import typing +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Final, + Iterable, + Iterator, + Sequence, + TypeVar, + Union, +) from .utils import canonicalize_version from .version import InvalidVersion, Version +if sys.version_info >= (3, 10): + from typing import TypeGuard # pragma: no cover +elif TYPE_CHECKING: + from typing_extensions import TypeGuard + +__all__ = [ + "BaseSpecifier", + "InvalidSpecifier", + "Specifier", + "SpecifierSet", +] + + +def __dir__() -> list[str]: + return __all__ + + +def _validate_spec(spec: object, /) -> TypeGuard[tuple[str, str]]: + return ( + isinstance(spec, tuple) + and len(spec) == 2 + and isinstance(spec[0], str) + and isinstance(spec[1], str) + ) + + +def _validate_pre(pre: object, /) -> TypeGuard[bool | None]: + return pre is None or isinstance(pre, bool) + + +T = TypeVar("T") UnparsedVersion = Union[Version, str] UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) CallableOperator = Callable[[Version, str], bool] +# The smallest possible PEP 440 version. No valid version is less than this. +_MIN_VERSION: Final[Version] = Version("0.dev0") + + +def _trim_release(release: tuple[int, ...]) -> tuple[int, ...]: + """Strip trailing zeros from a release tuple for normalized comparison.""" + end = len(release) + while end > 1 and release[end - 1] == 0: + end -= 1 + return release if end == len(release) else release[:end] + + +class _BoundaryKind(enum.Enum): + """Where a boundary marker sits in the version ordering.""" + + AFTER_LOCALS = enum.auto() # after V+local, before V.post0 + AFTER_POSTS = enum.auto() # after V.postN, before next release + + [email protected]_ordering +class _BoundaryVersion: + """A point on the version line between two real PEP 440 versions. + + Some specifier semantics imply boundaries between real versions: + ``<=1.0`` includes ``1.0+local`` and ``>1.0`` excludes + ``1.0.post0``. No real :class:`Version` falls on those boundaries, + so this class creates values that sort between the real versions + on either side. + + Two kinds exist, shown relative to a base version V:: + + V < V+local < AFTER_LOCALS(V) < V.post0 < AFTER_POSTS(V) + + ``AFTER_LOCALS`` sits after V and every V+local, but before + V.post0. Upper bound of ``<=V``, ``==V``, ``!=V``. + + ``AFTER_POSTS`` sits after every V.postN, but before the next + release segment. Lower bound of ``>V`` (final or pre-release V) + to exclude post-releases per PEP 440. + """ + + __slots__ = ("_kind", "_trimmed_release", "version") + + def __init__(self, version: Version, kind: _BoundaryKind) -> None: + self.version = version + self._kind = kind + self._trimmed_release = _trim_release(version.release) + + def _is_family(self, other: Version) -> bool: + """Is ``other`` a version that this boundary sorts above?""" + v = self.version + if not ( + other.epoch == v.epoch + and _trim_release(other.release) == self._trimmed_release + and other.pre == v.pre + ): + return False + if self._kind == _BoundaryKind.AFTER_LOCALS: + # Local family: exact same public version (any local label). + return other.post == v.post and other.dev == v.dev + # Post family: same base + any post-release (or identical). + return other.dev == v.dev or other.post is not None + + def __eq__(self, other: object) -> bool: + if isinstance(other, _BoundaryVersion): + return self.version == other.version and self._kind == other._kind + return NotImplemented + + def __lt__(self, other: _BoundaryVersion | Version) -> bool: + if isinstance(other, _BoundaryVersion): + if self.version != other.version: + return self.version < other.version + return self._kind.value < other._kind.value + return not self._is_family(other) and self.version < other + + def __hash__(self) -> int: + return hash((self.version, self._kind)) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.version!r}, {self._kind.name})" + + [email protected]_ordering +class _LowerBound: + """Lower bound of a version range. + + A version *v* of ``None`` means unbounded below (-inf). + At equal versions, ``[v`` sorts before ``(v`` because an inclusive + bound starts earlier. + """ + + __slots__ = ("inclusive", "version") + + def __init__(self, version: _VersionOrBoundary, inclusive: bool) -> None: + self.version = version + self.inclusive = inclusive + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _LowerBound): + return NotImplemented # pragma: no cover + return self.version == other.version and self.inclusive == other.inclusive + + def __lt__(self, other: _LowerBound) -> bool: + if not isinstance(other, _LowerBound): # pragma: no cover + return NotImplemented + # -inf < anything (except -inf). + if self.version is None: + return other.version is not None + if other.version is None: + return False + if self.version != other.version: + return self.version < other.version + # [v < (v: inclusive starts earlier. + return self.inclusive and not other.inclusive + + def __hash__(self) -> int: + return hash((self.version, self.inclusive)) + + def __repr__(self) -> str: + bracket = "[" if self.inclusive else "(" + return f"<{self.__class__.__name__} {bracket}{self.version!r}>" + + [email protected]_ordering +class _UpperBound: + """Upper bound of a version range. + + A version *v* of ``None`` means unbounded above (+inf). + At equal versions, ``v)`` sorts before ``v]`` because an exclusive + bound ends earlier. + """ + + __slots__ = ("inclusive", "version") + + def __init__(self, version: _VersionOrBoundary, inclusive: bool) -> None: + self.version = version + self.inclusive = inclusive + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _UpperBound): + return NotImplemented # pragma: no cover + return self.version == other.version and self.inclusive == other.inclusive + + def __lt__(self, other: _UpperBound) -> bool: + if not isinstance(other, _UpperBound): # pragma: no cover + return NotImplemented + # Nothing < +inf (except +inf itself). + if self.version is None: + return False + if other.version is None: + return True + if self.version != other.version: + return self.version < other.version + # v) < v]: exclusive ends earlier. + return not self.inclusive and other.inclusive + + def __hash__(self) -> int: + return hash((self.version, self.inclusive)) + + def __repr__(self) -> str: + bracket = "]" if self.inclusive else ")" + return f"<{self.__class__.__name__} {self.version!r}{bracket}>" + + +if typing.TYPE_CHECKING: + _VersionOrBoundary = Union[Version, _BoundaryVersion, None] + + #: A single contiguous version range, represented as a + #: (lower bound, upper bound) pair. + _VersionRange = tuple[_LowerBound, _UpperBound] + +_NEG_INF = _LowerBound(None, False) +_POS_INF = _UpperBound(None, False) +_FULL_RANGE: tuple[_VersionRange] = ((_NEG_INF, _POS_INF),) + + +def _range_is_empty(lower: _LowerBound, upper: _UpperBound) -> bool: + """True when the range defined by *lower* and *upper* contains no versions.""" + if lower.version is None or upper.version is None: + return False + if lower.version == upper.version: + return not (lower.inclusive and upper.inclusive) + return lower.version > upper.version + + +def _intersect_ranges( + left: Sequence[_VersionRange], + right: Sequence[_VersionRange], +) -> list[_VersionRange]: + """Intersect two sorted, non-overlapping range lists (two-pointer merge).""" + result: list[_VersionRange] = [] + left_index = right_index = 0 + while left_index < len(left) and right_index < len(right): + left_lower, left_upper = left[left_index] + right_lower, right_upper = right[right_index] + + lower = max(left_lower, right_lower) + upper = min(left_upper, right_upper) + + if not _range_is_empty(lower, upper): + result.append((lower, upper)) + + # Advance whichever side has the smaller upper bound. + if left_upper < right_upper: + left_index += 1 + else: + right_index += 1 + + return result + + +def _next_prefix_dev0(version: Version) -> Version: + """Smallest version in the next prefix: 1.2 -> 1.3.dev0.""" + release = (*version.release[:-1], version.release[-1] + 1) + return Version.from_parts(epoch=version.epoch, release=release, dev=0) + + +def _base_dev0(version: Version) -> Version: + """The .dev0 of a version's base release: 1.2 -> 1.2.dev0.""" + return Version.from_parts(epoch=version.epoch, release=version.release, dev=0) + def _coerce_version(version: UnparsedVersion) -> Version | None: if not isinstance(version, Version): @@ -33,11 +299,46 @@ def _coerce_version(version: UnparsedVersion) -> Version | None: def _public_version(version: Version) -> Version: + if version.local is None: + return version return version.__replace__(local=None) -def _base_version(version: Version) -> Version: - return version.__replace__(pre=None, post=None, dev=None, local=None) +def _post_base(version: Version) -> Version: + """The version that *version* is a post-release of. + + 1.0.post1 -> 1.0, 1.0a1.post0 -> 1.0a1, 1.0.post0.dev1 -> 1.0. + """ + return version.__replace__(post=None, dev=None, local=None) + + +def _earliest_prerelease(version: Version) -> Version: + """Earliest pre-release of *version*. + + 1.2 -> 1.2.dev0, 1.2.post1 -> 1.2.post1.dev0. + """ + return version.__replace__(dev=0, local=None) + + +def _nearest_non_prerelease( + v: _VersionOrBoundary, +) -> Version | None: + """Smallest non-pre-release version at or above *v*, or None.""" + if v is None: + return None + if isinstance(v, _BoundaryVersion): + inner = v.version + if inner.is_prerelease: + # AFTER_LOCALS(1.0a1) -> nearest non-pre is 1.0 + return inner.__replace__(pre=None, dev=None, local=None) + # AFTER_LOCALS(1.0) -> nearest non-pre is 1.0.post0 + # AFTER_LOCALS(1.0.post0) -> nearest non-pre is 1.0.post1 + k = (inner.post + 1) if inner.post is not None else 0 + return inner.__replace__(post=k, local=None) + if not v.is_prerelease: + return v + # Strip pre/dev to get the final or post-release form. + return v.__replace__(pre=None, dev=None, local=None) class InvalidSpecifier(ValueError): @@ -105,10 +406,29 @@ class BaseSpecifier(metaclass=abc.ABCMeta): Determines if the given item is contained within this specifier. """ + @typing.overload + def filter( + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload + def filter( + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + @abc.abstractmethod def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """ Takes an iterable of items and filters them so that only items which are contained within this specifier are allowed in it. @@ -123,22 +443,35 @@ class Specifier(BaseSpecifier): It is generally not required to instantiate this manually. You should instead prefer to work with :class:`SpecifierSet` instead, which can parse comma-separated version specifiers (which is what package metadata contains). + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with packaging 26.2+ can + be unpickled with future releases. Backward compatibility with pickles + from pip._vendor.packaging < 26.2 is supported but may be removed in a future + release. """ - __slots__ = ("_prereleases", "_spec", "_spec_version") + __slots__ = ( + "_prereleases", + "_ranges", + "_spec", + "_spec_version", + "_wildcard_split", + ) - _operator_regex_str = r""" - (?P<operator>(~=|==|!=|<=|>=|<|>|===)) - """ - _version_regex_str = r""" - (?P<version> + _specifier_regex_str = r""" + (?: (?: # The identity operators allow for an escape hatch that will # do an exact string match of the version you wish to install. # This will not be parsed by PEP 440 and we cannot determine # any semantic meaning from it. This operator is discouraged # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator + === # Only match for the identity operator \s* [^\s;)]* # The arbitrary version can be just about anything, # we match everything except for whitespace, a @@ -150,7 +483,7 @@ class Specifier(BaseSpecifier): # The (non)equality operators allow for wild card and local # versions to be specified so we have to define these two # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals + (?:==|!=) # Only match for equals and not equals \s* v? @@ -162,24 +495,24 @@ class Specifier(BaseSpecifier): (?: \.\* # Wild card syntax of .* | - (?: # pre release + (?a: # pre release [-_\.]? (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? - (?: # post release + (?a: # post release (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + (?a:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?a:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local )? ) | (?: # The compatible operator requires at least two digits in the # release segment. - (?<=~=) # Only match for the compatible operator + (?:~=) # Only match for the compatible operator \s* v? @@ -202,31 +535,28 @@ class Specifier(BaseSpecifier): # (non)equality operators do. Specifically they do not allow # local versions to be specified nor do they allow the prefix # matching wild cards. - (?<!==|!=|~=) # We have special cases for these - # operators so we want to make sure they - # don't match here. + (?:<=|>=|<|>) \s* v? (?:[0-9]+!)? # epoch [0-9]+(?:\.[0-9]+)* # release - (?: # pre release + (?a: # pre release [-_\.]? (alpha|beta|preview|pre|a|b|c|rc) [-_\.]? [0-9]* )? - (?: # post release + (?a: # post release (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?a:[-_\.]?dev[-_\.]?[0-9]*)? # dev release ) ) """ _regex = re.compile( - r"\s*" + _operator_regex_str + _version_regex_str + r"\s*", - re.VERBOSE | re.IGNORECASE, + r"\s*" + _specifier_regex_str + r"\s*", re.VERBOSE | re.IGNORECASE ) _operators: Final = { @@ -253,14 +583,18 @@ class Specifier(BaseSpecifier): :raises InvalidSpecifier: If the given specifier is invalid (i.e. bad syntax). """ - match = self._regex.fullmatch(spec) - if not match: + if not self._regex.fullmatch(spec): raise InvalidSpecifier(f"Invalid specifier: {spec!r}") - self._spec: tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + spec = spec.strip() + if spec.startswith("==="): + operator, version = spec[:3], spec[3:].strip() + elif spec.startswith(("~=", "==", "!=", "<=", ">=")): + operator, version = spec[:2], spec[2:].strip() + else: + operator, version = spec[:1], spec[1:].strip() + + self._spec: tuple[str, str] = (operator, version) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -268,6 +602,12 @@ class Specifier(BaseSpecifier): # Specifier version cache self._spec_version: tuple[str, Version] | None = None + # Populated on first wildcard (==X.*) comparison + self._wildcard_split: tuple[list[str], int] | None = None + + # Version range cache (populated by _to_ranges) + self._ranges: Sequence[_VersionRange] | None = None + def _get_spec_version(self, version: str) -> Version | None: """One element cache, as only one spec Version is needed per Specifier.""" if self._spec_version is not None and self._spec_version[0] == version: @@ -290,6 +630,105 @@ class Specifier(BaseSpecifier): assert spec_version is not None return spec_version + def _to_ranges(self) -> Sequence[_VersionRange]: + """Convert this specifier to sorted, non-overlapping version ranges. + + Each standard operator maps to one or two ranges. ``===`` is + modeled as full range (actual check done separately). Cached. + """ + if self._ranges is not None: + return self._ranges + + op = self.operator + ver_str = self.version + + if op == "===": + self._ranges = _FULL_RANGE + return _FULL_RANGE + + if ver_str.endswith(".*"): + result = self._wildcard_ranges(op, ver_str) + else: + result = self._standard_ranges(op, ver_str) + + self._ranges = result + return result + + def _wildcard_ranges(self, op: str, ver_str: str) -> list[_VersionRange]: + # ==1.2.* -> [1.2.dev0, 1.3.dev0); !=1.2.* -> complement. + base = self._require_spec_version(ver_str[:-2]) + lower = _base_dev0(base) + upper = _next_prefix_dev0(base) + if op == "==": + return [(_LowerBound(lower, True), _UpperBound(upper, False))] + # != + return [ + (_NEG_INF, _UpperBound(lower, False)), + (_LowerBound(upper, True), _POS_INF), + ] + + def _standard_ranges(self, op: str, ver_str: str) -> list[_VersionRange]: + v = self._require_spec_version(ver_str) + + if op == ">=": + return [(_LowerBound(v, True), _POS_INF)] + + if op == "<=": + return [ + ( + _NEG_INF, + _UpperBound(_BoundaryVersion(v, _BoundaryKind.AFTER_LOCALS), True), + ) + ] + + if op == ">": + if v.dev is not None: + # >V.devN: dev versions have no post-releases, so the + # next real version is V.dev(N+1). + lower_ver = v.__replace__(dev=v.dev + 1, local=None) + return [(_LowerBound(lower_ver, True), _POS_INF)] + if v.post is not None: + # >V.postN: next real version is V.post(N+1).dev0. + lower_ver = v.__replace__(post=v.post + 1, dev=0, local=None) + return [(_LowerBound(lower_ver, True), _POS_INF)] + # >V (final or pre-release): skip V+local and all V.postN. + return [ + ( + _LowerBound(_BoundaryVersion(v, _BoundaryKind.AFTER_POSTS), False), + _POS_INF, + ) + ] + + if op == "<": + # <V excludes prereleases of V when V is not a prerelease. + # V.dev0 is the earliest prerelease of V (final, post, etc.). + bound = v if v.is_prerelease else v.__replace__(dev=0, local=None) + if bound <= _MIN_VERSION: + return [] + return [(_NEG_INF, _UpperBound(bound, False))] + + # ==, !=: local versions of V match when spec has no local segment. + has_local = "+" in ver_str + after_locals = _BoundaryVersion(v, _BoundaryKind.AFTER_LOCALS) + upper = v if has_local else after_locals + + if op == "==": + return [(_LowerBound(v, True), _UpperBound(upper, True))] + + if op == "!=": + return [ + (_NEG_INF, _UpperBound(v, False)), + (_LowerBound(upper, False), _POS_INF), + ] + + if op == "~=": + prefix = v.__replace__(release=v.release[:-1]) + return [ + (_LowerBound(v, True), _UpperBound(_next_prefix_dev0(prefix), False)) + ] + + raise ValueError(f"Unknown operator: {op!r}") # pragma: no cover + @property def prereleases(self) -> bool | None: # If there is an explicit prereleases set for this, then we'll just @@ -300,29 +739,68 @@ class Specifier(BaseSpecifier): # Only the "!=" operator does not imply prereleases when # the version in the specifier is a prerelease. operator, version_str = self._spec - if operator != "!=": - # The == specifier with trailing .* cannot include prereleases - # e.g. "==1.0a1.*" is not valid. - if operator == "==" and version_str.endswith(".*"): - return False + if operator == "!=": + return False - # "===" can have arbitrary string versions, so we cannot parse - # those, we take prereleases as unknown (None) for those. - version = self._get_spec_version(version_str) - if version is None: - return None + # The == specifier with trailing .* cannot include prereleases + # e.g. "==1.0a1.*" is not valid. + if operator == "==" and version_str.endswith(".*"): + return False - # For all other operators, use the check if spec Version - # object implies pre-releases. - if version.is_prerelease: - return True + # "===" can have arbitrary string versions, so we cannot parse + # those, we take prereleases as unknown (None) for those. + version = self._get_spec_version(version_str) + if version is None: + return None - return False + # For all other operators, use the check if spec Version + # object implies pre-releases. + return version.is_prerelease @prereleases.setter def prereleases(self, value: bool | None) -> None: self._prereleases = value + def __getstate__(self) -> tuple[tuple[str, str], bool | None]: + # Return state as a 2-item tuple for compactness: + # ((operator, version), prereleases) + # Cache members are excluded and will be recomputed on demand. + return (self._spec, self._prereleases) + + def __setstate__(self, state: object) -> None: + # Always discard cached values - they will be recomputed on demand. + self._spec_version = None + self._wildcard_split = None + self._ranges = None + + if isinstance(state, tuple): + if len(state) == 2: + # New format (26.2+): ((operator, version), prereleases) + spec, prereleases = state + if _validate_spec(spec) and _validate_pre(prereleases): + self._spec = spec + self._prereleases = prereleases + return + if len(state) == 2 and isinstance(state[1], dict): + # Format (packaging 26.0-26.1): (None, {slot: value}). + _, slot_dict = state + spec = slot_dict.get("_spec") + prereleases = slot_dict.get("_prereleases", "invalid") + if _validate_spec(spec) and _validate_pre(prereleases): + self._spec = spec + self._prereleases = prereleases + return + if isinstance(state, dict): + # Old format (packaging <= 25.x, no __slots__): state is a plain dict. + spec = state.get("_spec") + prereleases = state.get("_prereleases", "invalid") + if _validate_spec(spec) and _validate_pre(prereleases): + self._spec = spec + self._prereleases = prereleases + return + + raise TypeError(f"Cannot restore Specifier from {state!r}") + @property def operator(self) -> str: """The operator of this specifier. @@ -437,23 +915,35 @@ class Specifier(BaseSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( - prospective, prefix + return (self._compare_greater_than_equal(prospective, spec)) and ( + self._compare_equal(prospective, prefix) ) + def _get_wildcard_split(self, spec: str) -> tuple[list[str], int]: + """Cached split of a wildcard spec into components and numeric length. + + >>> Specifier("==1.*")._get_wildcard_split("1.*") + (['0', '1'], 2) + >>> Specifier("==3.10.*")._get_wildcard_split("3.10.*") + (['0', '3', '10'], 3) + """ + wildcard_split = self._wildcard_split + if wildcard_split is None: + normalized = canonicalize_version(spec[:-2], strip_trailing_zero=False) + split_spec = _version_split(normalized) + wildcard_split = (split_spec, _numeric_prefix_len(split_spec)) + self._wildcard_split = wildcard_split + return wildcard_split + def _compare_equal(self, prospective: Version, spec: str) -> bool: # We need special logic to handle prefix matching if spec.endswith(".*"): + split_spec, spec_numeric_len = self._get_wildcard_split(spec) + # In the case of prefix matching we want to ignore local segment. normalized_prospective = canonicalize_version( _public_version(prospective), strip_trailing_zero=False ) - # Get the normalized version string ignoring the trailing .* - normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) - # Split the spec out by bangs and dots, and pretend that there is - # an implicit dot in between a release segment and a pre-release segment. - split_spec = _version_split(normalized_spec) - # Split the prospective version out by bangs and dots, and pretend # that there is an implicit dot in between a release segment and # a pre-release segment. @@ -461,7 +951,7 @@ class Specifier(BaseSpecifier): # 0-pad the prospective version before shortening it to get the correct # shortened version. - padded_prospective, _ = _pad_version(split_prospective, split_spec) + padded_prospective = _left_pad(split_prospective, spec_numeric_len) # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the @@ -507,14 +997,12 @@ class Specifier(BaseSpecifier): if not prospective < spec: return False - # This special case is here so that, unless the specifier itself - # includes is a pre-release version, that we do not accept pre-release - # versions for the version mentioned in the specifier (e.g. <3.1 should - # not match 3.1.dev0, but should match 3.0.dev0). + # The spec says: "<V MUST NOT allow a pre-release of the specified + # version unless the specified version is itself a pre-release." if ( not spec.is_prerelease and prospective.is_prerelease - and _base_version(prospective) == _base_version(spec) + and prospective >= _earliest_prerelease(spec) ): return False @@ -534,22 +1022,20 @@ class Specifier(BaseSpecifier): if not prospective > spec: return False - # This special case is here so that, unless the specifier itself - # includes is a post-release version, that we do not accept - # post-release versions for the version mentioned in the specifier - # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + # The spec says: ">V MUST NOT allow a post-release of the specified + # version unless the specified version is itself a post-release." if ( not spec.is_postrelease and prospective.is_postrelease - and _base_version(prospective) == _base_version(spec) + and _post_base(prospective) == spec ): return False - # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is technically greater than, to match. - if prospective.local is not None and _base_version( - prospective - ) == _base_version(spec): + # Per the spec: ">V MUST NOT match a local version of the specified + # version". A "local version of V" is any version whose public part + # equals V. So >1.0a1 must not match 1.0a1+local, but must still + # match 1.0a2+local. + if prospective.local is not None and _public_version(prospective) == spec: return False # If we've gotten to here, it means that prospective version is both @@ -608,9 +1094,28 @@ class Specifier(BaseSpecifier): return bool(list(self.filter([item], prereleases=prereleases))) + @typing.overload def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload + def filter( + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + + def filter( + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """Filter items in the given iterable, that match the specifier. :param iterable: @@ -620,6 +1125,10 @@ class Specifier(BaseSpecifier): Whether or not to allow prereleases in the returned iterator. If set to ``None`` (the default), it will follow the recommendation from :pep:`440` and match prereleases if there are no other versions. + :param key: + A callable that takes a single argument (an item from the iterable) and + returns a version string or :class:`Version` instance to be used for + filtering. >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) ['1.3'] @@ -631,6 +1140,10 @@ class Specifier(BaseSpecifier): ['1.3', '1.5a1'] >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) ['1.3', '1.5a1'] + >>> list(Specifier(">=1.2.3").filter( + ... [{"ver": "1.2"}, {"ver": "1.3"}], + ... key=lambda x: x["ver"])) + [{'ver': '1.3'}] """ prereleases_versions = [] found_non_prereleases = False @@ -645,14 +1158,22 @@ class Specifier(BaseSpecifier): # Filter versions for version in iterable: - parsed_version = _coerce_version(version) + parsed_version = _coerce_version(version if key is None else key(version)) + match = False if parsed_version is None: # === operator can match arbitrary (non-version) strings if self.operator == "===" and self._compare_arbitrary( version, self.version ): yield version - elif operator_callable(parsed_version, self.version): + elif self.operator == "===": + match = self._compare_arbitrary( + version if key is None else key(version), self.version + ) + else: + match = operator_callable(parsed_version, self.version) + + if match and parsed_version is not None: # If it's not a prerelease or prereleases are allowed, yield it directly if not parsed_version.is_prerelease or include_prereleases: found_non_prereleases = True @@ -674,6 +1195,48 @@ class Specifier(BaseSpecifier): _prefix_regex = re.compile(r"([0-9]+)((?:a|b|c|rc)[0-9]+)") +def _pep440_filter_prereleases( + iterable: Iterable[Any], key: Callable[[Any], UnparsedVersion] | None +) -> Iterator[Any]: + """Filter per PEP 440: exclude prereleases unless no finals exist.""" + # Two lists used: + # * all_nonfinal to preserve order if no finals exist + # * arbitrary_strings for streaming when first final found + all_nonfinal: list[Any] = [] + arbitrary_strings: list[Any] = [] + + found_final = False + for item in iterable: + parsed = _coerce_version(item if key is None else key(item)) + + if parsed is None: + # Arbitrary strings are always included as it is not + # possible to determine if they are prereleases, + # and they have already passed all specifiers. + if found_final: + yield item + else: + arbitrary_strings.append(item) + all_nonfinal.append(item) + continue + + if not parsed.is_prerelease: + # Final release found - flush arbitrary strings, then yield + if not found_final: + yield from arbitrary_strings + found_final = True + yield item + continue + + # Prerelease - buffer if no finals yet, otherwise skip + if not found_final: + all_nonfinal.append(item) + + # No finals found - yield all buffered items + if not found_final: + yield from all_nonfinal + + def _version_split(version: str) -> list[str]: """Split version into components. @@ -713,25 +1276,59 @@ def _is_not_suffix(segment: str) -> bool: ) -def _pad_version(left: list[str], right: list[str]) -> tuple[list[str], list[str]]: - left_split, right_split = [], [] +def _numeric_prefix_len(split: list[str]) -> int: + """Count leading numeric components in a :func:`_version_split` result. - # Get the release segment of our versions - left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) - right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) + >>> _numeric_prefix_len(["0", "1", "2", "a1"]) + 3 + """ + count = 0 + for segment in split: + if not segment.isdigit(): + break + count += 1 + return count - # Get the rest of our versions - left_split.append(left[len(left_split[0]) :]) - right_split.append(right[len(right_split[0]) :]) - # Insert our padding - left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) - right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) +def _left_pad(split: list[str], target_numeric_len: int) -> list[str]: + """Pad a :func:`_version_split` result with ``"0"`` segments to reach + ``target_numeric_len`` numeric components. Suffix segments are preserved. - return ( - list(itertools.chain.from_iterable(left_split)), - list(itertools.chain.from_iterable(right_split)), - ) + >>> _left_pad(["0", "1", "a1"], 4) + ['0', '1', '0', '0', 'a1'] + """ + numeric_len = _numeric_prefix_len(split) + pad_needed = target_numeric_len - numeric_len + if pad_needed <= 0: + return split + return [*split[:numeric_len], *(["0"] * pad_needed), *split[numeric_len:]] + + +def _operator_cost(op_entry: tuple[CallableOperator, str, str]) -> int: + """Sort key for Cost Based Ordering of specifier operators in _filter_versions. + + Operators run sequentially on a shrinking candidate set, so operators that + reject the most versions should run first to minimize work for later ones. + + Tier 0: Exact equality (==, ===), likely to narrow candidates to one version + Tier 1: Range checks (>=, <=, >, <), cheap and usually reject a large portion + Tier 2: Wildcard equality (==.*) and compatible release (~=), more expensive + Tier 3: Exact !=, cheap but rarely rejects + Tier 4: Wildcard !=.*, expensive and rarely rejects + """ + _, ver, op = op_entry + if op == "==": + return 0 if not ver.endswith(".*") else 2 + if op in (">=", "<=", ">", "<"): + return 1 + if op == "~=": + return 2 + if op == "!=": + return 3 if not ver.endswith(".*") else 4 + if op == "===": + return 0 + + raise ValueError(f"Unknown operator: {op!r}") # pragma: no cover class SpecifierSet(BaseSpecifier): @@ -739,9 +1336,28 @@ class SpecifierSet(BaseSpecifier): It can be passed a single specifier (``>=3.0``), a comma-separated list of specifiers (``>=3.0,!=3.1``), or no specifier at all. + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging + releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with + packaging 26.2+ can be unpickled with future releases. + Backward compatibility with pickles from + packaging < 26.2 is supported but may be removed in a future + release. """ - __slots__ = ("_prereleases", "_specs") + __slots__ = ( + "_canonicalized", + "_has_arbitrary", + "_is_unsatisfiable", + "_prereleases", + "_resolved_ops", + "_specs", + ) def __init__( self, @@ -770,17 +1386,33 @@ class SpecifierSet(BaseSpecifier): # strip each item to remove leading/trailing whitespace. split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] - # Make each individual specifier a Specifier and save in a frozen set - # for later. - self._specs = frozenset(map(Specifier, split_specifiers)) + self._specs: tuple[Specifier, ...] = tuple(map(Specifier, split_specifiers)) + # Fast substring check; avoids iterating parsed specs. + self._has_arbitrary = "===" in specifiers else: - # Save the supplied specifiers in a frozen set. - self._specs = frozenset(specifiers) + self._specs = tuple(specifiers) + # Substring check works for both Specifier objects and plain + # strings (setuptools passes lists of strings). + self._has_arbitrary = any("===" in str(s) for s in self._specs) + + self._canonicalized = len(self._specs) <= 1 + self._resolved_ops: list[tuple[CallableOperator, str, str]] | None = None # Store our prereleases value so we can use it later to determine if # we accept prereleases or not. self._prereleases = prereleases + self._is_unsatisfiable: bool | None = None + + def _canonical_specs(self) -> tuple[Specifier, ...]: + """Deduplicate, sort, and cache specs for order-sensitive operations.""" + if not self._canonicalized: + self._specs = tuple(dict.fromkeys(sorted(self._specs, key=str))) + self._canonicalized = True + self._resolved_ops = None + self._is_unsatisfiable = None + return self._specs + @property def prereleases(self) -> bool | None: # If we have been given an explicit prerelease modifier, then we'll @@ -804,6 +1436,70 @@ class SpecifierSet(BaseSpecifier): @prereleases.setter def prereleases(self, value: bool | None) -> None: self._prereleases = value + self._is_unsatisfiable = None + + def __getstate__(self) -> tuple[tuple[Specifier, ...], bool | None]: + # Return state as a 2-item tuple for compactness: + # (specs, prereleases) + # Cache members are excluded and will be recomputed on demand. + return (self._specs, self._prereleases) + + def __setstate__(self, state: object) -> None: + # Always discard cached values - they will be recomputed on demand. + self._resolved_ops = None + self._is_unsatisfiable = None + + if isinstance(state, tuple): + if len(state) == 2: + # New format (26.2+): (specs, prereleases) + specs, prereleases = state + if ( + isinstance(specs, tuple) + and all(isinstance(s, Specifier) for s in specs) + and _validate_pre(prereleases) + ): + self._specs = specs + self._prereleases = prereleases + self._canonicalized = len(specs) <= 1 + self._has_arbitrary = any("===" in str(s) for s in specs) + return + if len(state) == 2 and isinstance(state[1], dict): + # Format (packaging 26.0-26.1): (None, {slot: value}). + _, slot_dict = state + specs = slot_dict.get("_specs", ()) + prereleases = slot_dict.get("_prereleases") + # Convert frozenset to tuple (26.0 stored as frozenset) + if isinstance(specs, frozenset): + specs = tuple(sorted(specs, key=str)) + if ( + isinstance(specs, tuple) + and all(isinstance(s, Specifier) for s in specs) + and _validate_pre(prereleases) + ): + self._specs = specs + self._prereleases = prereleases + self._canonicalized = len(self._specs) <= 1 + self._has_arbitrary = any("===" in str(s) for s in self._specs) + return + if isinstance(state, dict): + # Old format (packaging <= 25.x, no __slots__): state is a plain dict. + specs = state.get("_specs", ()) + prereleases = state.get("_prereleases") + # Convert frozenset to tuple (26.0 stored as frozenset) + if isinstance(specs, frozenset): + specs = tuple(sorted(specs, key=str)) + if ( + isinstance(specs, tuple) + and all(isinstance(s, Specifier) for s in specs) + and _validate_pre(prereleases) + ): + self._specs = specs + self._prereleases = prereleases + self._canonicalized = len(self._specs) <= 1 + self._has_arbitrary = any("===" in str(s) for s in self._specs) + return + + raise TypeError(f"Cannot restore SpecifierSet from {state!r}") def __repr__(self) -> str: """A representation of the specifier set that shows all internal state. @@ -824,7 +1520,7 @@ class SpecifierSet(BaseSpecifier): else "" ) - return f"<SpecifierSet({str(self)!r}{pre})>" + return f"<{self.__class__.__name__}({str(self)!r}{pre})>" def __str__(self) -> str: """A string representation of the specifier set that can be round-tripped. @@ -837,10 +1533,10 @@ class SpecifierSet(BaseSpecifier): >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) '!=1.0.1,>=1.0.0' """ - return ",".join(sorted(str(s) for s in self._specs)) + return ",".join(str(s) for s in self._canonical_specs()) def __hash__(self) -> int: - return hash(self._specs) + return hash(self._canonical_specs()) def __and__(self, other: SpecifierSet | str) -> SpecifierSet: """Return a SpecifierSet which is a combination of the two sets. @@ -858,13 +1554,15 @@ class SpecifierSet(BaseSpecifier): return NotImplemented specifier = SpecifierSet() - specifier._specs = frozenset(self._specs | other._specs) + specifier._specs = self._specs + other._specs + specifier._canonicalized = len(specifier._specs) <= 1 + specifier._has_arbitrary = self._has_arbitrary or other._has_arbitrary + specifier._resolved_ops = None - if self._prereleases is None and other._prereleases is not None: + # Combine prerelease settings: use common or non-None value + if self._prereleases is None or self._prereleases == other._prereleases: specifier._prereleases = other._prereleases - elif ( - self._prereleases is not None and other._prereleases is None - ) or self._prereleases == other._prereleases: + elif other._prereleases is None: specifier._prereleases = self._prereleases else: raise ValueError( @@ -897,7 +1595,7 @@ class SpecifierSet(BaseSpecifier): elif not isinstance(other, SpecifierSet): return NotImplemented - return self._specs == other._specs + return self._canonical_specs() == other._canonical_specs() def __len__(self) -> int: """Returns the number of specifiers in this specifier set.""" @@ -913,6 +1611,113 @@ class SpecifierSet(BaseSpecifier): """ return iter(self._specs) + def _get_ranges(self) -> Sequence[_VersionRange]: + """Intersect all specifiers into a single list of version ranges. + + Returns an empty list when unsatisfiable. ``===`` specs are + modeled as full range; string matching is checked separately + by :meth:`_check_arbitrary_unsatisfiable`. + """ + specs = self._specs + + result: Sequence[_VersionRange] | None = None + for s in specs: + if result is None: + result = s._to_ranges() + else: + result = _intersect_ranges(result, s._to_ranges()) + if not result: + break + + if result is None: # pragma: no cover + raise RuntimeError("_get_ranges called with no specs") + return result + + def is_unsatisfiable(self) -> bool: + """Check whether this specifier set can never be satisfied. + + Returns True if no version can satisfy all specifiers simultaneously. + + >>> SpecifierSet(">=2.0,<1.0").is_unsatisfiable() + True + >>> SpecifierSet(">=1.0,<2.0").is_unsatisfiable() + False + >>> SpecifierSet("").is_unsatisfiable() + False + >>> SpecifierSet("==1.0,!=1.0").is_unsatisfiable() + True + """ + cached = self._is_unsatisfiable + if cached is not None: + return cached + + if not self._specs: + self._is_unsatisfiable = False + return False + + result = not self._get_ranges() + + if not result: + result = self._check_arbitrary_unsatisfiable() + + if not result and self.prereleases is False: + result = self._check_prerelease_only_ranges() + + self._is_unsatisfiable = result + return result + + def _check_prerelease_only_ranges(self) -> bool: + """With prereleases=False, check if every range contains only + pre-release versions (which would be excluded from matching).""" + for lower, upper in self._get_ranges(): + nearest = _nearest_non_prerelease(lower.version) + if nearest is None: + return False + if upper.version is None or nearest < upper.version: + return False + if nearest == upper.version and upper.inclusive: + return False + return True + + def _check_arbitrary_unsatisfiable(self) -> bool: + """Check === (arbitrary equality) specs for unsatisfiability. + + === uses case-insensitive string comparison, so the only candidate + that can match ``===V`` is the literal string V. This method + checks whether that candidate is excluded by other specifiers. + """ + arbitrary = [s for s in self._specs if s.operator == "==="] + if not arbitrary: + return False + + # Multiple === must agree on the same string (case-insensitive). + first = arbitrary[0].version.lower() + if any(s.version.lower() != first for s in arbitrary[1:]): + return True + + # The sole candidate is the === version string. Check whether + # it can satisfy every standard spec. + candidate = _coerce_version(arbitrary[0].version) + + # With prereleases=False, a prerelease candidate is excluded + # by contains() before the === string check even runs. + if ( + self.prereleases is False + and candidate is not None + and candidate.is_prerelease + ): + return True + + standard = [s for s in self._specs if s.operator != "==="] + if not standard: + return False + + if candidate is None: + # Unparsable string cannot satisfy any standard spec. + return True + + return not all(s.contains(candidate) for s in standard) + def __contains__(self, item: UnparsedVersion) -> bool: """Return whether or not the item is contained in this specifier. @@ -971,12 +1776,36 @@ class SpecifierSet(BaseSpecifier): if version is not None and installed and version.is_prerelease: prereleases = True - check_item = item if version is None else version + # When item is a string and === is involved, keep it as-is + # so the comparison isn't done against the normalized form. + if version is None or (self._has_arbitrary and not isinstance(item, Version)): + check_item = item + else: + check_item = version return bool(list(self.filter([check_item], prereleases=prereleases))) + @typing.overload + def filter( + self, + iterable: Iterable[UnparsedVersionVar], + prereleases: bool | None = None, + key: None = ..., + ) -> Iterator[UnparsedVersionVar]: ... + + @typing.overload def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None - ) -> Iterator[UnparsedVersionVar]: + self, + iterable: Iterable[T], + prereleases: bool | None = None, + key: Callable[[T], UnparsedVersion] = ..., + ) -> Iterator[T]: ... + + def filter( + self, + iterable: Iterable[Any], + prereleases: bool | None = None, + key: Callable[[Any], UnparsedVersion] | None = None, + ) -> Iterator[Any]: """Filter items in the given iterable, that match the specifiers in this set. :param iterable: @@ -986,6 +1815,10 @@ class SpecifierSet(BaseSpecifier): Whether or not to allow prereleases in the returned iterator. If set to ``None`` (the default), it will follow the recommendation from :pep:`440` and match prereleases if there are no other versions. + :param key: + A callable that takes a single argument (an item from the iterable) and + returns a version string or :class:`Version` instance to be used for + filtering. >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) ['1.3'] @@ -997,6 +1830,10 @@ class SpecifierSet(BaseSpecifier): ['1.3', '1.5a1'] >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) ['1.3', '1.5a1'] + >>> list(SpecifierSet(">=1.2.3").filter( + ... [{"ver": "1.2"}, {"ver": "1.3"}], + ... key=lambda x: x["ver"])) + [{'ver': '1.3'}] An "empty" SpecifierSet will filter items based on the presence of prerelease versions in the set. @@ -1016,53 +1853,91 @@ class SpecifierSet(BaseSpecifier): if prereleases is None and self.prereleases is not None: prereleases = self.prereleases - # If we have any specifiers, then we want to wrap our iterable in the - # filter method for each one, this will act as a logical AND amongst - # each specifier. + # Filter versions that match all specifiers using Cost Based Ordering. if self._specs: # When prereleases is None, we need to let all versions through # the individual filters, then decide about prereleases at the end # based on whether any non-prereleases matched ALL specs. - for spec in self._specs: - iterable = spec.filter( - iterable, prereleases=True if prereleases is None else prereleases + + # Fast path: single specifier, delegate directly. + if len(self._specs) == 1: + filtered = self._specs[0].filter( + iterable, + prereleases=True if prereleases is None else prereleases, + key=key, + ) + else: + filtered = self._filter_versions( + iterable, + key, + prereleases=True if prereleases is None else prereleases, ) if prereleases is not None: - # If we have a forced prereleases value, - # we can immediately return the iterator. - return iter(iterable) - else: - # Handle empty SpecifierSet cases where prereleases is not None. - if prereleases is True: - return iter(iterable) + return filtered + + return _pep440_filter_prereleases(filtered, key) - if prereleases is False: - return ( - item - for item in iterable - if (version := _coerce_version(item)) is None + # Handle Empty SpecifierSet. + if prereleases is True: + return iter(iterable) + + if prereleases is False: + return ( + item + for item in iterable + if ( + (version := _coerce_version(item if key is None else key(item))) + is None or not version.is_prerelease ) + ) - # Finally if prereleases is None, apply PEP 440 logic: - # exclude prereleases unless there are no final releases that matched. - filtered_items: list[UnparsedVersionVar] = [] - found_prereleases: list[UnparsedVersionVar] = [] - found_final_release = False + # PEP 440: exclude prereleases unless no final releases matched + return _pep440_filter_prereleases(iterable, key) + + def _filter_versions( + self, + iterable: Iterable[Any], + key: Callable[[Any], UnparsedVersion] | None, + prereleases: bool | None = None, + ) -> Iterator[Any]: + """Filter versions against all specifiers in a single pass. + + Uses Cost Based Ordering: specifiers are sorted by _operator_cost so + that cheap range operators reject versions early, avoiding expensive + wildcard or compatible operators on versions that would have been + rejected anyway. + """ + # Pre-resolve operators and sort (cached after first call). + if self._resolved_ops is None: + self._resolved_ops = sorted( + ( + (spec._get_operator(spec.operator), spec.version, spec.operator) + for spec in self._specs + ), + key=_operator_cost, + ) + ops = self._resolved_ops + exclude_prereleases = prereleases is False for item in iterable: - parsed_version = _coerce_version(item) - # Arbitrary strings are always included as it is not - # possible to determine if they are prereleases, - # and they have already passed all specifiers. - if parsed_version is None: - filtered_items.append(item) - found_prereleases.append(item) - elif parsed_version.is_prerelease: - found_prereleases.append(item) - else: - filtered_items.append(item) - found_final_release = True + parsed = _coerce_version(item if key is None else key(item)) - return iter(filtered_items if found_final_release else found_prereleases) + if parsed is None: + # Only === can match non-parseable versions. + if all( + op == "===" and str(item).lower() == ver.lower() + for _, ver, op in ops + ): + yield item + elif exclude_prereleases and parsed.is_prerelease: + pass + elif all( + str(item if key is None else key(item)).lower() == ver.lower() + if op == "===" + else op_fn(parsed, ver) + for op_fn, ver, op in ops + ): + # Short-circuits on the first failing operator. + yield item diff --git a/contrib/python/pip/pip/_vendor/packaging/tags.py b/contrib/python/pip/pip/_vendor/packaging/tags.py index 5ef27c897a4..979fbd2eedf 100644 --- a/contrib/python/pip/pip/_vendor/packaging/tags.py +++ b/contrib/python/pip/pip/_vendor/packaging/tags.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging +import operator import platform import re import struct @@ -13,20 +14,52 @@ import sys import sysconfig from importlib.machinery import EXTENSION_SUFFIXES from typing import ( - Any, + TYPE_CHECKING, Iterable, Iterator, Sequence, Tuple, + TypeVar, cast, ) from . import _manylinux, _musllinux +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from typing import AbstractSet + + +__all__ = [ + "INTERPRETER_SHORT_NAMES", + "AppleVersion", + "PythonVersion", + "Tag", + "UnsortedTagsError", + "android_platforms", + "compatible_tags", + "cpython_tags", + "create_compatible_tags_selector", + "generic_tags", + "interpreter_name", + "interpreter_version", + "ios_platforms", + "mac_platforms", + "parse_tag", + "platform_tags", + "sys_tags", +] + + +def __dir__() -> list[str]: + return __all__ + + logger = logging.getLogger(__name__) PythonVersion = Sequence[int] AppleVersion = Tuple[int, int] +_T = TypeVar("_T") INTERPRETER_SHORT_NAMES: dict[str, str] = { "python": "py", # Generic. @@ -37,7 +70,19 @@ INTERPRETER_SHORT_NAMES: dict[str, str] = { } -_32_BIT_INTERPRETER = struct.calcsize("P") == 4 +# This function can be unit tested without reloading the module +# (Unlike _32_BIT_INTERPRETER) +def _compute_32_bit_interpreter() -> bool: + return struct.calcsize("P") == 4 + + +_32_BIT_INTERPRETER = _compute_32_bit_interpreter() + + +class UnsortedTagsError(ValueError): + """ + Raised when a tag component is not in sorted order per PEP 425. + """ class Tag: @@ -46,11 +91,29 @@ class Tag: Instances are considered immutable and thus are hashable. Equality checking is also supported. + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with packaging 26.2+ can + be unpickled with future releases. Backward compatibility with pickles + from pip._vendor.packaging < 26.2 is supported but may be removed in a future + release. """ __slots__ = ["_abi", "_hash", "_interpreter", "_platform"] def __init__(self, interpreter: str, abi: str, platform: str) -> None: + """ + :param str interpreter: The interpreter name, e.g. ``"py"`` + (see :attr:`INTERPRETER_SHORT_NAMES` for mapping + well-known interpreter names to their short names). + :param str abi: The ABI that a wheel supports, e.g. ``"cp37m"``. + :param str platform: The OS/platform the wheel supports, + e.g. ``"win_amd64"``. + """ self._interpreter = interpreter.lower() self._abi = abi.lower() self._platform = platform.lower() @@ -63,14 +126,25 @@ class Tag: @property def interpreter(self) -> str: + """ + The interpreter name, e.g. ``"py"`` (see + :attr:`INTERPRETER_SHORT_NAMES` for mapping well-known interpreter + names to their short names). + """ return self._interpreter @property def abi(self) -> str: + """ + The supported ABI. + """ return self._abi @property def platform(self) -> str: + """ + The OS/platform. + """ return self._platform def __eq__(self, other: object) -> bool: @@ -93,23 +167,69 @@ class Tag: def __repr__(self) -> str: return f"<{self} @ {id(self)}>" - def __setstate__(self, state: tuple[None, dict[str, Any]]) -> None: - # The cached _hash is wrong when unpickling. - _, slots = state - for k, v in slots.items(): - setattr(self, k, v) - self._hash = hash((self._interpreter, self._abi, self._platform)) + def __getstate__(self) -> tuple[str, str, str]: + # Return state as a 3-item tuple: (interpreter, abi, platform). + # Cache member _hash is excluded and will be recomputed. + return (self._interpreter, self._abi, self._platform) + def __setstate__(self, state: object) -> None: + if isinstance(state, tuple): + if len(state) == 3 and all(isinstance(s, str) for s in state): + # New format (26.2+): (interpreter, abi, platform) + self._interpreter, self._abi, self._platform = state + self._hash = hash((self._interpreter, self._abi, self._platform)) + return + if len(state) == 2 and isinstance(state[1], dict): + # Old format (packaging <= 26.1, __slots__): (None, {slot: value}). + _, slots = state + try: + interpreter = slots["_interpreter"] + abi = slots["_abi"] + platform = slots["_platform"] + except KeyError: + raise TypeError(f"Cannot restore Tag from {state!r}") from None + if not all( + isinstance(value, str) for value in (interpreter, abi, platform) + ): + raise TypeError(f"Cannot restore Tag from {state!r}") + self._interpreter = interpreter.lower() + self._abi = abi.lower() + self._platform = platform.lower() + self._hash = hash((self._interpreter, self._abi, self._platform)) + return + raise TypeError(f"Cannot restore Tag from {state!r}") -def parse_tag(tag: str) -> frozenset[Tag]: + +def parse_tag(tag: str, *, validate_order: bool = False) -> frozenset[Tag]: """ - Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. + Parses the provided tag (e.g. `py3-none-any`) into a frozenset of + :class:`Tag` instances. Returning a set is required due to the possibility that the tag is a - compressed tag set. + `compressed tag set`_, e.g. ``"py2.py3-none-any"`` which supports both + Python 2 and Python 3. + + If **validate_order** is true, compressed tag set components are checked + to be in sorted order as required by PEP 425. + + :param str tag: The tag to parse, e.g. ``"py3-none-any"``. + :param bool validate_order: Check whether compressed tag set components + are in sorted order. + :raises UnsortedTagsError: If **validate_order** is true and any compressed tag + set component is not in sorted order. + + .. versionadded:: 26.1 + The *validate_order* parameter. """ tags = set() interpreters, abis, platforms = tag.split("-") + if validate_order: + for component in (interpreters, abis, platforms): + parts = component.split(".") + if parts != sorted(parts): + raise UnsortedTagsError( + f"Tag component {component!r} is not in sorted order per PEP 425" + ) for interpreter in interpreters.split("."): for abi in abis.split("."): for platform_ in platforms.split("."): @@ -150,12 +270,25 @@ def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: """ Determine if the Python version supports abi3. - PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) + PEP 384 was first implemented in Python 3.2. The free-threaded builds do not support abi3. """ return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading +def _abi3t_applies(python_version: PythonVersion, threading: bool) -> bool: + """ + Determine if the Python version supports abi3t. + + PEP 803 was first implemented in Python 3.15 but, per PEP 803, this + returns tags going back to Python 3.2 to mirror the abi3 + implementation and leave open the possibility of abi3t wheels + supporting older Python versions. + + """ + return len(python_version) > 1 and tuple(python_version) >= (3, 2) and threading + + def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> list[str]: py_version = tuple(py_version) # To allow for version comparison. abis = [] @@ -197,19 +330,31 @@ def cpython_tags( warn: bool = False, ) -> Iterator[Tag]: """ - Yields the tags for a CPython interpreter. + Yields the tags for the CPython interpreter. + + The specific tags generated are: + + - ``cp<python_version>-<abi>-<platform>`` + - ``cp<python_version>-<stable_abi>-<platform>`` + - ``cp<python_version>-none-<platform>`` + - ``cp<older version>-<stable_abi>-<platform>`` where "older version" is all older + minor versions down to Python 3.2 (when ``abi3`` was introduced) - The tags consist of: - - cp<python_version>-<abi>-<platform> - - cp<python_version>-abi3-<platform> - - cp<python_version>-none-<platform> - - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. + If ``python_version`` only provides a major-only version then only + user-provided ABIs via ``abis`` and the ``none`` ABI will be used. - If python_version only specifies a major version then user-provided ABIs and - the 'none' ABItag will be used. + The ``stable_abi`` will be either ``abi3`` or ``abi3t`` if `abi` is a + GIL-enabled ABI like `"cp315"` or a free-threaded ABI like `"cp315t"`, + respectively. - If 'abi3' or 'none' are specified in 'abis' then they will be yielded at - their normal position and not at the beginning. + :param Sequence python_version: A one- or two-item sequence representing the + targeted Python version. Defaults to + ``sys.version_info[:2]``. + :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs + compatible with the current system. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ if not python_version: python_version = sys.version_info[:2] @@ -233,16 +378,27 @@ def cpython_tags( threading = _is_threaded_cpython(abis) use_abi3 = _abi3_applies(python_version, threading) + use_abi3t = _abi3t_applies(python_version, threading) + if use_abi3: yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) + if use_abi3t: + yield from (Tag(interpreter, "abi3t", platform_) for platform_ in platforms) + yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) - if use_abi3: + if use_abi3 or use_abi3t: for minor_version in range(python_version[1] - 1, 1, -1): for platform_ in platforms: version = _version_nodot((python_version[0], minor_version)) interpreter = f"cp{version}" - yield Tag(interpreter, "abi3", platform_) + if use_abi3: + yield Tag(interpreter, "abi3", platform_) + if use_abi3t: + # Support for abi3t was introduced in Python 3.15, but in + # principle abi3t wheels are possible for older limited API + # versions, so allow things like ("cp37", "abi3t", "platform") + yield Tag(interpreter, "abi3t", platform_) def _generic_abi() -> list[str]: @@ -294,12 +450,25 @@ def generic_tags( warn: bool = False, ) -> Iterator[Tag]: """ - Yields the tags for a generic interpreter. + Yields the tags for an interpreter which requires no specialization. + + This function should be used if one of the other interpreter-specific + functions provided by this module is not appropriate (i.e. not calculating + tags for a CPython interpreter). - The tags consist of: - - <interpreter>-<abi>-<platform> + The specific tags generated are: - The "none" ABI will be added if it was not explicitly provided. + - ``<interpreter>-<abi>-<platform>`` + + The ``"none"`` ABI will be added if it was not explicitly provided. + + :param str interpreter: The name of the interpreter. Defaults to being + calculated. + :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs + compatible with the current system. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ if not interpreter: interp_name = interpreter_name() @@ -335,12 +504,22 @@ def compatible_tags( platforms: Iterable[str] | None = None, ) -> Iterator[Tag]: """ - Yields the sequence of tags that are compatible with a specific version of Python. + Yields the tags for an interpreter compatible with the Python version + specified by ``python_version``. + + The specific tags generated are: + + - ``py*-none-<platform>`` + - ``<interpreter>-none-any`` if ``interpreter`` is provided + - ``py*-none-any`` - The tags consist of: - - py*-none-<platform> - - <interpreter>-none-any # ... if `interpreter` is provided. - - py*-none-any + :param Sequence python_version: A one- or two-item sequence representing the + compatible version of Python. Defaults to + ``sys.version_info[:2]``. + :param str interpreter: The name of the interpreter (if known), e.g. + ``"cp38"``. Defaults to the current interpreter. + :param Iterable platforms: Iterable of compatible platforms. Defaults to the + platforms compatible with the current system. """ if not python_version: python_version = sys.version_info[:2] @@ -400,12 +579,25 @@ def mac_platforms( version: AppleVersion | None = None, arch: str | None = None ) -> Iterator[str]: """ - Yields the platform tags for a macOS system. + Yields the :attr:`~Tag.platform` tags for macOS. The `version` parameter is a two-item tuple specifying the macOS version to generate platform tags for. The `arch` parameter is the CPU architecture to generate platform tags for. Both parameters default to the appropriate value for the current system. + + :param tuple version: A two-item tuple representing the version of macOS. + Defaults to the current system's version. + :param str arch: The CPU architecture. Defaults to the architecture of the + current system, e.g. ``"x86_64"``. + + .. note:: + Equivalent support for the other major platforms is purposefully not + provided: + + - On Windows, platform compatibility is statically specified + - On Linux, code must be run on the system itself to determine + compatibility """ version_str, _, cpu_arch = platform.mac_ver() if version is None: @@ -476,14 +668,19 @@ def ios_platforms( version: AppleVersion | None = None, multiarch: str | None = None ) -> Iterator[str]: """ - Yields the platform tags for an iOS system. - :param version: A two-item tuple specifying the iOS version to generate - platform tags for. Defaults to the current iOS version. - :param multiarch: The CPU architecture+ABI to generate platform tags for - - (the value used by `sys.implementation._multiarch` e.g., - `arm64_iphoneos` or `x84_64_iphonesimulator`). Defaults to the current - multiarch value. + Yields the :attr:`~Tag.platform` tags for iOS. + + :param tuple version: A two-item tuple representing the version of iOS. + Defaults to the current system's version. + :param str multiarch: The CPU architecture+ABI to be used. This should be in + the format by ``sys.implementation._multiarch`` (e.g., + ``arm64_iphoneos`` or ``x86_64_iphonesimulator``). + Defaults to the current system's multiarch value. + + .. note:: + Behavior of this method is undefined if invoked on non-iOS platforms + without providing explicit version and multiarch arguments. """ if version is None: # if iOS is the current platform, ios_ver *must* be defined. However, @@ -585,13 +782,22 @@ def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: yield f"linux_{arch}" +def _emscripten_platforms() -> Iterator[str]: + pyemscripten_platform_version = sysconfig.get_config_var( + "PYEMSCRIPTEN_PLATFORM_VERSION" + ) + if pyemscripten_platform_version: + yield f"pyemscripten_{pyemscripten_platform_version}_wasm32" + yield from _generic_platforms() + + def _generic_platforms() -> Iterator[str]: yield _normalize_string(sysconfig.get_platform()) def platform_tags() -> Iterator[str]: """ - Provides the platform tags for this installation. + Yields the :attr:`~Tag.platform` tags for the running interpreter. """ if platform.system() == "Darwin": return mac_platforms() @@ -601,6 +807,8 @@ def platform_tags() -> Iterator[str]: return android_platforms() elif platform.system() == "Linux": return _linux_platforms() + elif platform.system() == "Emscripten": + return _emscripten_platforms() else: return _generic_platforms() @@ -611,6 +819,8 @@ def interpreter_name() -> str: Some implementations have a reserved, two-letter abbreviation which will be returned when appropriate. + + This typically acts as the prefix to the :attr:`~Tag.interpreter` tag. """ name = sys.implementation.name return INTERPRETER_SHORT_NAMES.get(name) or name @@ -618,7 +828,11 @@ def interpreter_name() -> str: def interpreter_version(*, warn: bool = False) -> str: """ - Returns the version of the running interpreter. + Returns the running interpreter's version. + + This typically acts as the suffix to the :attr:`~Tag.interpreter` tag. + + :param bool warn: Whether warnings should be logged. Defaults to ``False``. """ version = _get_config_var("py_version_nodot", warn=warn) return str(version) if version else _version_nodot(sys.version_info[:2]) @@ -630,10 +844,31 @@ def _version_nodot(version: PythonVersion) -> str: def sys_tags(*, warn: bool = False) -> Iterator[Tag]: """ - Returns the sequence of tag triples for the running interpreter. + Yields the sequence of tag triples that the running interpreter supports. + + The iterable is ordered so that the best-matching tag is first in the + sequence. The exact preferential order to tags is interpreter-specific, but + in general the tag importance is in the order of: - The order of the sequence corresponds to priority order for the - interpreter, from most to least important. + 1. Interpreter + 2. Platform + 3. ABI + + This order is due to the fact that an ABI is inherently tied to the + platform, but platform-specific code is not necessarily tied to the ABI. The + interpreter is the most important tag as it dictates basic support for any + wheel. + + The function returns an iterable in order to allow for the possible + short-circuiting of tag generation if the entire sequence is not necessary + and tag calculation happens to be expensive. + + :param bool warn: Whether warnings should be logged. Defaults to ``False``. + + .. versionchanged:: 21.3 + Added the `pp3-none-any` tag (:issue:`311`). + .. versionchanged:: 27.0 + Added the `abi3t` tag (:issue:`1099`). """ interp_name = interpreter_name() @@ -649,3 +884,49 @@ def sys_tags(*, warn: bool = False) -> Iterator[Tag]: else: interp = None yield from compatible_tags(interpreter=interp) + + +def create_compatible_tags_selector( + tags: Iterable[Tag], +) -> Callable[[Iterable[tuple[_T, AbstractSet[Tag]]]], Iterator[_T]]: + """Create a callable to select things compatible with supported tags. + + This function accepts an ordered sequence of tags, with the preferred + tags first. + + The returned callable accepts an iterable of tuples (thing, set[Tag]), + and returns an iterator of things, with the things with the best + matching tags first. + + Example to select compatible wheel filenames: + + >>> from packaging import tags + >>> from packaging.utils import parse_wheel_filename + >>> selector = tags.create_compatible_tags_selector(tags.sys_tags()) + >>> filenames = ["foo-1.0-py3-none-any.whl", "foo-1.0-py2-none-any.whl"] + >>> list(selector([ + ... (filename, parse_wheel_filename(filename)[-1]) for filename in filenames + ... ])) + ['foo-1.0-py3-none-any.whl'] + + .. versionadded:: 26.1 + """ + tag_ranks: dict[Tag, int] = {} + for rank, tag in enumerate(tags): + tag_ranks.setdefault(tag, rank) # ignore duplicate tags, keep first + supported_tags = tag_ranks.keys() + + def selector( + tagged_things: Iterable[tuple[_T, AbstractSet[Tag]]], + ) -> Iterator[_T]: + ranked_things: list[tuple[_T, int]] = [] + for thing, thing_tags in tagged_things: + supported_thing_tags = thing_tags & supported_tags + if supported_thing_tags: + thing_rank = min(tag_ranks[t] for t in supported_thing_tags) + ranked_things.append((thing, thing_rank)) + return iter( + thing for thing, _ in sorted(ranked_things, key=operator.itemgetter(1)) + ) + + return selector diff --git a/contrib/python/pip/pip/_vendor/packaging/utils.py b/contrib/python/pip/pip/_vendor/packaging/utils.py index c41c8137f26..cbd3be27c42 100644 --- a/contrib/python/pip/pip/_vendor/packaging/utils.py +++ b/contrib/python/pip/pip/_vendor/packaging/utils.py @@ -7,11 +7,33 @@ from __future__ import annotations import re from typing import NewType, Tuple, Union, cast -from .tags import Tag, parse_tag +from .tags import Tag, UnsortedTagsError, parse_tag from .version import InvalidVersion, Version, _TrimmedRelease +__all__ = [ + "BuildTag", + "InvalidName", + "InvalidSdistFilename", + "InvalidWheelFilename", + "NormalizedName", + "canonicalize_name", + "canonicalize_version", + "is_normalized_name", + "parse_sdist_filename", + "parse_wheel_filename", +] + + +def __dir__() -> list[str]: + return __all__ + + BuildTag = Union[Tuple[()], Tuple[int, str]] + NormalizedName = NewType("NormalizedName", str) +""" +A :class:`typing.NewType` of :class:`str`, representing a normalized name. +""" class InvalidName(ValueError): @@ -33,13 +55,39 @@ class InvalidSdistFilename(ValueError): # Core metadata spec for `Name` -_validate_regex = re.compile(r"[A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9]", re.IGNORECASE) -_normalized_regex = re.compile(r"[a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9]") +_validate_regex = re.compile( + r"[a-z0-9]|[a-z0-9][a-z0-9._-]*[a-z0-9]", re.IGNORECASE | re.ASCII +) +_normalized_regex = re.compile(r"[a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9]", re.ASCII) # PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") +_build_tag_regex = re.compile(r"(\d+)(.*)", re.ASCII) def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: + """ + This function takes a valid Python package or extra name, and returns the + normalized form of it. + + The return type is typed as :class:`NormalizedName`. This allows type + checkers to help require that a string has passed through this function + before use. + + If **validate** is true, then the function will check if **name** is a valid + distribution name before normalizing. + + :param str name: The name to normalize. + :param bool validate: Check whether the name is a valid distribution name. + :raises InvalidName: If **validate** is true and the name is not an + acceptable distribution name. + + >>> from packaging.utils import canonicalize_name + >>> canonicalize_name("Django") + 'django' + >>> canonicalize_name("oslo.concurrency") + 'oslo-concurrency' + >>> canonicalize_name("requests") + 'requests' + """ if validate and not _validate_regex.fullmatch(name): raise InvalidName(f"name is invalid: {name!r}") # Ensure all ``.`` and ``_`` are ``-`` @@ -53,15 +101,32 @@ def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: def is_normalized_name(name: str) -> bool: + """ + Check if a name is already normalized (i.e. :func:`canonicalize_name` would + roundtrip to the same value). + + :param str name: The name to check. + + >>> from packaging.utils import is_normalized_name + >>> is_normalized_name("requests") + True + >>> is_normalized_name("Django") + False + """ return _normalized_regex.fullmatch(name) is not None def canonicalize_version( version: Version | str, *, strip_trailing_zero: bool = True ) -> str: - """ - Return a canonical form of a version as a string. + """Return a canonical form of a version as a string. + This function takes a string representing a package version (or a + :class:`~packaging.version.Version` instance), and returns the + normalized form of it. By default, it strips trailing zeros from + the release segment. + + >>> from packaging.utils import canonicalize_version >>> canonicalize_version('1.0.1') '1.0.1' @@ -77,6 +142,9 @@ def canonicalize_version( >>> canonicalize_version('foo bar baz') 'foo bar baz' + + >>> canonicalize_version('1.4.0.0.0') + '1.4' """ if isinstance(version, str): try: @@ -88,7 +156,49 @@ def canonicalize_version( def parse_wheel_filename( filename: str, + *, + validate_order: bool = False, ) -> tuple[NormalizedName, Version, BuildTag, frozenset[Tag]]: + """ + This function takes the filename of a wheel file, and parses it, + returning a tuple of name, version, build number, and tags. + + The name part of the tuple is normalized and typed as + :class:`NormalizedName`. The version portion is an instance of + :class:`~packaging.version.Version`. The build number is ``()`` if + there is no build number in the wheel filename, otherwise a + two-item tuple of an integer for the leading digits and + a string for the rest of the build number. The tags portion is a + frozen set of :class:`~packaging.tags.Tag` instances (as the tag + string format allows multiple tags to be combined into a single + string). + + If **validate_order** is true, compressed tag set components are + checked to be in sorted order as required by PEP 425. + + :param str filename: The name of the wheel file. + :param bool validate_order: Check whether compressed tag set components + are in sorted order. + :raises InvalidWheelFilename: If the filename in question + does not follow the :ref:`wheel specification + <pypug:binary-distribution-format>`. + + >>> from packaging.utils import parse_wheel_filename + >>> from packaging.tags import Tag + >>> from packaging.version import Version + >>> name, ver, build, tags = parse_wheel_filename("foo-1.0-py3-none-any.whl") + >>> name + 'foo' + >>> ver == Version('1.0') + True + >>> tags == {Tag("py3", "none", "any")} + True + >>> not build + True + + .. versionadded:: 26.1 + The *validate_order* parameter. + """ if not filename.endswith(".whl"): raise InvalidWheelFilename( f"Invalid wheel filename (extension must be '.whl'): {filename!r}" @@ -125,11 +235,39 @@ def parse_wheel_filename( build = cast("BuildTag", (int(build_match.group(1)), build_match.group(2))) else: build = () - tags = parse_tag(parts[-1]) + tag_str = parts[-1] + try: + tags = parse_tag(tag_str, validate_order=validate_order) + except UnsortedTagsError: + raise InvalidWheelFilename( + f"Invalid wheel filename (compressed tag set components must be in " + f"sorted order per PEP 425): {filename!r}" + ) from None return (name, version, build, tags) def parse_sdist_filename(filename: str) -> tuple[NormalizedName, Version]: + """ + This function takes the filename of a sdist file (as specified + in the `Source distribution format`_ documentation), and parses + it, returning a tuple of the normalized name and version as + represented by an instance of :class:`~packaging.version.Version`. + + :param str filename: The name of the sdist file. + :raises InvalidSdistFilename: If the filename does not end + with an sdist extension (``.zip`` or ``.tar.gz``), or if it does not + contain a dash separating the name and the version of the distribution. + + >>> from packaging.utils import parse_sdist_filename + >>> from packaging.version import Version + >>> name, ver = parse_sdist_filename("foo-1.0.tar.gz") + >>> name + 'foo' + >>> ver == Version('1.0') + True + + .. _Source distribution format: https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-name + """ if filename.endswith(".tar.gz"): file_stem = filename[: -len(".tar.gz")] elif filename.endswith(".zip"): diff --git a/contrib/python/pip/pip/_vendor/packaging/version.py b/contrib/python/pip/pip/_vendor/packaging/version.py index d250a2627d7..905b47e65c4 100644 --- a/contrib/python/pip/pip/_vendor/packaging/version.py +++ b/contrib/python/pip/pip/_vendor/packaging/version.py @@ -4,7 +4,7 @@ """ .. testsetup:: - from pip._vendor.packaging.version import parse, Version + from pip._vendor.packaging.version import parse, normalize_pre, Version, _cmpkey """ from __future__ import annotations @@ -23,8 +23,6 @@ from typing import ( Union, ) -from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType - if typing.TYPE_CHECKING: from typing_extensions import Self, Unpack @@ -37,7 +35,7 @@ else: # pragma: no cover import warnings def _deprecated(message: str) -> object: - def decorator(func: object) -> object: + def decorator(func: Callable[[...], object]) -> object: @functools.wraps(func) def wrapper(*args: object, **kwargs: object) -> object: warnings.warn( @@ -62,22 +60,20 @@ _LETTER_NORMALIZATION = { "r": "post", } -__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "parse"] +__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "normalize_pre", "parse"] + + +def __dir__() -> list[str]: + return __all__ + LocalType = Tuple[Union[int, str], ...] -CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] -CmpLocalType = Union[ - NegativeInfinityType, - Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], -] -CmpKey = Tuple[ - int, - Tuple[int, ...], - CmpPrePostDevType, - CmpPrePostDevType, - CmpPrePostDevType, - CmpLocalType, +CmpLocalType = Tuple[Tuple[int, str], ...] +CmpSuffix = Tuple[int, int, int, int, int, int] +CmpKey = Union[ + Tuple[int, Tuple[int, ...], CmpSuffix], + Tuple[int, Tuple[int, ...], CmpSuffix, CmpLocalType], ] VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] @@ -85,15 +81,38 @@ VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] class _VersionReplace(TypedDict, total=False): epoch: int | None release: tuple[int, ...] | None - pre: tuple[Literal["a", "b", "rc"], int] | None + pre: tuple[str, int] | None post: int | None dev: int | None local: str | None +def normalize_pre(letter: str, /) -> str: + """Normalize the pre-release segment of a version string. + + Returns a lowercase version of the string if not a known pre-release + identifier. + + >>> normalize_pre('alpha') + 'a' + >>> normalize_pre('BETA') + 'b' + >>> normalize_pre('rc') + 'rc' + + :param letter: + + .. versionadded:: 26.1 + """ + letter = letter.lower() + return _LETTER_NORMALIZATION.get(letter, letter) + + def parse(version: str) -> Version: """Parse the given version string. + This is identical to the :class:`Version` constructor. + >>> parse('1.0.dev1') <Version('1.0.dev1')> @@ -173,7 +192,7 @@ class _BaseVersion: # Note that ++ doesn't behave identically on CPython and PyPy, so not using it here _VERSION_PATTERN = r""" v?+ # optional leading v - (?: + (?a: (?:(?P<epoch>[0-9]+)!)?+ # epoch (?P<release>[0-9]+(?:\.[0-9]+)*+) # release segment (?P<pre> # pre-release @@ -199,7 +218,7 @@ _VERSION_PATTERN = r""" (?P<dev_n>[0-9]+)? )?+ ) - (?:\+ + (?a:\+ (?P<local> # local version [a-z0-9]+ (?:[._-][a-z0-9]+)*+ @@ -227,12 +246,21 @@ expressions (for example, matching a version number as part of a file name). The regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` flags set. +.. versionchanged:: 26.0 + + The regex now uses possessive qualifiers on Python 3.11 if they are + supported (CPython 3.11.5+, PyPy 3.11.13+). + :meta hide-value: """ # Validation pattern for local version in replace() -_LOCAL_PATTERN = re.compile(r"[a-z0-9]+(?:[._-][a-z0-9]+)*", re.IGNORECASE) +_LOCAL_PATTERN = re.compile(r"[a-z0-9]+(?:[._-][a-z0-9]+)*", re.IGNORECASE | re.ASCII) + +# Fast path: If a version has only digits and dots then we +# can skip the regex and parse it as a release segment +_SIMPLE_VERSION_INDICATORS = frozenset(".0123456789") def _validate_epoch(value: object, /) -> int: @@ -258,14 +286,12 @@ def _validate_release(value: object, /) -> tuple[int, ...]: def _validate_pre(value: object, /) -> tuple[Literal["a", "b", "rc"], int] | None: if value is None: return value - if ( - isinstance(value, tuple) - and len(value) == 2 - and value[0] in ("a", "b", "rc") - and isinstance(value[1], int) - and value[1] >= 0 - ): - return value + if isinstance(value, tuple) and len(value) == 2: + letter, number = value + letter = normalize_pre(letter) + if letter in {"a", "b", "rc"} and isinstance(number, int) and number >= 0: + # type checkers can't infer the Literal type here on letter + return (letter, number) # type: ignore[return-value] msg = f"pre must be a tuple of ('a'|'b'|'rc', non-negative int), got {value}" raise InvalidVersion(msg) @@ -301,9 +327,9 @@ def _validate_local(value: object, /) -> LocalType | None: class _Version(NamedTuple): epoch: int release: tuple[int, ...] - dev: tuple[str, int] | None - pre: tuple[str, int] | None - post: tuple[str, int] | None + dev: tuple[Literal["dev"], int] | None + pre: tuple[Literal["a", "b", "rc"], int] | None + post: tuple[Literal["post"], int] | None local: LocalType | None @@ -329,20 +355,50 @@ class Version(_BaseVersion): False >>> v1 <= v2 True + + :class:`Version` is immutable; use :meth:`__replace__` to change + part of a version. + + Instances are safe to serialize with :mod:`pickle`. They use a stable + format so the same pickle can be loaded in future packaging releases. + + .. versionchanged:: 26.2 + + Added a stable pickle format. Pickles created with packaging 26.2+ can + be unpickled with future releases. Backward compatibility with pickles + from pip._vendor.packaging < 26.2 is supported but may be removed in a future + release. """ - __slots__ = ("_dev", "_epoch", "_key_cache", "_local", "_post", "_pre", "_release") + __slots__ = ( + "_dev", + "_epoch", + "_hash_cache", + "_key_cache", + "_local", + "_post", + "_pre", + "_release", + ) __match_args__ = ("_str",) + """ + Pattern matching is supported on Python 3.10+. + + .. versionadded:: 26.0 + + :meta hide-value: + """ _regex = re.compile(r"\s*" + VERSION_PATTERN + r"\s*", re.VERBOSE | re.IGNORECASE) _epoch: int _release: tuple[int, ...] - _dev: tuple[str, int] | None - _pre: tuple[str, int] | None - _post: tuple[str, int] | None + _dev: tuple[Literal["dev"], int] | None + _pre: tuple[Literal["a", "b", "rc"], int] | None + _post: tuple[Literal["post"], int] | None _local: LocalType | None + _hash_cache: int | None _key_cache: CmpKey | None def __init__(self, version: str) -> None: @@ -355,23 +411,118 @@ class Version(_BaseVersion): If the ``version`` does not conform to PEP 440 in any way then this exception will be raised. """ + if _SIMPLE_VERSION_INDICATORS.issuperset(version): + try: + self._release = tuple(map(int, version.split("."))) + except ValueError: + # Empty parts (from "1..2", ".1", etc.) are invalid versions. + # Any other ValueError (e.g. int str-digits limit) should + # propagate to the caller. + if "" in version.split("."): + raise InvalidVersion(f"Invalid version: {version!r}") from None + # TODO: remove "no cover" when Python 3.9 is dropped. + raise # pragma: no cover + + self._epoch = 0 + self._pre = None + self._post = None + self._dev = None + self._local = None + self._key_cache = None + self._hash_cache = None + return + # Validate the version and parse it into pieces match = self._regex.fullmatch(version) if not match: raise InvalidVersion(f"Invalid version: {version!r}") self._epoch = int(match.group("epoch")) if match.group("epoch") else 0 self._release = tuple(map(int, match.group("release").split("."))) - self._pre = _parse_letter_version(match.group("pre_l"), match.group("pre_n")) - self._post = _parse_letter_version( + # We can type ignore the assignments below because the regex guarantees + # the correct strings + self._pre = _parse_letter_version(match.group("pre_l"), match.group("pre_n")) # type: ignore[assignment] + self._post = _parse_letter_version( # type: ignore[assignment] match.group("post_l"), match.group("post_n1") or match.group("post_n2") ) - self._dev = _parse_letter_version(match.group("dev_l"), match.group("dev_n")) + self._dev = _parse_letter_version(match.group("dev_l"), match.group("dev_n")) # type: ignore[assignment] self._local = _parse_local_version(match.group("local")) # Key which will be used for sorting self._key_cache = None + self._hash_cache = None + + @classmethod + def from_parts( + cls, + *, + epoch: int = 0, + release: tuple[int, ...], + pre: tuple[str, int] | None = None, + post: int | None = None, + dev: int | None = None, + local: str | None = None, + ) -> Self: + """ + Return a new version composed of the various parts. + + This allows you to build a version without going though a string and + running a regular expression. It normalizes pre-release strings. The + ``release=`` keyword argument is required. + + >>> Version.from_parts(release=(1,2,3)) + <Version('1.2.3')> + >>> Version.from_parts(release=(0,1,0), pre=("b", 1)) + <Version('0.1.0b1')> + + :param epoch: + :param release: This version tuple is required + + .. versionadded:: 26.1 + """ + _epoch = _validate_epoch(epoch) + _release = _validate_release(release) + _pre = _validate_pre(pre) if pre is not None else None + _post = _validate_post(post) if post is not None else None + _dev = _validate_dev(dev) if dev is not None else None + _local = _validate_local(local) if local is not None else None + + new_version = cls.__new__(cls) + new_version._key_cache = None + new_version._hash_cache = None + new_version._epoch = _epoch + new_version._release = _release + new_version._pre = _pre + new_version._post = _post + new_version._dev = _dev + new_version._local = _local + + return new_version def __replace__(self, **kwargs: Unpack[_VersionReplace]) -> Self: + """ + __replace__(*, epoch=..., release=..., pre=..., post=..., dev=..., local=...) + + Return a new version with parts replaced. + + This returns a new version (unless no parts were changed). The + pre-release is normalized. Setting a value to ``None`` clears it. + + >>> v = Version("1.2.3") + >>> v.__replace__(pre=("a", 1)) + <Version('1.2.3a1')> + + :param int | None epoch: + :param tuple[int, ...] | None release: + :param tuple[str, int] | None pre: + :param int | None post: + :param int | None dev: + :param str | None local: + + .. versionadded:: 26.0 + .. versionchanged:: 26.1 + + The pre-release portion is now normalized. + """ epoch = _validate_epoch(kwargs["epoch"]) if "epoch" in kwargs else self._epoch release = ( _validate_release(kwargs["release"]) @@ -395,6 +546,7 @@ class Version(_BaseVersion): new_version = self.__class__.__new__(self.__class__) new_version._key_cache = None + new_version._hash_cache = None new_version._epoch = epoch new_version._release = release new_version._pre = pre @@ -417,6 +569,255 @@ class Version(_BaseVersion): ) return self._key_cache + # __hash__ must be defined when __eq__ is overridden, + # otherwise Python sets __hash__ to None. + def __hash__(self) -> int: + if (cached_hash := self._hash_cache) is not None: + return cached_hash + + if (key := self._key_cache) is None: + self._key_cache = key = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + self._hash_cache = cached_hash = hash(key) + return cached_hash + + # Override comparison methods to use direct _key_cache access + # This is faster than property access, especially before Python 3.12 + def __lt__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache < other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__lt__(other) + + def __le__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache <= other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__le__(other) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache == other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__eq__(other) + + def __ge__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache >= other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__ge__(other) + + def __gt__(self, other: _BaseVersion) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache > other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__gt__(other) + + def __ne__(self, other: object) -> bool: + if isinstance(other, Version): + if self._key_cache is None: + self._key_cache = _cmpkey( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + if other._key_cache is None: + other._key_cache = _cmpkey( + other._epoch, + other._release, + other._pre, + other._post, + other._dev, + other._local, + ) + return self._key_cache != other._key_cache + + if not isinstance(other, _BaseVersion): + return NotImplemented + + return super().__ne__(other) + + def __getstate__( + self, + ) -> tuple[ + int, + tuple[int, ...], + tuple[str, int] | None, + tuple[str, int] | None, + tuple[str, int] | None, + LocalType | None, + ]: + # Return state as a 6-item tuple for compactness: + # (epoch, release, pre, post, dev, local) + # Cache members are excluded and will be recomputed on demand + return ( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) + + def __setstate__(self, state: object) -> None: + # Always discard cached values — they may contain stale references + # (e.g. packaging._structures.InfinityType from pre-26.1 pickles) + # and will be recomputed on demand from the core fields above. + self._key_cache = None + self._hash_cache = None + + if isinstance(state, tuple): + if len(state) == 6: + # New format (26.2+): (epoch, release, pre, post, dev, local) + ( + self._epoch, + self._release, + self._pre, + self._post, + self._dev, + self._local, + ) = state + return + if len(state) == 2: + # Format (packaging 26.0-26.1): (None, {slot: value}). + _, slot_dict = state + if isinstance(slot_dict, dict): + self._epoch = slot_dict["_epoch"] + self._release = slot_dict["_release"] + self._pre = slot_dict.get("_pre") + self._post = slot_dict.get("_post") + self._dev = slot_dict.get("_dev") + self._local = slot_dict.get("_local") + return + if isinstance(state, dict): + # Old format (packaging <= 25.x, no __slots__): state is a plain + # dict with "_version" (_Version NamedTuple) and "_key" entries. + version_nt = state.get("_version") + if version_nt is not None: + self._epoch = version_nt.epoch + self._release = version_nt.release + self._pre = version_nt.pre + self._post = version_nt.post + self._dev = version_nt.dev + self._local = version_nt.local + return + + raise TypeError(f"Cannot restore Version from {state!r}") + @property @_deprecated("Version._version is private and will be removed soon") def _version(self) -> _Version: @@ -434,6 +835,7 @@ class Version(_BaseVersion): self._post = value.post self._local = value.local self._key_cache = None + self._hash_cache = None def __repr__(self) -> str: """A representation of the Version that shows all internal state. @@ -441,7 +843,7 @@ class Version(_BaseVersion): >>> Version('1.0.0') <Version('1.0.0')> """ - return f"<Version('{self}')>" + return f"<{self.__class__.__name__}({str(self)!r})>" def __str__(self) -> str: """A string representation of the version that can be round-tripped. @@ -507,7 +909,7 @@ class Version(_BaseVersion): return self._release @property - def pre(self) -> tuple[str, int] | None: + def pre(self) -> tuple[Literal["a", "b", "rc"], int] | None: """The pre-release segment of the version. >>> print(Version("1.2.3").pre) @@ -561,6 +963,9 @@ class Version(_BaseVersion): def public(self) -> str: """The public portion of the version. + This returns a string. If you want a :class:`Version` again and care + about performance, use ``v.__replace__(local=None)`` instead. + >>> Version("1.2.3").public '1.2.3' >>> Version("1.2.3+abc").public @@ -574,6 +979,10 @@ class Version(_BaseVersion): def base_version(self) -> str: """The "base version" of the version. + This returns a string. If you want a :class:`Version` again and care + about performance, use + ``v.__replace__(pre=None, post=None, dev=None, local=None)`` instead. + >>> Version("1.2.3").base_version '1.2.3' >>> Version("1.2.3+abc").base_version @@ -721,7 +1130,8 @@ _local_version_separators = re.compile(r"[\._-]") def _parse_local_version(local: str | None) -> LocalType | None: """ - Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). + Takes a string like ``"abc.1.twelve"`` and turns it into + ``("abc", 1, "twelve")``. """ if local is not None: return tuple( @@ -731,6 +1141,19 @@ def _parse_local_version(local: str | None) -> LocalType | None: return None +# Sort ranks for pre-release: dev-only < a < b < rc < stable (no pre-release). +_PRE_RANK = {"a": 0, "b": 1, "rc": 2} +_PRE_RANK_DEV_ONLY = -1 # sorts before a(0) +_PRE_RANK_STABLE = 3 # sorts after rc(2) + +# In local version segments, strings sort before ints per PEP 440. +_LOCAL_STR_RANK = -1 # sorts before all non-negative ints + +# Pre-computed suffix for stable releases (no pre, post, or dev segments). +# See _cmpkey() for the suffix layout. +_STABLE_SUFFIX = (_PRE_RANK_STABLE, 0, 0, 0, 1, 0) + + def _cmpkey( epoch: int, release: tuple[int, ...], @@ -739,54 +1162,70 @@ def _cmpkey( dev: tuple[str, int] | None, local: LocalType | None, ) -> CmpKey: - # When we compare a release version, we want to compare it with all of the - # trailing zeros removed. We will use this for our sorting key. + """Build a comparison key for PEP 440 ordering. + + Returns ``(epoch, release, suffix)`` or + ``(epoch, release, suffix, local)`` so that plain tuple + comparison gives the correct order. + + Trailing zeros are stripped from the release so that ``1.0.0 == 1``. + + The suffix is a flat 6-int tuple that encodes pre/post/dev: + ``(pre_rank, pre_n, post_rank, post_n, dev_rank, dev_n)`` + + pre_rank: dev-only=-1, a=0, b=1, rc=2, no-pre=3 + Dev-only releases (no pre or post) get -1 so they sort before + any alpha/beta/rc. Releases without a pre-release tag get 3 + so they sort after rc. + post_rank: no-post=0, post=1 + Releases without a post segment sort before those with one. + dev_rank: dev=0, no-dev=1 + Releases without a dev segment sort after those with one. + + Local segments use ``(n, "")`` for ints and ``(-1, s)`` for strings, + following PEP 440: strings sort before ints, strings compare + lexicographically, ints compare numerically, and shorter segments + sort before longer when prefixes match. Versions without a local + segment sort before those with one (3-tuple < 4-tuple). + + >>> _cmpkey(0, (1, 0, 0), None, None, None, None) + (0, (1,), (3, 0, 0, 0, 1, 0)) + >>> _cmpkey(0, (1,), ("a", 1), None, None, None) + (0, (1,), (0, 1, 0, 0, 1, 0)) + >>> _cmpkey(0, (1,), None, None, None, ("ubuntu", 1)) + (0, (1,), (3, 0, 0, 0, 1, 0), ((-1, 'ubuntu'), (1, ''))) + """ + # Strip trailing zeros: 1.0.0 compares equal to 1. len_release = len(release) i = len_release while i and release[i - 1] == 0: i -= 1 - _release = release if i == len_release else release[:i] + trimmed = release if i == len_release else release[:i] + + # Fast path: stable release with no local segment. + if pre is None and post is None and dev is None and local is None: + return epoch, trimmed, _STABLE_SUFFIX - # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. - # We'll do this by abusing the pre segment, but we _only_ want to do this - # if there is not a pre or a post segment. If we have one of those then - # the normal sorting rules will handle this case correctly. if pre is None and post is None and dev is not None: - _pre: CmpPrePostDevType = NegativeInfinity - # Versions without a pre-release (except as noted above) should sort after - # those with one. + # dev-only (e.g. 1.0.dev1) sorts before all pre-releases. + pre_rank, pre_n = _PRE_RANK_DEV_ONLY, 0 elif pre is None: - _pre = Infinity + pre_rank, pre_n = _PRE_RANK_STABLE, 0 else: - _pre = pre + pre_rank, pre_n = _PRE_RANK[pre[0]], pre[1] - # Versions without a post segment should sort before those with one. - if post is None: - _post: CmpPrePostDevType = NegativeInfinity - - else: - _post = post + post_rank = 0 if post is None else 1 + post_n = 0 if post is None else post[1] - # Versions without a development segment should sort after those with one. - if dev is None: - _dev: CmpPrePostDevType = Infinity + dev_rank = 1 if dev is None else 0 + dev_n = 0 if dev is None else dev[1] - else: - _dev = dev + suffix = (pre_rank, pre_n, post_rank, post_n, dev_rank, dev_n) if local is None: - # Versions without a local segment should sort before those with one. - _local: CmpLocalType = NegativeInfinity - else: - # Versions with a local segment need that segment parsed to implement - # the sorting rules in PEP440. - # - Alpha numeric segments sort before numeric segments - # - Alpha numeric segments sort lexicographically - # - Numeric segments sort numerically - # - Shorter versions sort before longer versions when the prefixes - # match exactly - _local = tuple( - (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local - ) + return epoch, trimmed, suffix - return epoch, _release, _pre, _post, _dev, _local + cmp_local: CmpLocalType = tuple( + (seg, "") if isinstance(seg, int) else (_LOCAL_STR_RANK, seg) for seg in local + ) + return epoch, trimmed, suffix, cmp_local diff --git a/contrib/python/pip/pip/_vendor/requests/__init__.py b/contrib/python/pip/pip/_vendor/requests/__init__.py index 04230fc8d9a..81d38590ad2 100644 --- a/contrib/python/pip/pip/_vendor/requests/__init__.py +++ b/contrib/python/pip/pip/_vendor/requests/__init__.py @@ -68,8 +68,8 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver if chardet_version: major, minor, patch = chardet_version.split(".")[:3] major, minor, patch = int(major), int(minor), int(patch) - # chardet_version >= 3.0.2, < 6.0.0 - assert (3, 0, 2) <= (major, minor, patch) < (6, 0, 0) + # chardet_version >= 3.0.2, < 8.0.0 + assert (3, 0, 2) <= (major, minor, patch) < (8, 0, 0) elif charset_normalizer_version: major, minor, patch = charset_normalizer_version.split(".")[:3] major, minor, patch = int(major), int(minor), int(patch) @@ -88,8 +88,8 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = "Old version of cryptography ({}) may cause slowdown.".format( - cryptography_version + warning = ( + f"Old version of cryptography ({cryptography_version}) may cause slowdown." ) warnings.warn(warning, RequestsDependencyWarning) @@ -101,10 +101,9 @@ try: ) except (AssertionError, ValueError): warnings.warn( - "urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " - "version!".format( - urllib3.__version__, chardet_version, charset_normalizer_version - ), + f"urllib3 ({urllib3.__version__}) or chardet " + f"({chardet_version})/charset_normalizer ({charset_normalizer_version}) " + "doesn't match a supported version!", RequestsDependencyWarning, ) diff --git a/contrib/python/pip/pip/_vendor/requests/__version__.py b/contrib/python/pip/pip/_vendor/requests/__version__.py index effdd98cf15..7f8a52c8565 100644 --- a/contrib/python/pip/pip/_vendor/requests/__version__.py +++ b/contrib/python/pip/pip/_vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = "requests" __description__ = "Python HTTP for Humans." __url__ = "https://requests.readthedocs.io" -__version__ = "2.32.5" -__build__ = 0x023205 +__version__ = "2.33.1" +__build__ = 0x023301 __author__ = "Kenneth Reitz" __author_email__ = "[email protected]" __license__ = "Apache-2.0" diff --git a/contrib/python/pip/pip/_vendor/requests/_internal_utils.py b/contrib/python/pip/pip/_vendor/requests/_internal_utils.py index f2cf635e293..b7cf4695b07 100644 --- a/contrib/python/pip/pip/_vendor/requests/_internal_utils.py +++ b/contrib/python/pip/pip/_vendor/requests/_internal_utils.py @@ -5,14 +5,15 @@ requests._internal_utils Provides utility functions that are consumed internally by Requests which depend on extremely few external helpers (such as compat) """ + import re from .compat import builtin_str -_VALID_HEADER_NAME_RE_BYTE = re.compile(rb"^[^:\s][^:\r\n]*$") -_VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$") -_VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$") -_VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$") +_VALID_HEADER_NAME_RE_BYTE = re.compile(rb"^[^:\s][^:\r\n]*\Z") +_VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*\Z") +_VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*\Z|^\Z") +_VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*\Z|^\Z") _HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR) _HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE) diff --git a/contrib/python/pip/pip/_vendor/requests/adapters.py b/contrib/python/pip/pip/_vendor/requests/adapters.py index 67ccebcbea0..da8959e7787 100644 --- a/contrib/python/pip/pip/_vendor/requests/adapters.py +++ b/contrib/python/pip/pip/_vendor/requests/adapters.py @@ -11,17 +11,19 @@ import socket # noqa: F401 import typing import warnings -from pip._vendor.urllib3.exceptions import ClosedPoolError, ConnectTimeoutError -from pip._vendor.urllib3.exceptions import HTTPError as _HTTPError -from pip._vendor.urllib3.exceptions import InvalidHeader as _InvalidHeader from pip._vendor.urllib3.exceptions import ( + ClosedPoolError, + ConnectTimeoutError, LocationValueError, MaxRetryError, NewConnectionError, ProtocolError, + ReadTimeoutError, + ResponseError, ) +from pip._vendor.urllib3.exceptions import HTTPError as _HTTPError +from pip._vendor.urllib3.exceptions import InvalidHeader as _InvalidHeader from pip._vendor.urllib3.exceptions import ProxyError as _ProxyError -from pip._vendor.urllib3.exceptions import ReadTimeoutError, ResponseError from pip._vendor.urllib3.exceptions import SSLError as _SSLError from pip._vendor.urllib3.poolmanager import PoolManager, proxy_from_url from pip._vendor.urllib3.util import Timeout as TimeoutSauce @@ -47,7 +49,6 @@ from .models import Response from .structures import CaseInsensitiveDict from .utils import ( DEFAULT_CA_BUNDLE_PATH, - extract_zipped_paths, get_auth_from_url, get_encoding_from_headers, prepend_scheme_if_needed, @@ -76,9 +77,9 @@ DEFAULT_POOL_TIMEOUT = None def _urllib3_request_context( request: "PreparedRequest", verify: "bool | str | None", - client_cert: "typing.Tuple[str, str] | str | None", + client_cert: "tuple[str, str] | str | None", poolmanager: "PoolManager", -) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": +) -> "(dict[str, typing.Any], dict[str, typing.Any])": host_params = {} pool_kwargs = {} parsed_request_url = urlparse(request.url) @@ -297,7 +298,7 @@ class HTTPAdapter(BaseAdapter): cert_loc = verify if not cert_loc: - cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) + cert_loc = DEFAULT_CA_BUNDLE_PATH if not cert_loc or not os.path.exists(cert_loc): raise OSError( diff --git a/contrib/python/pip/pip/_vendor/requests/auth.py b/contrib/python/pip/pip/_vendor/requests/auth.py index 4a7ce6dc146..c39b645189d 100644 --- a/contrib/python/pip/pip/_vendor/requests/auth.py +++ b/contrib/python/pip/pip/_vendor/requests/auth.py @@ -35,9 +35,9 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({!r}) to " + f"3.0.0. Please convert the object you've passed in ({username!r}) to " "a string or bytes object in the near future to avoid " - "problems.".format(username), + "problems.", category=DeprecationWarning, ) username = str(username) @@ -45,9 +45,9 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({!r}) to " + f"3.0.0. Please convert the object you've passed in ({type(password)!r}) to " "a string or bytes object in the near future to avoid " - "problems.".format(type(password)), + "problems.", category=DeprecationWarning, ) password = str(password) diff --git a/contrib/python/pip/pip/_vendor/requests/certs.py b/contrib/python/pip/pip/_vendor/requests/certs.py index 2743144b994..c5953485fe8 100644 --- a/contrib/python/pip/pip/_vendor/requests/certs.py +++ b/contrib/python/pip/pip/_vendor/requests/certs.py @@ -11,6 +11,7 @@ If you are packaging Requests, e.g., for a Linux distribution or a managed environment, you can change the definition of where() to return a separately packaged CA bundle. """ + from pip._vendor.certifi import where if __name__ == "__main__": diff --git a/contrib/python/pip/pip/_vendor/requests/exceptions.py b/contrib/python/pip/pip/_vendor/requests/exceptions.py index 7f3660f00d9..d84504881a4 100644 --- a/contrib/python/pip/pip/_vendor/requests/exceptions.py +++ b/contrib/python/pip/pip/_vendor/requests/exceptions.py @@ -4,6 +4,7 @@ requests.exceptions This module contains the set of Requests' exceptions. """ + from pip._vendor.urllib3.exceptions import HTTPError as BaseHTTPError from .compat import JSONDecodeError as CompatJSONDecodeError diff --git a/contrib/python/pip/pip/_vendor/requests/help.py b/contrib/python/pip/pip/_vendor/requests/help.py index ddbb6150d64..022cd6062e0 100644 --- a/contrib/python/pip/pip/_vendor/requests/help.py +++ b/contrib/python/pip/pip/_vendor/requests/help.py @@ -40,11 +40,8 @@ def _implementation(): if implementation == "CPython": implementation_version = platform.python_version() elif implementation == "PyPy": - implementation_version = "{}.{}.{}".format( - sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro, - ) + pypy = sys.pypy_version_info + implementation_version = f"{pypy.major}.{pypy.minor}.{pypy.micro}" if sys.pypy_version_info.releaselevel != "final": implementation_version = "".join( [implementation_version, sys.pypy_version_info.releaselevel] diff --git a/contrib/python/pip/pip/_vendor/requests/hooks.py b/contrib/python/pip/pip/_vendor/requests/hooks.py index d181ba2ec2e..5976bc7d0f2 100644 --- a/contrib/python/pip/pip/_vendor/requests/hooks.py +++ b/contrib/python/pip/pip/_vendor/requests/hooks.py @@ -9,6 +9,7 @@ Available hooks: ``response``: The response generated from a Request. """ + HOOKS = ["response"] diff --git a/contrib/python/pip/pip/_vendor/requests/models.py b/contrib/python/pip/pip/_vendor/requests/models.py index 22de95c0612..ba7c01d9b3e 100644 --- a/contrib/python/pip/pip/_vendor/requests/models.py +++ b/contrib/python/pip/pip/_vendor/requests/models.py @@ -34,9 +34,11 @@ from .compat import ( builtin_str, chardet, cookielib, + urlencode, + urlsplit, + urlunparse, ) from .compat import json as complexjson -from .compat import urlencode, urlsplit, urlunparse from .cookies import _copy_cookie_jar, cookiejar_from_dict, get_cookie_header from .exceptions import ( ChunkedEncodingError, @@ -45,11 +47,11 @@ from .exceptions import ( HTTPError, InvalidJSONError, InvalidURL, + MissingSchema, + StreamConsumedError, ) from .exceptions import JSONDecodeError as RequestsJSONDecodeError -from .exceptions import MissingSchema from .exceptions import SSLError as RequestsSSLError -from .exceptions import StreamConsumedError from .hooks import default_hooks from .status_codes import codes from .structures import CaseInsensitiveDict diff --git a/contrib/python/pip/pip/_vendor/requests/sessions.py b/contrib/python/pip/pip/_vendor/requests/sessions.py index 731550de88a..578cc44d5c1 100644 --- a/contrib/python/pip/pip/_vendor/requests/sessions.py +++ b/contrib/python/pip/pip/_vendor/requests/sessions.py @@ -5,6 +5,7 @@ requests.sessions This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """ + import os import sys import time @@ -421,6 +422,8 @@ class Session(SessionRedirectMixin): #: expired certificates, which will make your application vulnerable to #: man-in-the-middle (MitM) attacks. #: Only set this to `False` for testing. + #: If verify is set to a string, it must be the path to a CA bundle file + #: that will be used to verify the TLS certificate. self.verify = True #: SSL client certificate default, if String, path to ssl client diff --git a/contrib/python/pip/pip/_vendor/requests/utils.py b/contrib/python/pip/pip/_vendor/requests/utils.py index e8ea5ad3674..6ee16b58037 100644 --- a/contrib/python/pip/pip/_vendor/requests/utils.py +++ b/contrib/python/pip/pip/_vendor/requests/utils.py @@ -39,9 +39,6 @@ from .compat import ( getproxies_environment, integer_types, is_urllib3_1, -) -from .compat import parse_http_list as _parse_list_header -from .compat import ( proxy_bypass, proxy_bypass_environment, quote, @@ -50,6 +47,7 @@ from .compat import ( urlparse, urlunparse, ) +from .compat import parse_http_list as _parse_list_header from .cookies import cookiejar_from_dict from .exceptions import ( FileModeWarning, @@ -61,6 +59,7 @@ from .structures import CaseInsensitiveDict NETRC_FILES = (".netrc", "_netrc") +# Certificate is extracted by certifi when needed. DEFAULT_CA_BUNDLE_PATH = certs.where() DEFAULT_PORTS = {"http": 80, "https": 443} @@ -233,7 +232,7 @@ def get_netrc_auth(url, raise_errors=False): try: _netrc = netrc(netrc_path).authenticators(host) - if _netrc: + if _netrc and any(_netrc): # Return with login / password login_i = 0 if _netrc[0] else 1 return (_netrc[login_i], _netrc[2]) @@ -283,12 +282,13 @@ def extract_zipped_paths(path): return path # we have a valid zip archive and a valid member of that archive - tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, member.split("/")[-1]) - if not os.path.exists(extracted_path): - # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition - with atomic_open(extracted_path) as file_handler: - file_handler.write(zip_file.read(member)) + suffix = os.path.splitext(member.split("/")[-1])[-1] + fd, extracted_path = tempfile.mkstemp(suffix=suffix) + try: + os.write(fd, zip_file.read(member)) + finally: + os.close(fd) + return extracted_path @@ -502,26 +502,23 @@ def get_encodings_from_content(content): def _parse_content_type_header(header): - """Returns content type and parameters from given header + """Returns content type and parameters from given header. :param header: string :return: tuple containing content type and dictionary of - parameters + parameters. """ tokens = header.split(";") content_type, params = tokens[0].strip(), tokens[1:] params_dict = {} - items_to_strip = "\"' " + strip_chars = "\"' " for param in params: param = param.strip() - if param: - key, value = param, True - index_of_equals = param.find("=") - if index_of_equals != -1: - key = param[:index_of_equals].strip(items_to_strip) - value = param[index_of_equals + 1 :].strip(items_to_strip) + if param and (idx := param.find("=")) != -1: + key = param[:idx].strip(strip_chars) + value = param[idx + 1 :].strip(strip_chars) params_dict[key.lower()] = value return content_type, params_dict diff --git a/contrib/python/pip/pip/_vendor/tomli/__init__.py b/contrib/python/pip/pip/_vendor/tomli/__init__.py index 9395b043827..aaecab11f9c 100644 --- a/contrib/python/pip/pip/_vendor/tomli/__init__.py +++ b/contrib/python/pip/pip/_vendor/tomli/__init__.py @@ -3,6 +3,6 @@ # Licensed to PSF under a Contributor Agreement. __all__ = ("loads", "load", "TOMLDecodeError") -__version__ = "2.3.0" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT +__version__ = "2.3.1" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT from ._parser import TOMLDecodeError, load, loads diff --git a/contrib/python/pip/pip/_vendor/tomli/_parser.py b/contrib/python/pip/pip/_vendor/tomli/_parser.py index 26170e8ed56..f81f468a84f 100644 --- a/contrib/python/pip/pip/_vendor/tomli/_parser.py +++ b/contrib/python/pip/pip/_vendor/tomli/_parser.py @@ -34,6 +34,13 @@ if TYPE_CHECKING: # lower number than where mypyc binaries crash. MAX_INLINE_NESTING: Final = sys.getrecursionlimit() +# Pathologically excessive number of parts in a key runs into quadratic +# behavior (e.g. in Flags.is_). +# Even if keys aren't currently parsed using recursion, they name a +# recursive structure, so it makes sense to limit it using getrecursionlimit() +# and RecursionError. +MAX_KEY_PARTS: Final = sys.getrecursionlimit() + ASCII_CTRL: Final = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) # Neither of these sets include quotation mark or backslash. They are @@ -145,7 +152,7 @@ def load(__fp: IO[bytes], *, parse_float: ParseFloat = float) -> dict[str, Any]: return loads(s, parse_float=parse_float) -def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901 +def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: """Parse TOML from a string.""" # The spec allows converting "\r\n" to "\n", even in string @@ -474,6 +481,10 @@ def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: pos = skip_chars(src, pos, TOML_WS) pos, key_part = parse_key_part(src, pos) key += (key_part,) + if len(key) > MAX_KEY_PARTS: + raise RecursionError( + f"TOML key has more than the allowed {MAX_KEY_PARTS} parts" + ) pos = skip_chars(src, pos, TOML_WS) @@ -676,7 +687,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: pos += 1 -def parse_value( # noqa: C901 +def parse_value( src: str, pos: Pos, parse_float: ParseFloat, nest_lvl: int ) -> tuple[Pos, Any]: if nest_lvl > MAX_INLINE_NESTING: diff --git a/contrib/python/pip/pip/_vendor/urllib3/LICENSE.txt b/contrib/python/pip/pip/_vendor/urllib3/LICENSE.txt index 429a1767e44..e6183d0276b 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/LICENSE.txt +++ b/contrib/python/pip/pip/_vendor/urllib3/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2008-2020 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2020 Andrey Petrov and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/contrib/python/pip/pip/_vendor/urllib3/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/__init__.py index c6fa38212fb..3fe782c8a45 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/__init__.py +++ b/contrib/python/pip/pip/_vendor/urllib3/__init__.py @@ -1,40 +1,49 @@ """ Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more """ -from __future__ import absolute_import + +from __future__ import annotations # Set default logging handler to avoid "No handler found" warnings. import logging +import sys +import typing import warnings from logging import NullHandler from . import exceptions +from ._base_connection import _TYPE_BODY +from ._collections import HTTPHeaderDict from ._version import __version__ from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url -from .filepost import encode_multipart_formdata +from .filepost import _TYPE_FIELDS, encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url -from .response import HTTPResponse +from .response import BaseHTTPResponse, HTTPResponse from .util.request import make_headers from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host -# === NOTE TO REPACKAGERS AND VENDORS === -# Please delete this block, this logic is only -# for urllib3 being distributed via PyPI. -# See: https://github.com/urllib3/urllib3/issues/2680 +# Ensure that Python is compiled with OpenSSL 1.1.1+ +# If the 'ssl' module isn't available at all that's +# fine, we only care if the module is available. try: - import urllib3_secure_extra # type: ignore # noqa: F401 + import ssl except ImportError: pass else: - warnings.warn( - "'urllib3[secure]' extra is deprecated and will be removed " - "in a future release of urllib3 2.x. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2680", - category=DeprecationWarning, - stacklevel=2, - ) + if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive: + warnings.warn( + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " + f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " + "See: https://github.com/urllib3/urllib3/issues/3020", + exceptions.NotOpenSSLWarning, + ) + elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive: + raise ImportError( + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " + f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " + "See: https://github.com/urllib3/urllib3/issues/2168" + ) __author__ = "Andrey Petrov ([email protected])" __license__ = "MIT" @@ -42,6 +51,7 @@ __version__ = __version__ __all__ = ( "HTTPConnectionPool", + "HTTPHeaderDict", "HTTPSConnectionPool", "PoolManager", "ProxyManager", @@ -52,15 +62,18 @@ __all__ = ( "connection_from_url", "disable_warnings", "encode_multipart_formdata", - "get_host", "make_headers", "proxy_from_url", + "request", + "BaseHTTPResponse", ) logging.getLogger(__name__).addHandler(NullHandler()) -def add_stderr_logger(level=logging.DEBUG): +def add_stderr_logger( + level: int = logging.DEBUG, +) -> logging.StreamHandler[typing.TextIO]: """ Helper for quickly adding a StreamHandler to the logger. Useful for debugging. @@ -87,16 +100,112 @@ del NullHandler # mechanisms to silence them. # SecurityWarning's always go off by default. warnings.simplefilter("always", exceptions.SecurityWarning, append=True) -# SubjectAltNameWarning's should go off once per host -warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) -# SNIMissingWarnings should go off only once. -warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) -def disable_warnings(category=exceptions.HTTPWarning): +def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None: """ Helper for quickly disabling all urllib3 warnings. """ warnings.simplefilter("ignore", category) + + +_DEFAULT_POOL = PoolManager() + + +def request( + method: str, + url: str, + *, + body: _TYPE_BODY | None = None, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + preload_content: bool | None = True, + decode_content: bool | None = True, + redirect: bool | None = True, + retries: Retry | bool | int | None = None, + timeout: Timeout | float | int | None = 3, + json: typing.Any | None = None, +) -> BaseHTTPResponse: + """ + A convenience, top-level request method. It uses a module-global ``PoolManager`` instance. + Therefore, its side effects could be shared across dependencies relying on it. + To avoid side effects create a new ``PoolManager`` instance and use it instead. + The method does not accept low-level ``**urlopen_kw`` keyword arguments. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. + + :param bool preload_content: + If True, the response's body will be preloaded into memory. + + :param bool decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param redirect: + If True, automatically handle redirects (status codes 301, 302, + 303, 307, 308). Each redirect counts as a retry. Disabling retries + will disable redirect, too. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + + :param timeout: + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. + """ + + return _DEFAULT_POOL.request( + method, + url, + body=body, + fields=fields, + headers=headers, + preload_content=preload_content, + decode_content=decode_content, + redirect=redirect, + retries=retries, + timeout=timeout, + json=json, + ) + + +if sys.platform == "emscripten": + from .contrib.emscripten import inject_into_urllib3 # noqa: 401 + + inject_into_urllib3() diff --git a/contrib/python/pip/pip/_vendor/urllib3/_base_connection.py b/contrib/python/pip/pip/_vendor/urllib3/_base_connection.py new file mode 100644 index 00000000000..dc0f318c0b3 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/_base_connection.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import typing + +from .util.connection import _TYPE_SOCKET_OPTIONS +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from .util.url import Url + +_TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], str] + + +class ProxyConfig(typing.NamedTuple): + ssl_context: ssl.SSLContext | None + use_forwarding_for_https: bool + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None + + +class _ResponseOptions(typing.NamedTuple): + # TODO: Remove this in favor of a better + # HTTP request/response lifecycle tracking. + request_method: str + request_url: str + preload_content: bool + decode_content: bool + enforce_content_length: bool + + +if typing.TYPE_CHECKING: + import ssl + from typing import Protocol + + from .response import BaseHTTPResponse + + class BaseHTTPConnection(Protocol): + default_port: typing.ClassVar[int] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + host: str + port: int + timeout: None | ( + float + ) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved. + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool + proxy_is_verified: bool | None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: ... + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: ... + + def connect(self) -> None: ... + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: ... + + def getresponse(self) -> BaseHTTPResponse: ... + + def close(self) -> None: ... + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + + class BaseHTTPSConnection(BaseHTTPConnection, Protocol): + default_port: typing.ClassVar[int] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + # Certificate verification methods + cert_reqs: int | str | None + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None + ssl_context: ssl.SSLContext | None + + # Trusted CAs + ca_certs: str | None + ca_cert_dir: str | None + ca_cert_data: None | str | bytes + + # TLS version + ssl_minimum_version: int | None + ssl_maximum_version: int | None + ssl_version: int | str | None # Deprecated + + # Client certificates + cert_file: str | None + key_file: str | None + key_password: str | None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: ... diff --git a/contrib/python/pip/pip/_vendor/urllib3/_collections.py b/contrib/python/pip/pip/_vendor/urllib3/_collections.py index bceb8451f0e..0378aab1b1a 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/_collections.py +++ b/contrib/python/pip/pip/_vendor/urllib3/_collections.py @@ -1,34 +1,66 @@ -from __future__ import absolute_import +from __future__ import annotations -try: - from collections.abc import Mapping, MutableMapping -except ImportError: - from collections import Mapping, MutableMapping -try: - from threading import RLock -except ImportError: # Platform-specific: No threads available +import typing +from collections import OrderedDict +from enum import Enum, auto +from threading import RLock - class RLock: - def __enter__(self): - pass +if typing.TYPE_CHECKING: + # We can only import Protocol if TYPE_CHECKING because it's a development + # dependency, and is not available at runtime. + from typing import Protocol - def __exit__(self, exc_type, exc_value, traceback): - pass + from typing_extensions import Self + class HasGettableStringKeys(Protocol): + def keys(self) -> typing.Iterator[str]: ... -from collections import OrderedDict + def __getitem__(self, key: str) -> str: ... -from .exceptions import InvalidHeader -from .packages import six -from .packages.six import iterkeys, itervalues __all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] -_Null = object() +# Key type +_KT = typing.TypeVar("_KT") +# Value type +_VT = typing.TypeVar("_VT") +# Default type +_DT = typing.TypeVar("_DT") + +ValidHTTPHeaderSource = typing.Union[ + "HTTPHeaderDict", + typing.Mapping[str, str], + typing.Iterable[tuple[str, str]], + "HasGettableStringKeys", +] + + +class _Sentinel(Enum): + not_passed = auto() -class RecentlyUsedContainer(MutableMapping): +def ensure_can_construct_http_header_dict( + potential: object, +) -> ValidHTTPHeaderSource | None: + if isinstance(potential, HTTPHeaderDict): + return potential + elif isinstance(potential, typing.Mapping): + # Full runtime checking of the contents of a Mapping is expensive, so for the + # purposes of typechecking, we assume that any Mapping is the right shape. + return typing.cast(typing.Mapping[str, str], potential) + elif isinstance(potential, typing.Iterable): + # Similarly to Mapping, full runtime checking of the contents of an Iterable is + # expensive, so for the purposes of typechecking, we assume that any Iterable + # is the right shape. + return typing.cast(typing.Iterable[tuple[str, str]], potential) + elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"): + return typing.cast("HasGettableStringKeys", potential) + else: + return None + + +class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]): """ Provides a thread-safe dict-like container which maintains up to ``maxsize`` keys while throwing away the least-recently-used keys beyond @@ -42,69 +74,134 @@ class RecentlyUsedContainer(MutableMapping): ``dispose_func(value)`` is called. Callback which will get called """ - ContainerCls = OrderedDict + _container: typing.OrderedDict[_KT, _VT] + _maxsize: int + dispose_func: typing.Callable[[_VT], None] | None + lock: RLock - def __init__(self, maxsize=10, dispose_func=None): + def __init__( + self, + maxsize: int = 10, + dispose_func: typing.Callable[[_VT], None] | None = None, + ) -> None: + super().__init__() self._maxsize = maxsize self.dispose_func = dispose_func - - self._container = self.ContainerCls() + self._container = OrderedDict() self.lock = RLock() - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: # Re-insert the item, moving it to the end of the eviction line. with self.lock: item = self._container.pop(key) self._container[key] = item return item - def __setitem__(self, key, value): - evicted_value = _Null + def __setitem__(self, key: _KT, value: _VT) -> None: + evicted_item = None with self.lock: # Possibly evict the existing value of 'key' - evicted_value = self._container.get(key, _Null) - self._container[key] = value + try: + # If the key exists, we'll overwrite it, which won't change the + # size of the pool. Because accessing a key should move it to + # the end of the eviction line, we pop it out first. + evicted_item = key, self._container.pop(key) + self._container[key] = value + except KeyError: + # When the key does not exist, we insert the value first so that + # evicting works in all cases, including when self._maxsize is 0 + self._container[key] = value + if len(self._container) > self._maxsize: + # If we didn't evict an existing value, and we've hit our maximum + # size, then we have to evict the least recently used item from + # the beginning of the container. + evicted_item = self._container.popitem(last=False) - # If we didn't evict an existing value, we might have to evict the - # least recently used item from the beginning of the container. - if len(self._container) > self._maxsize: - _key, evicted_value = self._container.popitem(last=False) - - if self.dispose_func and evicted_value is not _Null: + # After releasing the lock on the pool, dispose of any evicted value. + if evicted_item is not None and self.dispose_func: + _, evicted_value = evicted_item self.dispose_func(evicted_value) - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: with self.lock: value = self._container.pop(key) if self.dispose_func: self.dispose_func(value) - def __len__(self): + def __len__(self) -> int: with self.lock: return len(self._container) - def __iter__(self): + def __iter__(self) -> typing.NoReturn: raise NotImplementedError( "Iteration over this class is unlikely to be threadsafe." ) - def clear(self): + def clear(self) -> None: with self.lock: # Copy pointers to all values, then wipe the mapping - values = list(itervalues(self._container)) + values = list(self._container.values()) self._container.clear() if self.dispose_func: for value in values: self.dispose_func(value) - def keys(self): + def keys(self) -> set[_KT]: # type: ignore[override] with self.lock: - return list(iterkeys(self._container)) + return set(self._container.keys()) -class HTTPHeaderDict(MutableMapping): +class HTTPHeaderDictItemView(set[tuple[str, str]]): + """ + HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of + address. + + If we directly try to get an item with a particular name, we will get a string + back that is the concatenated version of all the values: + + >>> d['X-Header-Name'] + 'Value1, Value2, Value3' + + However, if we iterate over an HTTPHeaderDict's items, we will optionally combine + these values based on whether combine=True was called when building up the dictionary + + >>> d = HTTPHeaderDict({"A": "1", "B": "foo"}) + >>> d.add("A", "2", combine=True) + >>> d.add("B", "bar") + >>> list(d.items()) + [ + ('A', '1, 2'), + ('B', 'foo'), + ('B', 'bar'), + ] + + This class conforms to the interface required by the MutableMapping ABC while + also giving us the nonstandard iteration behavior we want; items with duplicate + keys, ordered by time of first insertion. + """ + + _headers: HTTPHeaderDict + + def __init__(self, headers: HTTPHeaderDict) -> None: + self._headers = headers + + def __len__(self) -> int: + return len(list(self._headers.iteritems())) + + def __iter__(self) -> typing.Iterator[tuple[str, str]]: + return self._headers.iteritems() + + def __contains__(self, item: object) -> bool: + if isinstance(item, tuple) and len(item) == 2: + passed_key, passed_val = item + if isinstance(passed_key, str) and isinstance(passed_val, str): + return self._headers._has_value_for_header(passed_key, passed_val) + return False + + +class HTTPHeaderDict(typing.MutableMapping[str, str]): """ :param headers: An iterable of field-value pairs. Must not contain multiple field names @@ -138,9 +235,11 @@ class HTTPHeaderDict(MutableMapping): '7' """ - def __init__(self, headers=None, **kwargs): - super(HTTPHeaderDict, self).__init__() - self._container = OrderedDict() + _container: typing.MutableMapping[str, list[str]] + + def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str): + super().__init__() + self._container = {} # 'dict' is insert-ordered if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) @@ -149,126 +248,156 @@ class HTTPHeaderDict(MutableMapping): if kwargs: self.extend(kwargs) - def __setitem__(self, key, val): + def __setitem__(self, key: str, val: str) -> None: + # avoid a bytes/str comparison by decoding before httplib + if isinstance(key, bytes): + key = key.decode("latin-1") self._container[key.lower()] = [key, val] - return self._container[key.lower()] - def __getitem__(self, key): + def __getitem__(self, key: str) -> str: + if isinstance(key, bytes): + key = key.decode("latin-1") val = self._container[key.lower()] return ", ".join(val[1:]) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: + if isinstance(key, bytes): + key = key.decode("latin-1") del self._container[key.lower()] - def __contains__(self, key): - return key.lower() in self._container + def __contains__(self, key: object) -> bool: + if isinstance(key, bytes): + key = key.decode("latin-1") + if isinstance(key, str): + return key.lower() in self._container + return False - def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, "keys"): - return False - if not isinstance(other, type(self)): - other = type(self)(other) - return dict((k.lower(), v) for k, v in self.itermerged()) == dict( - (k.lower(), v) for k, v in other.itermerged() - ) + def setdefault(self, key: str, default: str = "") -> str: + return super().setdefault(key, default) - def __ne__(self, other): - return not self.__eq__(other) + def __eq__(self, other: object) -> bool: + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return False + else: + other_as_http_header_dict = type(self)(maybe_constructable) - if six.PY2: # Python 2 - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues + return {k.lower(): v for k, v in self.itermerged()} == { + k.lower(): v for k, v in other_as_http_header_dict.itermerged() + } - __marker = object() + def __ne__(self, other: object) -> bool: + return not self.__eq__(other) - def __len__(self): + def __len__(self) -> int: return len(self._container) - def __iter__(self): + def __iter__(self) -> typing.Iterator[str]: # Only provide the originally cased names for vals in self._container.values(): yield vals[0] - def pop(self, key, default=__marker): - """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - """ - # Using the MutableMapping function directly fails due to the private marker. - # Using ordinary dict.pop would expose the internal structures. - # So let's reinvent the wheel. - try: - value = self[key] - except KeyError: - if default is self.__marker: - raise - return default - else: - del self[key] - return value - - def discard(self, key): + def discard(self, key: str) -> None: try: del self[key] except KeyError: pass - def add(self, key, val): + def add(self, key: str, val: str, *, combine: bool = False) -> None: """Adds a (name, value) pair, doesn't overwrite the value if it already exists. + If this is called with combine=True, instead of adding a new header value + as a distinct item during iteration, this will instead append the value to + any existing header value with a comma. If no existing header value exists + for the key, then the value will simply be added, ignoring the combine parameter. + >>> headers = HTTPHeaderDict(foo='bar') >>> headers.add('Foo', 'baz') >>> headers['foo'] 'bar, baz' + >>> list(headers.items()) + [('foo', 'bar'), ('foo', 'baz')] + >>> headers.add('foo', 'quz', combine=True) + >>> list(headers.items()) + [('foo', 'bar, baz, quz')] """ + # avoid a bytes/str comparison by decoding before httplib + if isinstance(key, bytes): + key = key.decode("latin-1") key_lower = key.lower() new_vals = [key, val] # Keep the common case aka no item present as fast as possible vals = self._container.setdefault(key_lower, new_vals) if new_vals is not vals: - vals.append(val) + # if there are values here, then there is at least the initial + # key/value pair + assert len(vals) >= 2 + if combine: + vals[-1] = vals[-1] + ", " + val + else: + vals.append(val) - def extend(self, *args, **kwargs): + def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None: """Generic import function for any type of header-like object. Adapted version of MutableMapping.update in order to insert items with self.add instead of self.__setitem__ """ if len(args) > 1: raise TypeError( - "extend() takes at most 1 positional " - "arguments ({0} given)".format(len(args)) + f"extend() takes at most 1 positional arguments ({len(args)} given)" ) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): for key, val in other.iteritems(): self.add(key, val) - elif isinstance(other, Mapping): - for key in other: - self.add(key, other[key]) - elif hasattr(other, "keys"): - for key in other.keys(): - self.add(key, other[key]) - else: + elif isinstance(other, typing.Mapping): + for key, val in other.items(): + self.add(key, val) + elif isinstance(other, typing.Iterable): + other = typing.cast(typing.Iterable[tuple[str, str]], other) for key, value in other: self.add(key, value) + elif hasattr(other, "keys") and hasattr(other, "__getitem__"): + # THIS IS NOT A TYPESAFE BRANCH + # In this branch, the object has a `keys` attr but is not a Mapping or any of + # the other types indicated in the method signature. We do some stuff with + # it as though it partially implements the Mapping interface, but we're not + # doing that stuff safely AT ALL. + for key in other.keys(): + self.add(key, other[key]) for key, value in kwargs.items(): self.add(key, value) - def getlist(self, key, default=__marker): + @typing.overload + def getlist(self, key: str) -> list[str]: ... + + @typing.overload + def getlist(self, key: str, default: _DT) -> list[str] | _DT: ... + + def getlist( + self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed + ) -> list[str] | _DT: """Returns a list of all the values for the named field. Returns an empty list if the key doesn't exist.""" + if isinstance(key, bytes): + key = key.decode("latin-1") try: vals = self._container[key.lower()] except KeyError: - if default is self.__marker: + if default is _Sentinel.not_passed: + # _DT is unbound; empty list is instance of List[str] return [] + # _DT is bound; default is instance of _DT return default else: + # _DT may or may not be bound; vals[1:] is instance of List[str], which + # meets our external interface requirement of `Union[List[str], _DT]`. return vals[1:] - def _prepare_for_method_change(self): + def _prepare_for_method_change(self) -> Self: """ Remove content-specific header fields before changing the request method to GET or HEAD according to RFC 9110, Section 15.4. @@ -294,62 +423,65 @@ class HTTPHeaderDict(MutableMapping): # Backwards compatibility for http.cookiejar get_all = getlist - def __repr__(self): - return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) + def __repr__(self) -> str: + return f"{type(self).__name__}({dict(self.itermerged())})" - def _copy_from(self, other): + def _copy_from(self, other: HTTPHeaderDict) -> None: for key in other: val = other.getlist(key) - if isinstance(val, list): - # Don't need to convert tuples - val = list(val) - self._container[key.lower()] = [key] + val + self._container[key.lower()] = [key, *val] - def copy(self): + def copy(self) -> Self: clone = type(self)() clone._copy_from(self) return clone - def iteritems(self): + def iteritems(self) -> typing.Iterator[tuple[str, str]]: """Iterate over all header lines, including duplicate ones.""" for key in self: vals = self._container[key.lower()] for val in vals[1:]: yield vals[0], val - def itermerged(self): + def itermerged(self) -> typing.Iterator[tuple[str, str]]: """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] yield val[0], ", ".join(val[1:]) - def items(self): - return list(self.iteritems()) + def items(self) -> HTTPHeaderDictItemView: # type: ignore[override] + return HTTPHeaderDictItemView(self) - @classmethod - def from_httplib(cls, message): # Python 2 - """Read headers from a Python 2 httplib message object.""" - # python2.7 does not expose a proper API for exporting multiheaders - # efficiently. This function re-reads raw lines from the message - # object and extracts the multiheaders properly. - obs_fold_continued_leaders = (" ", "\t") - headers = [] + def _has_value_for_header(self, header_name: str, potential_value: str) -> bool: + if header_name in self: + return potential_value in self._container[header_name.lower()][1:] + return False - for line in message.headers: - if line.startswith(obs_fold_continued_leaders): - if not headers: - # We received a header line that starts with OWS as described - # in RFC-7230 S3.2.4. This indicates a multiline header, but - # there exists no previous header to which we can attach it. - raise InvalidHeader( - "Header continuation with no previous header: %s" % line - ) - else: - key, value = headers[-1] - headers[-1] = (key, value + " " + line.strip()) - continue + def __ior__(self, other: object) -> HTTPHeaderDict: + # Supports extending a header dict in-place using operator |= + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + self.extend(maybe_constructable) + return self - key, value = line.split(":", 1) - headers.append((key, value.strip())) + def __or__(self, other: object) -> Self: + # Supports merging header dicts using operator | + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = self.copy() + result.extend(maybe_constructable) + return result - return cls(headers) + def __ror__(self, other: object) -> Self: + # Supports merging header dicts using operator | when other is on left side + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = type(self)(maybe_constructable) + result.extend(self) + return result diff --git a/contrib/python/pip/pip/_vendor/urllib3/request.py b/contrib/python/pip/pip/_vendor/urllib3/_request_methods.py index 3b4cf999225..297c271bf40 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/request.py +++ b/contrib/python/pip/pip/_vendor/urllib3/_request_methods.py @@ -1,15 +1,23 @@ -from __future__ import absolute_import +from __future__ import annotations -import sys +import json as _json +import typing +from urllib.parse import urlencode -from .filepost import encode_multipart_formdata -from .packages import six -from .packages.six.moves.urllib.parse import urlencode +from ._base_connection import _TYPE_BODY +from ._collections import HTTPHeaderDict +from .filepost import _TYPE_FIELDS, encode_multipart_formdata +from .response import BaseHTTPResponse __all__ = ["RequestMethods"] +_TYPE_ENCODE_URL_FIELDS = typing.Union[ + typing.Sequence[tuple[str, typing.Union[str, bytes]]], + typing.Mapping[str, typing.Union[str, bytes]], +] -class RequestMethods(object): + +class RequestMethods: """ Convenience mixin for classes who implement a :meth:`urlopen` method, such as :class:`urllib3.HTTPConnectionPool` and @@ -40,25 +48,34 @@ class RequestMethods(object): _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} - def __init__(self, headers=None): + def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None: self.headers = headers or {} def urlopen( self, - method, - url, - body=None, - headers=None, - encode_multipart=True, - multipart_boundary=None, - **kw - ): # Abstract + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + **kw: typing.Any, + ) -> BaseHTTPResponse: # Abstract raise NotImplementedError( "Classes extending RequestMethods must implement " "their own ``urlopen`` method." ) - def request(self, method, url, fields=None, headers=None, **urlopen_kw): + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + json: typing.Any | None = None, + **urlopen_kw: typing.Any, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the appropriate encoding of ``fields`` based on the ``method`` used. @@ -68,29 +85,95 @@ class RequestMethods(object): option to drop down to more specific methods when necessary, such as :meth:`request_encode_url`, :meth:`request_encode_body`, or even the lowest level :meth:`urlopen`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the URL or request body, depending on ``method``. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. """ method = method.upper() - urlopen_kw["request_url"] = url + if json is not None and body is not None: + raise TypeError( + "request got values for both 'body' and 'json' parameters which are mutually exclusive" + ) + + if json is not None: + if headers is None: + headers = self.headers + + if not ("content-type" in map(str.lower, headers.keys())): + headers = HTTPHeaderDict(headers) + headers["Content-Type"] = "application/json" + + body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode( + "utf-8" + ) + + if body is not None: + urlopen_kw["body"] = body if method in self._encode_url_methods: return self.request_encode_url( - method, url, fields=fields, headers=headers, **urlopen_kw + method, + url, + fields=fields, # type: ignore[arg-type] + headers=headers, + **urlopen_kw, ) else: return self.request_encode_body( method, url, fields=fields, headers=headers, **urlopen_kw ) - def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): + def request_encode_url( + self, + method: str, + url: str, + fields: _TYPE_ENCODE_URL_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + **urlopen_kw: str, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the URL. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. """ if headers is None: headers = self.headers - extra_kw = {"headers": headers} + extra_kw: dict[str, typing.Any] = {"headers": headers} extra_kw.update(urlopen_kw) if fields: @@ -100,14 +183,14 @@ class RequestMethods(object): def request_encode_body( self, - method, - url, - fields=None, - headers=None, - encode_multipart=True, - multipart_boundary=None, - **urlopen_kw - ): + method: str, + url: str, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + **urlopen_kw: str, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -142,11 +225,34 @@ class RequestMethods(object): be overwritten because it depends on the dynamic random boundary string which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param encode_multipart: + If True, encode the ``fields`` using the multipart/form-data MIME + format. + + :param multipart_boundary: + If not specified, then a random boundary will be generated using + :func:`urllib3.filepost.choose_boundary`. """ if headers is None: headers = self.headers - extra_kw = {"headers": {}} + extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)} + body: bytes | str if fields: if "body" in urlopen_kw: @@ -160,32 +266,13 @@ class RequestMethods(object): ) else: body, content_type = ( - urlencode(fields), + urlencode(fields), # type: ignore[arg-type] "application/x-www-form-urlencoded", ) extra_kw["body"] = body - extra_kw["headers"] = {"Content-Type": content_type} + extra_kw["headers"].setdefault("Content-Type", content_type) - extra_kw["headers"].update(headers) extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) - - -if not six.PY2: - - class RequestModule(sys.modules[__name__].__class__): - def __call__(self, *args, **kwargs): - """ - If user tries to call this module directly urllib3 v2.x style raise an error to the user - suggesting they may need urllib3 v2 - """ - raise TypeError( - "'module' object is not callable\n" - "urllib3.request() method is not supported in this release, " - "upgrade to urllib3 v2 to use it\n" - "see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html" - ) - - sys.modules[__name__].__class__ = RequestModule diff --git a/contrib/python/pip/pip/_vendor/urllib3/_version.py b/contrib/python/pip/pip/_vendor/urllib3/_version.py index d49df2a0c54..268d3b984dc 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/_version.py +++ b/contrib/python/pip/pip/_vendor/urllib3/_version.py @@ -1,2 +1,34 @@ -# This file is protected via CODEOWNERS -__version__ = "1.26.20" +# file generated by setuptools-scm +# don't change, don't track in version control + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] + COMMIT_ID = Union[str, None] +else: + VERSION_TUPLE = object + COMMIT_ID = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE +commit_id: COMMIT_ID +__commit_id__: COMMIT_ID + +__version__ = version = '2.6.3' +__version_tuple__ = version_tuple = (2, 6, 3) + +__commit_id__ = commit_id = None diff --git a/contrib/python/pip/pip/_vendor/urllib3/connection.py b/contrib/python/pip/pip/_vendor/urllib3/connection.py index de35b63d670..2ceeb0a5483 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/connection.py +++ b/contrib/python/pip/pip/_vendor/urllib3/connection.py @@ -1,59 +1,59 @@ -from __future__ import absolute_import +from __future__ import annotations import datetime +import http.client import logging import os import re import socket +import sys +import threading +import typing import warnings -from socket import error as SocketError +from http.client import HTTPConnection as _HTTPConnection +from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import ResponseNotReady from socket import timeout as SocketTimeout -from .packages import six -from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection -from .packages.six.moves.http_client import HTTPException # noqa: F401 -from .util.proxy import create_proxy_ssl_context +if typing.TYPE_CHECKING: + from .response import HTTPResponse + from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT + from .util.ssltransport import SSLTransport + +from ._collections import HTTPHeaderDict +from .http2 import probe as http2_probe +from .util.response import assert_header_parsing +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout +from .util.util import to_str +from .util.wait import wait_for_read try: # Compiled with SSL? import ssl BaseSSLError = ssl.SSLError -except (ImportError, AttributeError): # Platform-specific: No SSL. - ssl = None - - class BaseSSLError(BaseException): - pass - - -try: - # Python 3: not a no-op, we're adding this to the namespace so it can be imported. - ConnectionError = ConnectionError -except NameError: - # Python 2 - class ConnectionError(Exception): - pass +except (ImportError, AttributeError): + ssl = None # type: ignore[assignment] - -try: # Python 3: - # Not a no-op, we're adding this to the namespace so it can be imported. - BrokenPipeError = BrokenPipeError -except NameError: # Python 2: - - class BrokenPipeError(Exception): + class BaseSSLError(BaseException): # type: ignore[no-redef] pass -from ._collections import HTTPHeaderDict # noqa (historical, removed in v2) +from ._base_connection import _TYPE_BODY +from ._base_connection import ProxyConfig as ProxyConfig +from ._base_connection import _ResponseOptions as _ResponseOptions from ._version import __version__ from .exceptions import ( ConnectTimeoutError, + HeaderParsingError, + NameResolutionError, NewConnectionError, - SubjectAltNameWarning, + ProxyError, SystemTimeWarning, ) -from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection +from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_ +from .util.request import body_to_chunks +from .util.ssl_ import assert_fingerprint as _assert_fingerprint from .util.ssl_ import ( - assert_fingerprint, create_urllib3_context, is_ipaddress, resolve_cert_reqs, @@ -61,6 +61,12 @@ from .util.ssl_ import ( ssl_wrap_socket, ) from .util.ssl_match_hostname import CertificateError, match_hostname +from .util.url import Url + +# Not a no-op, we're adding this to the namespace so it can be imported. +ConnectionError = ConnectionError +BrokenPipeError = BrokenPipeError + log = logging.getLogger(__name__) @@ -68,12 +74,12 @@ port_by_scheme = {"http": 80, "https": 443} # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2024, 1, 1) +RECENT_DATE = datetime.date(2025, 1, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") -class HTTPConnection(_HTTPConnection, object): +class HTTPConnection(_HTTPConnection): """ Based on :class:`http.client.HTTPConnection` but provides an extra constructor backwards-compatibility layer between older and newer Pythons. @@ -81,7 +87,6 @@ class HTTPConnection(_HTTPConnection, object): Additional keyword parameters are used to configure attributes of the connection. Accepted parameters include: - - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling @@ -99,38 +104,70 @@ class HTTPConnection(_HTTPConnection, object): Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ - default_port = port_by_scheme["http"] + default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc] #: Disable Nagle's algorithm by default. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` - default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [ + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + ] #: Whether this connection verifies the host's certificate. - is_verified = False + is_verified: bool = False - #: Whether this proxy connection (if used) verifies the proxy host's - #: certificate. - proxy_is_verified = None + #: Whether this proxy connection verified the proxy host's certificate. + # If no proxy is currently connected to the value will be ``None``. + proxy_is_verified: bool | None = None - def __init__(self, *args, **kw): - if not six.PY2: - kw.pop("strict", None) + blocksize: int + source_address: tuple[str, int] | None + socket_options: connection._TYPE_SOCKET_OPTIONS | None - # Pre-set source_address. - self.source_address = kw.get("source_address") + _has_connected_to_proxy: bool + _response_options: _ResponseOptions | None + _tunnel_host: str | None + _tunnel_port: int | None + _tunnel_scheme: str | None - #: The socket options provided by the user. If no options are - #: provided, we use the default options. - self.socket_options = kw.pop("socket_options", self.default_socket_options) + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None | ( + connection._TYPE_SOCKET_OPTIONS + ) = default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + super().__init__( + host=host, + port=port, + timeout=Timeout.resolve_default_timeout(timeout), + source_address=source_address, + blocksize=blocksize, + ) + self.socket_options = socket_options + self.proxy = proxy + self.proxy_config = proxy_config + + self._has_connected_to_proxy = False + self._response_options = None + self._tunnel_host: str | None = None + self._tunnel_port: int | None = None + self._tunnel_scheme: str | None = None - # Proxy options provided by the user. - self.proxy = kw.pop("proxy", None) - self.proxy_config = kw.pop("proxy_config", None) + def __str__(self) -> str: + return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})" - _HTTPConnection.__init__(self, *args, **kw) + def __repr__(self) -> str: + return f"<{self} at {id(self):#x}>" @property - def host(self): + def host(self) -> str: """ Getter method to remove any trailing dots that indicate the hostname is an FQDN. @@ -149,7 +186,7 @@ class HTTPConnection(_HTTPConnection, object): return self._dns_host.rstrip(".") @host.setter - def host(self, value): + def host(self, value: str) -> None: """ Setter for the `host` property. @@ -158,129 +195,409 @@ class HTTPConnection(_HTTPConnection, object): """ self._dns_host = value - def _new_conn(self): + def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ - extra_kw = {} - if self.source_address: - extra_kw["source_address"] = self.source_address - - if self.socket_options: - extra_kw["socket_options"] = self.socket_options - try: - conn = connection.create_connection( - (self._dns_host, self.port), self.timeout, **extra_kw + sock = connection.create_connection( + (self._dns_host, self.port), + self.timeout, + source_address=self.source_address, + socket_options=self.socket_options, ) - - except SocketTimeout: + except socket.gaierror as e: + raise NameResolutionError(self.host, self, e) from e + except SocketTimeout as e: raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e - except SocketError as e: + except OSError as e: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e + self, f"Failed to establish a new connection: {e}" + ) from e + + sys.audit("http.client.connect", self, self.host, self.port) + + return sock + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + if scheme not in ("http", "https"): + raise ValueError( + f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'" ) + super().set_tunnel(host, port=port, headers=headers) + self._tunnel_scheme = scheme - return conn + if sys.version_info < (3, 11, 9) or ((3, 12) <= sys.version_info < (3, 12, 3)): + # Taken from python/cpython#100986 which was backported in 3.11.9 and 3.12.3. + # When using connection_from_host, host will come without brackets. + def _wrap_ipv6(self, ip: bytes) -> bytes: + if b":" in ip and ip[0] != b"["[0]: + return b"[" + ip + b"]" + return ip - def _is_using_tunnel(self): - # Google App Engine's httplib does not define _tunnel_host - return getattr(self, "_tunnel_host", None) + if sys.version_info < (3, 11, 9): + # `_tunnel` copied from 3.11.13 backporting + # https://github.com/python/cpython/commit/0d4026432591d43185568dd31cef6a034c4b9261 + # and https://github.com/python/cpython/commit/6fbc61070fda2ffb8889e77e3b24bca4249ab4d1 + def _tunnel(self) -> None: + _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined] + connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format] + self._wrap_ipv6(self._tunnel_host.encode("ascii")), # type: ignore[union-attr] + self._tunnel_port, + ) + headers = [connect] + for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined] + headers.append(f"{header}: {value}\r\n".encode("latin-1")) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers + + response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined] + try: + (version, code, message) = response._read_status() # type: ignore[attr-defined] + + if code != http.HTTPStatus.OK: + self.close() + raise OSError( + f"Tunnel connection failed: {code} {message.strip()}" + ) + while True: + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise http.client.LineTooLong("header line") + if not line: + # for sites which EOF without sending a trailer + break + if line in (b"\r\n", b"\n", b""): + break + + if self.debuglevel > 0: + print("header:", line.decode()) + finally: + response.close() + + elif (3, 12) <= sys.version_info < (3, 12, 3): + # `_tunnel` copied from 3.12.11 backporting + # https://github.com/python/cpython/commit/23aef575c7629abcd4aaf028ebd226fb41a4b3c8 + def _tunnel(self) -> None: # noqa: F811 + connect = b"CONNECT %s:%d HTTP/1.1\r\n" % ( # type: ignore[str-format] + self._wrap_ipv6(self._tunnel_host.encode("idna")), # type: ignore[union-attr] + self._tunnel_port, + ) + headers = [connect] + for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined] + headers.append(f"{header}: {value}\r\n".encode("latin-1")) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers + + response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined] + try: + (version, code, message) = response._read_status() # type: ignore[attr-defined] + + self._raw_proxy_headers = http.client._read_headers(response.fp) # type: ignore[attr-defined] + + if self.debuglevel > 0: + for header in self._raw_proxy_headers: + print("header:", header.decode()) + + if code != http.HTTPStatus.OK: + self.close() + raise OSError( + f"Tunnel connection failed: {code} {message.strip()}" + ) + + finally: + response.close() + + def connect(self) -> None: + self.sock = self._new_conn() + if self._tunnel_host: + # If we're tunneling it means we're connected to our proxy. + self._has_connected_to_proxy = True - def _prepare_conn(self, conn): - self.sock = conn - if self._is_using_tunnel(): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() - # Mark this connection as not reusable - self.auto_open = 0 - def connect(self): - conn = self._new_conn() - self._prepare_conn(conn) + # If there's a proxy to be connected to we are fully connected. + # This is set twice (once above and here) due to forwarding proxies + # not using tunnelling. + self._has_connected_to_proxy = bool(self.proxy) + + if self._has_connected_to_proxy: + self.proxy_is_verified = False + + @property + def is_closed(self) -> bool: + return self.sock is None + + @property + def is_connected(self) -> bool: + if self.sock is None: + return False + return not wait_for_read(self.sock, timeout=0.0) + + @property + def has_connected_to_proxy(self) -> bool: + return self._has_connected_to_proxy + + @property + def proxy_is_forwarding(self) -> bool: + """ + Return True if a forwarding proxy is configured, else return False + """ + return bool(self.proxy) and self._tunnel_host is None + + @property + def proxy_is_tunneling(self) -> bool: + """ + Return True if a tunneling proxy is configured, else return False + """ + return self._tunnel_host is not None + + def close(self) -> None: + try: + super().close() + finally: + # Reset all stateful properties so connection + # can be re-used without leaking prior configs. + self.sock = None + self.is_verified = False + self.proxy_is_verified = None + self._has_connected_to_proxy = False + self._response_options = None + self._tunnel_host = None + self._tunnel_port = None + self._tunnel_scheme = None - def putrequest(self, method, url, *args, **kwargs): - """ """ + def putrequest( + self, + method: str, + url: str, + skip_host: bool = False, + skip_accept_encoding: bool = False, + ) -> None: + """""" # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) if match: raise ValueError( - "Method cannot contain non-token characters %r (found at least %r)" - % (method, match.group()) + f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})" ) - return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) + return super().putrequest( + method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding + ) - def putheader(self, header, *values): - """ """ + def putheader(self, header: str, *values: str) -> None: # type: ignore[override] + """""" if not any(isinstance(v, str) and v == SKIP_HEADER for v in values): - _HTTPConnection.putheader(self, header, *values) - elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS: + super().putheader(header, *values) + elif to_str(header.lower()) not in SKIPPABLE_HEADERS: + skippable_headers = "', '".join( + [str.title(header) for header in sorted(SKIPPABLE_HEADERS)] + ) raise ValueError( - "urllib3.util.SKIP_HEADER only supports '%s'" - % ("', '".join(map(str.title, sorted(SKIPPABLE_HEADERS))),) + f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'" ) - def request(self, method, url, body=None, headers=None): + # `request` method's signature intentionally violates LSP. + # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental. + def request( # type: ignore[override] + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: # Update the inner socket's timeout value to send the request. # This only triggers if the connection is re-used. - if getattr(self, "sock", None) is not None: + if self.sock is not None: self.sock.settimeout(self.timeout) + # Store these values to be fed into the HTTPResponse + # object later. TODO: Remove this in favor of a real + # HTTP lifecycle mechanism. + + # We have to store these before we call .request() + # because sometimes we can still salvage a response + # off the wire even if we aren't able to completely + # send the request body. + self._response_options = _ResponseOptions( + request_method=method, + request_url=url, + preload_content=preload_content, + decode_content=decode_content, + enforce_content_length=enforce_content_length, + ) + if headers is None: headers = {} - else: - # Avoid modifying the headers passed into .request() - headers = headers.copy() - if "user-agent" not in (six.ensure_str(k.lower()) for k in headers): - headers["User-Agent"] = _get_default_user_agent() - super(HTTPConnection, self).request(method, url, body=body, headers=headers) - - def request_chunked(self, method, url, body=None, headers=None): - """ - Alternative to the common request method, which sends the - body with chunked encoding and not as one block - """ - headers = headers or {} - header_keys = set([six.ensure_str(k.lower()) for k in headers]) + header_keys = frozenset(to_str(k.lower()) for k in headers) skip_accept_encoding = "accept-encoding" in header_keys skip_host = "host" in header_keys self.putrequest( method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host ) + + # Transform the body into an iterable of sendall()-able chunks + # and detect if an explicit Content-Length is doable. + chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize) + chunks = chunks_and_cl.chunks + content_length = chunks_and_cl.content_length + + # When chunked is explicit set to 'True' we respect that. + if chunked: + if "transfer-encoding" not in header_keys: + self.putheader("Transfer-Encoding", "chunked") + else: + # Detect whether a framing mechanism is already in use. If so + # we respect that value, otherwise we pick chunked vs content-length + # depending on the type of 'body'. + if "content-length" in header_keys: + chunked = False + elif "transfer-encoding" in header_keys: + chunked = True + + # Otherwise we go off the recommendation of 'body_to_chunks()'. + else: + chunked = False + if content_length is None: + if chunks is not None: + chunked = True + self.putheader("Transfer-Encoding", "chunked") + else: + self.putheader("Content-Length", str(content_length)) + + # Now that framing headers are out of the way we send all the other headers. if "user-agent" not in header_keys: self.putheader("User-Agent", _get_default_user_agent()) for header, value in headers.items(): self.putheader(header, value) - if "transfer-encoding" not in header_keys: - self.putheader("Transfer-Encoding", "chunked") self.endheaders() - if body is not None: - stringish_types = six.string_types + (bytes,) - if isinstance(body, stringish_types): - body = (body,) - for chunk in body: + # If we're given a body we start sending that in chunks. + if chunks is not None: + for chunk in chunks: + # Sending empty chunks isn't allowed for TE: chunked + # as it indicates the end of the body. if not chunk: continue - if not isinstance(chunk, bytes): - chunk = chunk.encode("utf8") - len_str = hex(len(chunk))[2:] - to_send = bytearray(len_str.encode()) - to_send += b"\r\n" - to_send += chunk - to_send += b"\r\n" - self.send(to_send) + if isinstance(chunk, str): + chunk = chunk.encode("utf-8") + if chunked: + self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk)) + else: + self.send(chunk) + + # Regardless of whether we have a body or not, if we're in + # chunked mode we want to send an explicit empty chunk. + if chunked: + self.send(b"0\r\n\r\n") + + def request_chunked( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + ) -> None: + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + warnings.warn( + "HTTPConnection.request_chunked() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).", + category=DeprecationWarning, + stacklevel=2, + ) + self.request(method, url, body=body, headers=headers, chunked=True) + + def getresponse( # type: ignore[override] + self, + ) -> HTTPResponse: + """ + Get the response from the server. + + If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable. + + If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. + """ + # Raise the same error as http.client.HTTPConnection + if self._response_options is None: + raise ResponseNotReady() + + # Reset this attribute for being used again. + resp_options = self._response_options + self._response_options = None + + # Since the connection's timeout value may have been updated + # we need to set the timeout on the socket. + self.sock.settimeout(self.timeout) + + # This is needed here to avoid circular import errors + from .response import HTTPResponse + + # Save a reference to the shutdown function before ownership is passed + # to httplib_response + # TODO should we implement it everywhere? + _shutdown = getattr(self.sock, "shutdown", None) + + # Get the response from http.client.HTTPConnection + httplib_response = super().getresponse() + + try: + assert_header_parsing(httplib_response.msg) + except (HeaderParsingError, TypeError) as hpe: + log.warning( + "Failed to parse headers (url=%s): %s", + _url_from_connection(self, resp_options.request_url), + hpe, + exc_info=True, + ) + + headers = HTTPHeaderDict(httplib_response.msg.items()) - # After the if clause, to always have a closed body - self.send(b"0\r\n\r\n") + response = HTTPResponse( + body=httplib_response, + headers=headers, + status=httplib_response.status, + version=httplib_response.version, + version_string=getattr(self, "_http_vsn_str", "HTTP/?"), + reason=httplib_response.reason, + preload_content=resp_options.preload_content, + decode_content=resp_options.decode_content, + original_response=httplib_response, + enforce_content_length=resp_options.enforce_content_length, + request_method=resp_options.request_method, + request_url=resp_options.request_url, + sock_shutdown=_shutdown, + ) + return response class HTTPSConnection(HTTPConnection): @@ -289,57 +606,103 @@ class HTTPSConnection(HTTPConnection): socket by means of :py:func:`urllib3.util.ssl_wrap_socket`. """ - default_port = port_by_scheme["https"] + default_port = port_by_scheme["https"] # type: ignore[misc] - cert_reqs = None - ca_certs = None - ca_cert_dir = None - ca_cert_data = None - ssl_version = None - assert_fingerprint = None - tls_in_tls_required = False + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_fingerprint: str | None = None + _connect_callback: typing.Callable[..., None] | None = None def __init__( self, - host, - port=None, - key_file=None, - cert_file=None, - key_password=None, - strict=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, - server_hostname=None, - **kw - ): - - HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None | ( + connection._TYPE_SOCKET_OPTIONS + ) = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) self.key_file = key_file self.cert_file = cert_file self.key_password = key_password self.ssl_context = ssl_context self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data - # Required property for Google AppEngine 1.9.0 which otherwise causes - # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = "https" + # cert_reqs depends on ssl_context so calculate last. + if cert_reqs is None: + if self.ssl_context is not None: + cert_reqs = self.ssl_context.verify_mode + else: + cert_reqs = resolve_cert_reqs(None) + self.cert_reqs = cert_reqs + self._connect_callback = None def set_cert( self, - key_file=None, - cert_file=None, - cert_reqs=None, - key_password=None, - ca_certs=None, - assert_hostname=None, - assert_fingerprint=None, - ca_cert_dir=None, - ca_cert_data=None, - ): + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: """ This method should only be called once, before the connection is used. """ + warnings.warn( + "HTTPSConnection.set_cert() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead provide the parameters to the " + "HTTPSConnection constructor.", + category=DeprecationWarning, + stacklevel=2, + ) + # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also # have an SSLContext object in which case we'll use its verify_mode. if cert_reqs is None: @@ -358,191 +721,322 @@ class HTTPSConnection(HTTPConnection): self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) self.ca_cert_data = ca_cert_data - def connect(self): - # Add certificate verification - self.sock = conn = self._new_conn() - hostname = self.host - tls_in_tls = False - - if self._is_using_tunnel(): - if self.tls_in_tls_required: - self.sock = conn = self._connect_tls_proxy(hostname, conn) - tls_in_tls = True - - # Calls self._set_hostport(), so self.host is - # self._tunnel_host below. - self._tunnel() - # Mark this connection as not reusable - self.auto_open = 0 - - # Override the host with the one we're requesting data from. - hostname = self._tunnel_host - - server_hostname = hostname - if self.server_hostname is not None: - server_hostname = self.server_hostname + def connect(self) -> None: + # Today we don't need to be doing this step before the /actual/ socket + # connection, however in the future we'll need to decide whether to + # create a new socket or re-use an existing "shared" socket as a part + # of the HTTP/2 handshake dance. + if self._tunnel_host is not None and self._tunnel_port is not None: + probe_http2_host = self._tunnel_host + probe_http2_port = self._tunnel_port + else: + probe_http2_host = self.host + probe_http2_port = self.port - is_time_off = datetime.date.today() < RECENT_DATE - if is_time_off: - warnings.warn( - ( - "System time is way off (before {0}). This will probably " - "lead to SSL verification errors" - ).format(RECENT_DATE), - SystemTimeWarning, + # Check if the target origin supports HTTP/2. + # If the value comes back as 'None' it means that the current thread + # is probing for HTTP/2 support. Otherwise, we're waiting for another + # probe to complete, or we get a value right away. + target_supports_http2: bool | None + if "h2" in ssl_.ALPN_PROTOCOLS: + target_supports_http2 = http2_probe.acquire_and_get( + host=probe_http2_host, port=probe_http2_port ) + else: + # If HTTP/2 isn't going to be offered it doesn't matter if + # the target supports HTTP/2. Don't want to make a probe. + target_supports_http2 = False - # Wrap socket using verification with the root certs in - # trusted_root_certs - default_ssl_context = False - if self.ssl_context is None: - default_ssl_context = True - self.ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(self.ssl_version), - cert_reqs=resolve_cert_reqs(self.cert_reqs), + if self._connect_callback is not None: + self._connect_callback( + "before connect", + thread_id=threading.get_ident(), + target_supports_http2=target_supports_http2, ) - context = self.ssl_context - context.verify_mode = resolve_cert_reqs(self.cert_reqs) + try: + sock: socket.socket | ssl.SSLSocket + self.sock = sock = self._new_conn() + server_hostname: str = self.host + tls_in_tls = False - # Try to load OS default certs if none are given. - # Works well on Windows (requires Python3.4+) - if ( - not self.ca_certs - and not self.ca_cert_dir - and not self.ca_cert_data - and default_ssl_context - and hasattr(context, "load_default_certs") - ): - context.load_default_certs() + # Do we need to establish a tunnel? + if self.proxy_is_tunneling: + # We're tunneling to an HTTPS origin so need to do TLS-in-TLS. + if self._tunnel_scheme == "https": + # _connect_tls_proxy will verify and assign proxy_is_verified + self.sock = sock = self._connect_tls_proxy(self.host, sock) + tls_in_tls = True + elif self._tunnel_scheme == "http": + self.proxy_is_verified = False - self.sock = ssl_wrap_socket( - sock=conn, - keyfile=self.key_file, - certfile=self.cert_file, - key_password=self.key_password, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - ca_cert_data=self.ca_cert_data, - server_hostname=server_hostname, - ssl_context=context, - tls_in_tls=tls_in_tls, - ) + # If we're tunneling it means we're connected to our proxy. + self._has_connected_to_proxy = True - # If we're using all defaults and the connection - # is TLSv1 or TLSv1.1 we throw a DeprecationWarning - # for the host. - if ( - default_ssl_context - and self.ssl_version is None - and hasattr(self.sock, "version") - and self.sock.version() in {"TLSv1", "TLSv1.1"} - ): # Defensive: - warnings.warn( - "Negotiating TLSv1/TLSv1.1 by default is deprecated " - "and will be disabled in urllib3 v2.0.0. Connecting to " - "'%s' with '%s' can be enabled by explicitly opting-in " - "with 'ssl_version'" % (self.host, self.sock.version()), - DeprecationWarning, - ) + self._tunnel() + # Override the host with the one we're requesting data from. + server_hostname = typing.cast(str, self._tunnel_host) - if self.assert_fingerprint: - assert_fingerprint( - self.sock.getpeercert(binary_form=True), self.assert_fingerprint - ) - elif ( - context.verify_mode != ssl.CERT_NONE - and not getattr(context, "check_hostname", False) - and self.assert_hostname is not False - ): - # While urllib3 attempts to always turn off hostname matching from - # the TLS library, this cannot always be done. So we check whether - # the TLS Library still thinks it's matching hostnames. - cert = self.sock.getpeercert() - if not cert.get("subjectAltName", ()): + if self.server_hostname is not None: + server_hostname = self.server_hostname + + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: warnings.warn( ( - "Certificate for {0} has no `subjectAltName`, falling back to check for a " - "`commonName` for now. This feature is being removed by major browsers and " - "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " - "for details.)".format(hostname) + f"System time is way off (before {RECENT_DATE}). This will probably " + "lead to SSL verification errors" ), - SubjectAltNameWarning, + SystemTimeWarning, ) - _match_hostname(cert, self.assert_hostname or server_hostname) - self.is_verified = ( - context.verify_mode == ssl.CERT_REQUIRED - or self.assert_fingerprint is not None - ) + # Remove trailing '.' from fqdn hostnames to allow certificate validation + server_hostname_rm_dot = server_hostname.rstrip(".") + + sock_and_verified = _ssl_wrap_socket_and_match_hostname( + sock=sock, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, + cert_file=self.cert_file, + key_file=self.key_file, + key_password=self.key_password, + server_hostname=server_hostname_rm_dot, + ssl_context=self.ssl_context, + tls_in_tls=tls_in_tls, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ) + self.sock = sock_and_verified.socket + + # If an error occurs during connection/handshake we may need to release + # our lock so another connection can probe the origin. + except BaseException: + if self._connect_callback is not None: + self._connect_callback( + "after connect failure", + thread_id=threading.get_ident(), + target_supports_http2=target_supports_http2, + ) + + if target_supports_http2 is None: + http2_probe.set_and_release( + host=probe_http2_host, port=probe_http2_port, supports_http2=None + ) + raise - def _connect_tls_proxy(self, hostname, conn): + # If this connection doesn't know if the origin supports HTTP/2 + # we report back to the HTTP/2 probe our result. + if target_supports_http2 is None: + supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2" + http2_probe.set_and_release( + host=probe_http2_host, + port=probe_http2_port, + supports_http2=supports_http2, + ) + + # Forwarding proxies can never have a verified target since + # the proxy is the one doing the verification. Should instead + # use a CONNECT tunnel in order to verify the target. + # See: https://github.com/urllib3/urllib3/issues/3267. + if self.proxy_is_forwarding: + self.is_verified = False + else: + self.is_verified = sock_and_verified.is_verified + + # If there's a proxy to be connected to we are fully connected. + # This is set twice (once above and here) due to forwarding proxies + # not using tunnelling. + self._has_connected_to_proxy = bool(self.proxy) + + # Set `self.proxy_is_verified` unless it's already set while + # establishing a tunnel. + if self._has_connected_to_proxy and self.proxy_is_verified is None: + self.proxy_is_verified = sock_and_verified.is_verified + + def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket: """ Establish a TLS connection to the proxy using the provided SSL context. """ - proxy_config = self.proxy_config + # `_connect_tls_proxy` is called when self._tunnel_host is truthy. + proxy_config = typing.cast(ProxyConfig, self.proxy_config) ssl_context = proxy_config.ssl_context - if ssl_context: - # If the user provided a proxy context, we assume CA and client - # certificates have already been set - return ssl_wrap_socket( - sock=conn, - server_hostname=hostname, - ssl_context=ssl_context, - ) - - ssl_context = create_proxy_ssl_context( - self.ssl_version, - self.cert_reqs, - self.ca_certs, - self.ca_cert_dir, - self.ca_cert_data, - ) - - # If no cert was provided, use only the default options for server - # certificate validation - socket = ssl_wrap_socket( - sock=conn, + sock_and_verified = _ssl_wrap_socket_and_match_hostname( + sock, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, ca_cert_data=self.ca_cert_data, server_hostname=hostname, ssl_context=ssl_context, + assert_hostname=proxy_config.assert_hostname, + assert_fingerprint=proxy_config.assert_fingerprint, + # Features that aren't implemented for proxies yet: + cert_file=None, + key_file=None, + key_password=None, + tls_in_tls=False, ) + self.proxy_is_verified = sock_and_verified.is_verified + return sock_and_verified.socket # type: ignore[return-value] + + +class _WrappedAndVerifiedSocket(typing.NamedTuple): + """ + Wrapped socket and whether the connection is + verified after the TLS handshake + """ + + socket: ssl.SSLSocket | SSLTransport + is_verified: bool + + +def _ssl_wrap_socket_and_match_hostname( + sock: socket.socket, + *, + cert_reqs: None | str | int, + ssl_version: None | str | int, + ssl_minimum_version: int | None, + ssl_maximum_version: int | None, + cert_file: str | None, + key_file: str | None, + key_password: str | None, + ca_certs: str | None, + ca_cert_dir: str | None, + ca_cert_data: None | str | bytes, + assert_hostname: None | str | typing.Literal[False], + assert_fingerprint: str | None, + server_hostname: str | None, + ssl_context: ssl.SSLContext | None, + tls_in_tls: bool = False, +) -> _WrappedAndVerifiedSocket: + """Logic for constructing an SSLContext from all TLS parameters, passing + that down into ssl_wrap_socket, and then doing certificate verification + either via hostname or fingerprint. This function exists to guarantee + that both proxies and targets have the same behavior when connecting via TLS. + """ + default_ssl_context = False + if ssl_context is None: + default_ssl_context = True + context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + ssl_minimum_version=ssl_minimum_version, + ssl_maximum_version=ssl_maximum_version, + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + else: + context = ssl_context + + context.verify_mode = resolve_cert_reqs(cert_reqs) + + # In some cases, we want to verify hostnames ourselves + if ( + # `ssl` can't verify fingerprints or alternate hostnames + assert_fingerprint + or assert_hostname + # assert_hostname can be set to False to disable hostname checking + or assert_hostname is False + # We still support OpenSSL 1.0.2, which prevents us from verifying + # hostnames easily: https://github.com/pyca/pyopenssl/pull/933 + or ssl_.IS_PYOPENSSL + or not ssl_.HAS_NEVER_CHECK_COMMON_NAME + ): + context.check_hostname = False + + # Try to load OS default certs if none are given. We need to do the hasattr() check + # for custom pyOpenSSL SSLContext objects because they don't support + # load_default_certs(). + if ( + not ca_certs + and not ca_cert_dir + and not ca_cert_data + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + + # Ensure that IPv6 addresses are in the proper format and don't have a + # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses + # and interprets them as DNS hostnames. + if server_hostname is not None: + normalized = server_hostname.strip("[]") + if "%" in normalized: + normalized = normalized[: normalized.rfind("%")] + if is_ipaddress(normalized): + server_hostname = normalized + + ssl_sock = ssl_wrap_socket( + sock=sock, + keyfile=key_file, + certfile=cert_file, + key_password=key_password, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ca_cert_data=ca_cert_data, + server_hostname=server_hostname, + ssl_context=context, + tls_in_tls=tls_in_tls, + ) - if ssl_context.verify_mode != ssl.CERT_NONE and not getattr( - ssl_context, "check_hostname", False + try: + if assert_fingerprint: + _assert_fingerprint( + ssl_sock.getpeercert(binary_form=True), assert_fingerprint + ) + elif ( + context.verify_mode != ssl.CERT_NONE + and not context.check_hostname + and assert_hostname is not False ): - # While urllib3 attempts to always turn off hostname matching from - # the TLS library, this cannot always be done. So we check whether - # the TLS Library still thinks it's matching hostnames. - cert = socket.getpeercert() - if not cert.get("subjectAltName", ()): - warnings.warn( - ( - "Certificate for {0} has no `subjectAltName`, falling back to check for a " - "`commonName` for now. This feature is being removed by major browsers and " - "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " - "for details.)".format(hostname) - ), - SubjectAltNameWarning, + cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment] + + # Need to signal to our match_hostname whether to use 'commonName' or not. + # If we're using our own constructed SSLContext we explicitly set 'False' + # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name. + if default_ssl_context: + hostname_checks_common_name = False + else: + hostname_checks_common_name = ( + getattr(context, "hostname_checks_common_name", False) or False ) - _match_hostname(cert, hostname) - self.proxy_is_verified = ssl_context.verify_mode == ssl.CERT_REQUIRED - return socket + _match_hostname( + cert, + assert_hostname or server_hostname, # type: ignore[arg-type] + hostname_checks_common_name, + ) + + return _WrappedAndVerifiedSocket( + socket=ssl_sock, + is_verified=context.verify_mode == ssl.CERT_REQUIRED + or bool(assert_fingerprint), + ) + except BaseException: + ssl_sock.close() + raise -def _match_hostname(cert, asserted_hostname): +def _match_hostname( + cert: _TYPE_PEER_CERT_RET_DICT | None, + asserted_hostname: str, + hostname_checks_common_name: bool = False, +) -> None: # Our upstream implementation of ssl.match_hostname() # only applies this normalization to IP addresses so it doesn't # match DNS SANs so we do the same thing! - stripped_hostname = asserted_hostname.strip("u[]") + stripped_hostname = asserted_hostname.strip("[]") if is_ipaddress(stripped_hostname): asserted_hostname = stripped_hostname try: - match_hostname(cert, asserted_hostname) + match_hostname(cert, asserted_hostname, hostname_checks_common_name) except CertificateError as e: log.warning( "Certificate did not match expected hostname: %s. Certificate: %s", @@ -551,22 +1045,55 @@ def _match_hostname(cert, asserted_hostname): ) # Add cert to exception and reraise so client code can inspect # the cert when catching the exception, if they want to - e._peer_cert = cert + e._peer_cert = cert # type: ignore[attr-defined] raise -def _get_default_user_agent(): - return "python-urllib3/%s" % __version__ +def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError: + # Look for the phrase 'wrong version number', if found + # then we should warn the user that we're very sure that + # this proxy is HTTP-only and they have a configuration issue. + error_normalized = " ".join(re.split("[^a-z]", str(err).lower())) + is_likely_http_proxy = ( + "wrong version number" in error_normalized + or "unknown protocol" in error_normalized + or "record layer failure" in error_normalized + ) + http_proxy_warning = ( + ". Your proxy appears to only use HTTP and not HTTPS, " + "try changing your proxy URL to be HTTP. See: " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#https-proxy-error-http-proxy" + ) + new_err = ProxyError( + f"Unable to connect to proxy" + f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}", + err, + ) + new_err.__cause__ = err + return new_err -class DummyConnection(object): - """Used to detect a failed ConnectionCls import.""" +def _get_default_user_agent() -> str: + return f"python-urllib3/{__version__}" - pass + +class DummyConnection: + """Used to detect a failed ConnectionCls import.""" if not ssl: - HTTPSConnection = DummyConnection # noqa: F811 + HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811 VerifiedHTTPSConnection = HTTPSConnection + + +def _url_from_connection( + conn: HTTPConnection | HTTPSConnection, path: str | None = None +) -> str: + """Returns the URL from a given connection. This is mainly used for testing and logging.""" + + scheme = "https" if isinstance(conn, HTTPSConnection) else "http" + + return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url diff --git a/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py b/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py index 0872ed77011..3a0685b4cdd 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py +++ b/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py @@ -1,15 +1,18 @@ -from __future__ import absolute_import +from __future__ import annotations import errno import logging -import re -import socket +import queue import sys +import typing import warnings -from socket import error as SocketError +import weakref from socket import timeout as SocketTimeout +from types import TracebackType +from ._base_connection import _TYPE_BODY from ._collections import HTTPHeaderDict +from ._request_methods import RequestMethods from .connection import ( BaseSSLError, BrokenPipeError, @@ -17,13 +20,14 @@ from .connection import ( HTTPConnection, HTTPException, HTTPSConnection, - VerifiedHTTPSConnection, - port_by_scheme, + ProxyConfig, + _wrap_proxy_error, ) +from .connection import port_by_scheme as port_by_scheme from .exceptions import ( ClosedPoolError, EmptyPoolError, - HeaderParsingError, + FullPoolError, HostChangedError, InsecureRequestWarning, LocationValueError, @@ -35,38 +39,32 @@ from .exceptions import ( SSLError, TimeoutError, ) -from .packages import six -from .packages.six.moves import queue -from .request import RequestMethods -from .response import HTTPResponse +from .response import BaseHTTPResponse from .util.connection import is_connection_dropped from .util.proxy import connection_requires_http_tunnel -from .util.queue import LifoQueue -from .util.request import set_file_position -from .util.response import assert_header_parsing +from .util.request import _TYPE_BODY_POSITION, set_file_position from .util.retry import Retry from .util.ssl_match_hostname import CertificateError -from .util.timeout import Timeout +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout from .util.url import Url, _encode_target from .util.url import _normalize_host as normalize_host -from .util.url import get_host, parse_url +from .util.url import parse_url +from .util.util import to_str -try: # Platform-specific: Python 3 - import weakref +if typing.TYPE_CHECKING: + import ssl - weakref_finalize = weakref.finalize -except AttributeError: # Platform-specific: Python 2 - from .packages.backports.weakref_finalize import weakref_finalize + from typing_extensions import Self -xrange = six.moves.xrange + from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection log = logging.getLogger(__name__) -_Default = object() +_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None] # Pool objects -class ConnectionPool(object): +class ConnectionPool: """ Base class for all connection pools, such as :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. @@ -77,33 +75,42 @@ class ConnectionPool(object): target URIs. """ - scheme = None - QueueCls = LifoQueue + scheme: str | None = None + QueueCls = queue.LifoQueue - def __init__(self, host, port=None): + def __init__(self, host: str, port: int | None = None) -> None: if not host: raise LocationValueError("No host specified.") self.host = _normalize_host(host, scheme=self.scheme) - self._proxy_host = host.lower() self.port = port - def __str__(self): - return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) + # This property uses 'normalize_host()' (not '_normalize_host()') + # to avoid removing square braces around IPv6 addresses. + # This value is sent to `HTTPConnection.set_tunnel()` if called + # because square braces are required for HTTP CONNECT tunneling. + self._tunnel_host = normalize_host(host, scheme=self.scheme).lower() - def __enter__(self): + def __str__(self) -> str: + return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})" + + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> typing.Literal[False]: self.close() # Return False to re-raise any potential exceptions return False - def close(self): + def close(self) -> None: """ Close all pooled connections and disable the pool. """ - pass # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 @@ -122,14 +129,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Port used for this HTTP Connection (None is equivalent to 80), passed into :class:`http.client.HTTPConnection`. - :param strict: - Causes BadStatusLine to be raised if the status line can't be parsed - as a valid HTTP/1.0 or 1.1 status line, passed into - :class:`http.client.HTTPConnection`. - - .. note:: - Only works in Python 2. This parameter is ignored in Python 3. - :param timeout: Socket timeout in seconds for each individual connection. This can be a float or integer, which sets the timeout for the HTTP request, @@ -171,29 +170,25 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ scheme = "http" - ConnectionCls = HTTPConnection - ResponseCls = HTTPResponse + ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection def __init__( self, - host, - port=None, - strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, - maxsize=1, - block=False, - headers=None, - retries=None, - _proxy=None, - _proxy_headers=None, - _proxy_config=None, - **conn_kw + host: str, + port: int | None = None, + timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT, + maxsize: int = 1, + block: bool = False, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + _proxy: Url | None = None, + _proxy_headers: typing.Mapping[str, str] | None = None, + _proxy_config: ProxyConfig | None = None, + **conn_kw: typing.Any, ): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) - self.strict = strict - if not isinstance(timeout, Timeout): timeout = Timeout.from_float(timeout) @@ -203,7 +198,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): self.timeout = timeout self.retries = retries - self.pool = self.QueueCls(maxsize) + self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize) self.block = block self.proxy = _proxy @@ -211,7 +206,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): self.proxy_config = _proxy_config # Fill the queue up so that doing get() on it will block properly - for _ in xrange(maxsize): + for _ in range(maxsize): self.pool.put(None) # These are mostly for testing and debugging purposes. @@ -236,9 +231,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Close all the HTTPConnections in the pool before the # HTTPConnectionPool object is garbage collected. - weakref_finalize(self, _close_pool_connections, pool) + weakref.finalize(self, _close_pool_connections, pool) - def _new_conn(self): + def _new_conn(self) -> BaseHTTPConnection: """ Return a fresh :class:`HTTPConnection`. """ @@ -254,12 +249,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): host=self.host, port=self.port, timeout=self.timeout.connect_timeout, - strict=self.strict, - **self.conn_kw + **self.conn_kw, ) return conn - def _get_conn(self, timeout=None): + def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection: """ Get a connection. Will return a pooled connection if one is available. @@ -272,33 +266,32 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :prop:`.block` is ``True``. """ conn = None + + if self.pool is None: + raise ClosedPoolError(self, "Pool is closed.") + try: conn = self.pool.get(block=self.block, timeout=timeout) except AttributeError: # self.pool is None - raise ClosedPoolError(self, "Pool is closed.") + raise ClosedPoolError(self, "Pool is closed.") from None # Defensive: except queue.Empty: if self.block: raise EmptyPoolError( self, - "Pool reached maximum size and no more connections are allowed.", - ) + "Pool is empty and a new connection can't be opened due to blocking mode.", + ) from None pass # Oh well, we'll create a new connection then # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) conn.close() - if getattr(conn, "auto_open", 1) == 0: - # This is a proxied connection that has been mutated by - # http.client._tunnel() and cannot be reused (since it would - # attempt to bypass the proxy) - conn = None return conn or self._new_conn() - def _put_conn(self, conn): + def _put_conn(self, conn: BaseHTTPConnection | None) -> None: """ Put a connection back into the pool. @@ -312,36 +305,47 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): If the pool is closed, then the connection will be closed and discarded. """ - try: - self.pool.put(conn, block=False) - return # Everything is dandy, done. - except AttributeError: - # self.pool is None. - pass - except queue.Full: - # This should never happen if self.block == True - log.warning( - "Connection pool is full, discarding connection: %s. Connection pool size: %s", - self.host, - self.pool.qsize(), - ) + if self.pool is not None: + try: + self.pool.put(conn, block=False) + return # Everything is dandy, done. + except AttributeError: + # self.pool is None. + pass + except queue.Full: + # Connection never got put back into the pool, close it. + if conn: + conn.close() + + if self.block: + # This should never happen if you got the conn from self._get_conn + raise FullPoolError( + self, + "Pool reached maximum size and no more connections are allowed.", + ) from None + + log.warning( + "Connection pool is full, discarding connection: %s. Connection pool size: %s", + self.host, + self.pool.qsize(), + ) + # Connection never got put back into the pool, close it. if conn: conn.close() - def _validate_conn(self, conn): + def _validate_conn(self, conn: BaseHTTPConnection) -> None: """ Called right before a request is made, after the socket is created. """ - pass - def _prepare_proxy(self, conn): + def _prepare_proxy(self, conn: BaseHTTPConnection) -> None: # Nothing to do for HTTP connections. pass - def _get_timeout(self, timeout): + def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout: """Helper that always returns a :class:`urllib3.util.Timeout`""" - if timeout is _Default: + if timeout is _DEFAULT_TIMEOUT: return self.timeout.clone() if isinstance(timeout, Timeout): @@ -351,34 +355,40 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # can be removed later return Timeout.from_float(timeout) - def _raise_timeout(self, err, url, timeout_value): + def _raise_timeout( + self, + err: BaseSSLError | OSError | SocketTimeout, + url: str, + timeout_value: _TYPE_TIMEOUT | None, + ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) + self, url, f"Read timed out. (read timeout={timeout_value})" + ) from err - # See the above comment about EAGAIN in Python 3. In Python 2 we have - # to specifically catch it and throw the timeout error + # See the above comment about EAGAIN in Python 3. if hasattr(err, "errno") and err.errno in _blocking_errnos: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) - - # Catch possible read timeouts thrown as SSL errors. If not the - # case, rethrow the original. We need to do this because of: - # http://bugs.python.org/issue10272 - if "timed out" in str(err) or "did not complete (read)" in str( - err - ): # Python < 2.7.4 - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) + self, url, f"Read timed out. (read timeout={timeout_value})" + ) from err def _make_request( - self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw - ): + self, + conn: BaseHTTPConnection, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | None = None, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + chunked: bool = False, + response_conn: BaseHTTPConnection | None = None, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. @@ -386,12 +396,61 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :param conn: a connection from one of our connection pools + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + :param timeout: - Socket timeout in seconds for the request. This can be a - float or integer, which will set the same timeout value for - the socket connect and the socket read, or an instance of - :class:`urllib3.util.Timeout`, which gives you more fine-grained - control over your timeouts. + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param chunked: + If True, urllib3 will send the body using chunked transfer + encoding. Otherwise, urllib3 will send the body using the standard + content-length form. Defaults to False. + + :param response_conn: + Set this to ``None`` if you will handle releasing the connection or + set the connection to have the response release it. + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param enforce_content_length: + Enforce content length checking. Body returned by server must match + value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 @@ -399,45 +458,66 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) - # Trigger any extra validation we need to do. try: - self._validate_conn(conn) - except (SocketTimeout, BaseSSLError) as e: - # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. - self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) - raise + # Trigger any extra validation we need to do. + try: + self._validate_conn(conn) + except (SocketTimeout, BaseSSLError) as e: + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise + + # _validate_conn() starts the connection to an HTTPS proxy + # so we need to wrap errors with 'ProxyError' here too. + except ( + OSError, + NewConnectionError, + TimeoutError, + BaseSSLError, + CertificateError, + SSLError, + ) as e: + new_e: Exception = e + if isinstance(e, (BaseSSLError, CertificateError)): + new_e = SSLError(e) + # If the connection didn't successfully connect to it's proxy + # then there + if isinstance( + new_e, (OSError, NewConnectionError, TimeoutError, SSLError) + ) and (conn and conn.proxy and not conn.has_connected_to_proxy): + new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) + raise new_e # conn.request() calls http.client.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. try: - if chunked: - conn.request_chunked(method, url, **httplib_request_kw) - else: - conn.request(method, url, **httplib_request_kw) + conn.request( + method, + url, + body=body, + headers=headers, + chunked=chunked, + preload_content=preload_content, + decode_content=decode_content, + enforce_content_length=enforce_content_length, + ) # We are swallowing BrokenPipeError (errno.EPIPE) since the server is # legitimately able to close the connection after sending a valid response. # With this behaviour, the received response is still readable. except BrokenPipeError: - # Python 3 pass - except IOError as e: - # Python 2 and macOS/Linux - # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE/ECONNRESET are needed on macOS + except OSError as e: + # MacOS/Linux + # EPROTOTYPE and ECONNRESET are needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ - if e.errno not in { - errno.EPIPE, - errno.ESHUTDOWN, - errno.EPROTOTYPE, - errno.ECONNRESET, - }: + # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE. + if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET: raise # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout - # App Engine doesn't have a sock attr - if getattr(conn, "sock", None): + if not conn.is_closed: # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -445,33 +525,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout + self, url, f"Read timed out. (read timeout={read_timeout})" ) - if read_timeout is Timeout.DEFAULT_TIMEOUT: - conn.sock.settimeout(socket.getdefaulttimeout()) - else: # None or a value - conn.sock.settimeout(read_timeout) + conn.timeout = read_timeout # Receive the response from the server try: - try: - # Python 2.7, use buffering of HTTP responses - httplib_response = conn.getresponse(buffering=True) - except TypeError: - # Python 3 - try: - httplib_response = conn.getresponse() - except BaseException as e: - # Remove the TypeError from the exception chain in - # Python 3 (including for exceptions like SystemExit). - # Otherwise it looks like a bug in the code. - six.raise_from(e, None) - except (SocketTimeout, BaseSSLError, SocketError) as e: + response = conn.getresponse() + except (BaseSSLError, OSError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise - # AppEngine doesn't have a version attr. - http_version = getattr(conn, "_http_vsn_str", "HTTP/?") + # Set properties that are used by the pooling layer. + response.retries = retries + response._connection = response_conn # type: ignore[attr-defined] + response._pool = self # type: ignore[attr-defined] + log.debug( '%s://%s:%s "%s %s %s" %s %s', self.scheme, @@ -479,27 +548,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): self.port, method, url, - http_version, - httplib_response.status, - httplib_response.length, + response.version_string, + response.status, + response.length_remaining, ) - try: - assert_header_parsing(httplib_response.msg) - except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 - log.warning( - "Failed to parse headers (url=%s): %s", - self._absolute_url(url), - hpe, - exc_info=True, - ) - - return httplib_response - - def _absolute_url(self, path): - return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url + return response - def close(self): + def close(self) -> None: """ Close all pooled connections and disable the pool. """ @@ -511,7 +567,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Close all the HTTPConnections in the pool. _close_pool_connections(old_pool) - def is_same_host(self, url): + def is_same_host(self, url: str) -> bool: """ Check if the given ``url`` is a member of the same host as this connection pool. @@ -520,7 +576,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return True # TODO: Add optional support for socket.gethostbyname checking. - scheme, host, port = get_host(url) + scheme, _, host, port, *_ = parse_url(url) + scheme = scheme or "http" if host is not None: host = _normalize_host(host, scheme=scheme) @@ -532,22 +589,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen( + def urlopen( # type: ignore[override] self, - method, - url, - body=None, - headers=None, - retries=None, - redirect=True, - assert_same_host=True, - timeout=_Default, - pool_timeout=None, - release_conn=None, - chunked=False, - body_pos=None, - **response_kw - ): + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + redirect: bool = True, + assert_same_host: bool = True, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + pool_timeout: int | None = None, + release_conn: bool | None = None, + chunked: bool = False, + body_pos: _TYPE_BODY_POSITION | None = None, + preload_content: bool = True, + decode_content: bool = True, + **response_kw: typing.Any, + ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -555,8 +614,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): .. note:: - More commonly, it's appropriate to use a convenience method provided - by :class:`.RequestMethods`, such as :meth:`request`. + More commonly, it's appropriate to use a convenience method + such as :meth:`request`. .. note:: @@ -584,7 +643,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. - Pass ``None`` to retry until you receive a response. Pass a + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, @@ -616,6 +675,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. + :param bool preload_content: + If True, the response's body will be preloaded into memory. + + :param bool decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if @@ -623,10 +689,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection - back into the pool. If None, it takes the value of - ``response_kw.get('preload_content', True)``. + back into the pool. If None, it takes the value of ``preload_content`` + which defaults to ``True``. - :param chunked: + :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. @@ -635,12 +701,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. - - :param \\**response_kw: - Additional parameters are passed to - :meth:`urllib3.response.HTTPResponse.from_httplib` """ - parsed_url = parse_url(url) destination_scheme = parsed_url.scheme @@ -651,7 +712,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: - release_conn = response_kw.get("preload_content", True) + release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): @@ -659,9 +720,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): - url = six.ensure_str(_encode_target(url)) + url = to_str(_encode_target(url)) else: - url = six.ensure_str(parsed_url.url) + url = to_str(parsed_url.url) conn = None @@ -684,8 +745,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: - headers = headers.copy() - headers.update(self.proxy_headers) + headers = headers.copy() # type: ignore[attr-defined] + headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. @@ -704,16 +765,26 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) - conn.timeout = timeout_obj.connect_timeout + conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] - is_new_proxy_conn = self.proxy is not None and not getattr( - conn, "sock", None - ) - if is_new_proxy_conn and http_tunnel_required: - self._prepare_proxy(conn) + # Is this a closed/new connection that requires CONNECT tunnelling? + if self.proxy is not None and http_tunnel_required and conn.is_closed: + try: + self._prepare_proxy(conn) + except (BaseSSLError, OSError, SocketTimeout) as e: + self._raise_timeout( + err=e, url=self.proxy.url, timeout_value=conn.timeout + ) + raise + + # If we're going to release the connection in ``finally:``, then + # the response doesn't need to know about the connection. Otherwise + # it will also try to release it and we'll have a double-release + # mess. + response_conn = conn if not release_conn else None - # Make the request on the httplib connection object. - httplib_response = self._make_request( + # Make the request on the HTTPConnection object + response = self._make_request( conn, method, url, @@ -721,24 +792,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): body=body, headers=headers, chunked=chunked, - ) - - # If we're going to release the connection in ``finally:``, then - # the response doesn't need to know about the connection. Otherwise - # it will also try to release it and we'll have a double-release - # mess. - response_conn = conn if not release_conn else None - - # Pass method to Response for length checking - response_kw["request_method"] = method - - # Import httplib's response into our own wrapper object - response = self.ResponseCls.from_httplib( - httplib_response, - pool=self, - connection=response_conn, retries=retries, - **response_kw + response_conn=response_conn, + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Everything went great! @@ -753,54 +811,35 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except ( TimeoutError, HTTPException, - SocketError, + OSError, ProtocolError, BaseSSLError, SSLError, CertificateError, + ProxyError, ) as e: # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False - - def _is_ssl_error_message_from_http_proxy(ssl_error): - # We're trying to detect the message 'WRONG_VERSION_NUMBER' but - # SSLErrors are kinda all over the place when it comes to the message, - # so we try to cover our bases here! - message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) - return ( - "wrong version number" in message - or "unknown protocol" in message - or "record layer failure" in message - ) - - # Try to detect a common user error with proxies which is to - # set an HTTP proxy to be HTTPS when it should be 'http://' - # (ie {'http': 'http://proxy', 'https': 'https://proxy'}) - # Instead we add a nice error message and point to a URL. - if ( - isinstance(e, BaseSSLError) - and self.proxy - and _is_ssl_error_message_from_http_proxy(e) - and conn.proxy - and conn.proxy.scheme == "https" - ): - e = ProxyError( - "Your proxy appears to only use HTTP and not HTTPS, " - "try changing your proxy URL to be HTTP. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#https-proxy-error-http-proxy", - SSLError(e), - ) - elif isinstance(e, (BaseSSLError, CertificateError)): - e = SSLError(e) - elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: - e = ProxyError("Cannot connect to proxy.", e) - elif isinstance(e, (SocketError, HTTPException)): - e = ProtocolError("Connection aborted.", e) + new_e: Exception = e + if isinstance(e, (BaseSSLError, CertificateError)): + new_e = SSLError(e) + if isinstance( + new_e, + ( + OSError, + NewConnectionError, + TimeoutError, + SSLError, + HTTPException, + ), + ) and (conn and conn.proxy and not conn.has_connected_to_proxy): + new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) + elif isinstance(new_e, (OSError, HTTPException)): + new_e = ProtocolError("Connection aborted.", new_e) retries = retries.increment( - method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2] ) retries.sleep() @@ -813,7 +852,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # to throw the connection away unless explicitly told not to. # Close the connection, set the variable to None, and make sure # we put the None back in the pool to avoid leaking it. - conn = conn and conn.close() + if conn: + conn.close() + conn = None release_this_conn = True if release_this_conn: @@ -840,7 +881,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Handle redirect? @@ -877,7 +920,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Check if we should retry the HTTP response. @@ -907,7 +952,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) return response @@ -928,37 +975,35 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ scheme = "https" - ConnectionCls = HTTPSConnection + ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection def __init__( self, - host, - port=None, - strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, - maxsize=1, - block=False, - headers=None, - retries=None, - _proxy=None, - _proxy_headers=None, - key_file=None, - cert_file=None, - cert_reqs=None, - key_password=None, - ca_certs=None, - ssl_version=None, - assert_hostname=None, - assert_fingerprint=None, - ca_cert_dir=None, - **conn_kw - ): - - HTTPConnectionPool.__init__( - self, + host: str, + port: int | None = None, + timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT, + maxsize: int = 1, + block: bool = False, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + _proxy: Url | None = None, + _proxy_headers: typing.Mapping[str, str] | None = None, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + ssl_version: int | str | None = None, + ssl_minimum_version: ssl.TLSVersion | None = None, + ssl_maximum_version: ssl.TLSVersion | None = None, + assert_hostname: str | typing.Literal[False] | None = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + **conn_kw: typing.Any, + ) -> None: + super().__init__( host, port, - strict, timeout, maxsize, block, @@ -966,7 +1011,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): retries, _proxy, _proxy_headers, - **conn_kw + **conn_kw, ) self.key_file = key_file @@ -976,47 +1021,29 @@ class HTTPSConnectionPool(HTTPConnectionPool): self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - def _prepare_conn(self, conn): - """ - Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` - and establish the tunnel if proxy is used. - """ - - if isinstance(conn, VerifiedHTTPSConnection): - conn.set_cert( - key_file=self.key_file, - key_password=self.key_password, - cert_file=self.cert_file, - cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint, - ) - conn.ssl_version = self.ssl_version - return conn - - def _prepare_proxy(self, conn): - """ - Establishes a tunnel connection through HTTP CONNECT. - - Tunnel connection is established early because otherwise httplib would - improperly set Host: header to proxy's IP:port. - """ - - conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) - - if self.proxy.scheme == "https": - conn.tls_in_tls_required = True + def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override] + """Establishes a tunnel connection through HTTP CONNECT.""" + if self.proxy and self.proxy.scheme == "https": + tunnel_scheme = "https" + else: + tunnel_scheme = "http" + conn.set_tunnel( + scheme=tunnel_scheme, + host=self._tunnel_host, + port=self.port, + headers=self.proxy_headers, + ) conn.connect() - def _new_conn(self): + def _new_conn(self) -> BaseHTTPSConnection: """ - Return a fresh :class:`http.client.HTTPSConnection`. + Return a fresh :class:`urllib3.connection.HTTPConnection`. """ self.num_connections += 1 log.debug( @@ -1026,64 +1053,59 @@ class HTTPSConnectionPool(HTTPConnectionPool): self.port or "443", ) - if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - raise SSLError( + if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap] + raise ImportError( "Can't connect to HTTPS URL because the SSL module is not available." ) - actual_host = self.host + actual_host: str = self.host actual_port = self.port - if self.proxy is not None: + if self.proxy is not None and self.proxy.host is not None: actual_host = self.proxy.host actual_port = self.proxy.port - conn = self.ConnectionCls( + return self.ConnectionCls( host=actual_host, port=actual_port, timeout=self.timeout.connect_timeout, - strict=self.strict, cert_file=self.cert_file, key_file=self.key_file, key_password=self.key_password, - **self.conn_kw + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, + **self.conn_kw, ) - return self._prepare_conn(conn) - - def _validate_conn(self, conn): + def _validate_conn(self, conn: BaseHTTPConnection) -> None: """ Called right before a request is made, after the socket is created. """ - super(HTTPSConnectionPool, self)._validate_conn(conn) + super()._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not getattr(conn, "sock", None): # AppEngine might not have `.sock` + if conn.is_closed: conn.connect() - if not conn.is_verified: - warnings.warn( - ( - "Unverified HTTPS request is being made to host '%s'. " - "Adding certificate verification is strongly advised. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings" % conn.host - ), - InsecureRequestWarning, - ) - - if getattr(conn, "proxy_is_verified", None) is False: + # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791 + if not conn.is_verified and not conn.proxy_is_verified: warnings.warn( ( - "Unverified HTTPS connection done to an HTTPS proxy. " + f"Unverified HTTPS request is being made to host '{conn.host}'. " "Adding certificate verification is strongly advised. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings" + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#tls-warnings" ), InsecureRequestWarning, ) -def connection_from_url(url, **kw): +def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool: """ Given a url, return an :class:`.ConnectionPool` instance of its host. @@ -1103,15 +1125,24 @@ def connection_from_url(url, **kw): >>> conn = connection_from_url('http://google.com/') >>> r = conn.request('GET', '/') """ - scheme, host, port = get_host(url) + scheme, _, host, port, *_ = parse_url(url) + scheme = scheme or "http" port = port or port_by_scheme.get(scheme, 80) if scheme == "https": - return HTTPSConnectionPool(host, port=port, **kw) + return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type] else: - return HTTPConnectionPool(host, port=port, **kw) + return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type] + +def _normalize_host(host: None, scheme: str | None) -> None: ... -def _normalize_host(host, scheme): + +def _normalize_host(host: str, scheme: str | None) -> str: ... + + +def _normalize_host(host: str | None, scheme: str | None) -> str | None: """ Normalize hosts for comparisons and use with sockets. """ @@ -1124,12 +1155,19 @@ def _normalize_host(host, scheme): # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - if host.startswith("[") and host.endswith("]"): + if host and host.startswith("[") and host.endswith("]"): host = host[1:-1] return host -def _close_pool_connections(pool): +def _url_from_pool( + pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None +) -> str: + """Returns the URL from a given connection pool. This is mainly used for testing and logging.""" + return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url + + +def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None: """Drains a queue of connections and closes each one.""" try: while True: diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/_appengine_environ.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/_appengine_environ.py deleted file mode 100644 index 8765b907d70..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/_appengine_environ.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -This module provides means to detect the App Engine environment. -""" - -import os - - -def is_appengine(): - return is_local_appengine() or is_prod_appengine() - - -def is_appengine_sandbox(): - """Reports if the app is running in the first generation sandbox. - - The second generation runtimes are technically still in a sandbox, but it - is much less restrictive, so generally you shouldn't need to check for it. - see https://cloud.google.com/appengine/docs/standard/runtimes - """ - return is_appengine() and os.environ["APPENGINE_RUNTIME"] == "python27" - - -def is_local_appengine(): - return "APPENGINE_RUNTIME" in os.environ and os.environ.get( - "SERVER_SOFTWARE", "" - ).startswith("Development/") - - -def is_prod_appengine(): - return "APPENGINE_RUNTIME" in os.environ and os.environ.get( - "SERVER_SOFTWARE", "" - ).startswith("Google App Engine/") - - -def is_prod_appengine_mvms(): - """Deprecated.""" - return False diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/__init__.py +++ /dev/null diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/bindings.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/bindings.py deleted file mode 100644 index 264d564dbda..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/bindings.py +++ /dev/null @@ -1,519 +0,0 @@ -""" -This module uses ctypes to bind a whole bunch of functions and constants from -SecureTransport. The goal here is to provide the low-level API to -SecureTransport. These are essentially the C-level functions and constants, and -they're pretty gross to work with. - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - - Copyright (c) 2015-2016 Will Bond <[email protected]> - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" -from __future__ import absolute_import - -import platform -from ctypes import ( - CDLL, - CFUNCTYPE, - POINTER, - c_bool, - c_byte, - c_char_p, - c_int32, - c_long, - c_size_t, - c_uint32, - c_ulong, - c_void_p, -) -from ctypes.util import find_library - -from ...packages.six import raise_from - -if platform.system() != "Darwin": - raise ImportError("Only macOS is supported") - -version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split("."))) -if version_info < (10, 8): - raise OSError( - "Only OS X 10.8 and newer are supported, not %s.%s" - % (version_info[0], version_info[1]) - ) - - -def load_cdll(name, macos10_16_path): - """Loads a CDLL by name, falling back to known path on 10.16+""" - try: - # Big Sur is technically 11 but we use 10.16 due to the Big Sur - # beta being labeled as 10.16. - if version_info >= (10, 16): - path = macos10_16_path - else: - path = find_library(name) - if not path: - raise OSError # Caught and reraised as 'ImportError' - return CDLL(path, use_errno=True) - except OSError: - raise_from(ImportError("The library %s failed to load" % name), None) - - -Security = load_cdll( - "Security", "/System/Library/Frameworks/Security.framework/Security" -) -CoreFoundation = load_cdll( - "CoreFoundation", - "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", -) - - -Boolean = c_bool -CFIndex = c_long -CFStringEncoding = c_uint32 -CFData = c_void_p -CFString = c_void_p -CFArray = c_void_p -CFMutableArray = c_void_p -CFDictionary = c_void_p -CFError = c_void_p -CFType = c_void_p -CFTypeID = c_ulong - -CFTypeRef = POINTER(CFType) -CFAllocatorRef = c_void_p - -OSStatus = c_int32 - -CFDataRef = POINTER(CFData) -CFStringRef = POINTER(CFString) -CFArrayRef = POINTER(CFArray) -CFMutableArrayRef = POINTER(CFMutableArray) -CFDictionaryRef = POINTER(CFDictionary) -CFArrayCallBacks = c_void_p -CFDictionaryKeyCallBacks = c_void_p -CFDictionaryValueCallBacks = c_void_p - -SecCertificateRef = POINTER(c_void_p) -SecExternalFormat = c_uint32 -SecExternalItemType = c_uint32 -SecIdentityRef = POINTER(c_void_p) -SecItemImportExportFlags = c_uint32 -SecItemImportExportKeyParameters = c_void_p -SecKeychainRef = POINTER(c_void_p) -SSLProtocol = c_uint32 -SSLCipherSuite = c_uint32 -SSLContextRef = POINTER(c_void_p) -SecTrustRef = POINTER(c_void_p) -SSLConnectionRef = c_uint32 -SecTrustResultType = c_uint32 -SecTrustOptionFlags = c_uint32 -SSLProtocolSide = c_uint32 -SSLConnectionType = c_uint32 -SSLSessionOption = c_uint32 - - -try: - Security.SecItemImport.argtypes = [ - CFDataRef, - CFStringRef, - POINTER(SecExternalFormat), - POINTER(SecExternalItemType), - SecItemImportExportFlags, - POINTER(SecItemImportExportKeyParameters), - SecKeychainRef, - POINTER(CFArrayRef), - ] - Security.SecItemImport.restype = OSStatus - - Security.SecCertificateGetTypeID.argtypes = [] - Security.SecCertificateGetTypeID.restype = CFTypeID - - Security.SecIdentityGetTypeID.argtypes = [] - Security.SecIdentityGetTypeID.restype = CFTypeID - - Security.SecKeyGetTypeID.argtypes = [] - Security.SecKeyGetTypeID.restype = CFTypeID - - Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] - Security.SecCertificateCreateWithData.restype = SecCertificateRef - - Security.SecCertificateCopyData.argtypes = [SecCertificateRef] - Security.SecCertificateCopyData.restype = CFDataRef - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SecIdentityCreateWithCertificate.argtypes = [ - CFTypeRef, - SecCertificateRef, - POINTER(SecIdentityRef), - ] - Security.SecIdentityCreateWithCertificate.restype = OSStatus - - Security.SecKeychainCreate.argtypes = [ - c_char_p, - c_uint32, - c_void_p, - Boolean, - c_void_p, - POINTER(SecKeychainRef), - ] - Security.SecKeychainCreate.restype = OSStatus - - Security.SecKeychainDelete.argtypes = [SecKeychainRef] - Security.SecKeychainDelete.restype = OSStatus - - Security.SecPKCS12Import.argtypes = [ - CFDataRef, - CFDictionaryRef, - POINTER(CFArrayRef), - ] - Security.SecPKCS12Import.restype = OSStatus - - SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE( - OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) - ) - - Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] - Security.SSLSetIOFuncs.restype = OSStatus - - Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerID.restype = OSStatus - - Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetCertificate.restype = OSStatus - - Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] - Security.SSLSetCertificateAuthorities.restype = OSStatus - - Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] - Security.SSLSetConnection.restype = OSStatus - - Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerDomainName.restype = OSStatus - - Security.SSLHandshake.argtypes = [SSLContextRef] - Security.SSLHandshake.restype = OSStatus - - Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLRead.restype = OSStatus - - Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLWrite.restype = OSStatus - - Security.SSLClose.argtypes = [SSLContextRef] - Security.SSLClose.restype = OSStatus - - Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberSupportedCiphers.restype = OSStatus - - Security.SSLGetSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetSupportedCiphers.restype = OSStatus - - Security.SSLSetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - c_size_t, - ] - Security.SSLSetEnabledCiphers.restype = OSStatus - - Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberEnabledCiphers.restype = OSStatus - - Security.SSLGetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetEnabledCiphers.restype = OSStatus - - Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] - Security.SSLGetNegotiatedCipher.restype = OSStatus - - Security.SSLGetNegotiatedProtocolVersion.argtypes = [ - SSLContextRef, - POINTER(SSLProtocol), - ] - Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - - Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] - Security.SSLCopyPeerTrust.restype = OSStatus - - Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] - Security.SecTrustSetAnchorCertificates.restype = OSStatus - - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] - Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - - Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] - Security.SecTrustEvaluate.restype = OSStatus - - Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] - Security.SecTrustGetCertificateCount.restype = CFIndex - - Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] - Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef - - Security.SSLCreateContext.argtypes = [ - CFAllocatorRef, - SSLProtocolSide, - SSLConnectionType, - ] - Security.SSLCreateContext.restype = SSLContextRef - - Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] - Security.SSLSetSessionOption.restype = OSStatus - - Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMin.restype = OSStatus - - Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMax.restype = OSStatus - - try: - Security.SSLSetALPNProtocols.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetALPNProtocols.restype = OSStatus - except AttributeError: - # Supported only in 10.12+ - pass - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SSLReadFunc = SSLReadFunc - Security.SSLWriteFunc = SSLWriteFunc - Security.SSLContextRef = SSLContextRef - Security.SSLProtocol = SSLProtocol - Security.SSLCipherSuite = SSLCipherSuite - Security.SecIdentityRef = SecIdentityRef - Security.SecKeychainRef = SecKeychainRef - Security.SecTrustRef = SecTrustRef - Security.SecTrustResultType = SecTrustResultType - Security.SecExternalFormat = SecExternalFormat - Security.OSStatus = OSStatus - - Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, "kSecImportExportPassphrase" - ) - Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, "kSecImportItemIdentity" - ) - - # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [CFTypeRef] - CoreFoundation.CFRetain.restype = CFTypeRef - - CoreFoundation.CFRelease.argtypes = [CFTypeRef] - CoreFoundation.CFRelease.restype = None - - CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] - CoreFoundation.CFGetTypeID.restype = CFTypeID - - CoreFoundation.CFStringCreateWithCString.argtypes = [ - CFAllocatorRef, - c_char_p, - CFStringEncoding, - ] - CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - - CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] - CoreFoundation.CFStringGetCStringPtr.restype = c_char_p - - CoreFoundation.CFStringGetCString.argtypes = [ - CFStringRef, - c_char_p, - CFIndex, - CFStringEncoding, - ] - CoreFoundation.CFStringGetCString.restype = c_bool - - CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] - CoreFoundation.CFDataCreate.restype = CFDataRef - - CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] - CoreFoundation.CFDataGetLength.restype = CFIndex - - CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] - CoreFoundation.CFDataGetBytePtr.restype = c_void_p - - CoreFoundation.CFDictionaryCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - POINTER(CFTypeRef), - CFIndex, - CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks, - ] - CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - - CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] - CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef - - CoreFoundation.CFArrayCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreate.restype = CFArrayRef - - CoreFoundation.CFArrayCreateMutable.argtypes = [ - CFAllocatorRef, - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - - CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] - CoreFoundation.CFArrayAppendValue.restype = None - - CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] - CoreFoundation.CFArrayGetCount.restype = CFIndex - - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] - CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p - - CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, "kCFAllocatorDefault" - ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeArrayCallBacks" - ) - CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryKeyCallBacks" - ) - CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryValueCallBacks" - ) - - CoreFoundation.CFTypeRef = CFTypeRef - CoreFoundation.CFArrayRef = CFArrayRef - CoreFoundation.CFStringRef = CFStringRef - CoreFoundation.CFDictionaryRef = CFDictionaryRef - -except (AttributeError): - raise ImportError("Error initializing ctypes") - - -class CFConst(object): - """ - A class object that acts as essentially a namespace for CoreFoundation - constants. - """ - - kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) - - -class SecurityConst(object): - """ - A class object that acts as essentially a namespace for Security constants. - """ - - kSSLSessionOptionBreakOnServerAuth = 0 - - kSSLProtocol2 = 1 - kSSLProtocol3 = 2 - kTLSProtocol1 = 4 - kTLSProtocol11 = 7 - kTLSProtocol12 = 8 - # SecureTransport does not support TLS 1.3 even if there's a constant for it - kTLSProtocol13 = 10 - kTLSProtocolMaxSupported = 999 - - kSSLClientSide = 1 - kSSLStreamType = 0 - - kSecFormatPEMSequence = 10 - - kSecTrustResultInvalid = 0 - kSecTrustResultProceed = 1 - # This gap is present on purpose: this was kSecTrustResultConfirm, which - # is deprecated. - kSecTrustResultDeny = 3 - kSecTrustResultUnspecified = 4 - kSecTrustResultRecoverableTrustFailure = 5 - kSecTrustResultFatalTrustFailure = 6 - kSecTrustResultOtherError = 7 - - errSSLProtocol = -9800 - errSSLWouldBlock = -9803 - errSSLClosedGraceful = -9805 - errSSLClosedNoNotify = -9816 - errSSLClosedAbort = -9806 - - errSSLXCertChainInvalid = -9807 - errSSLCrypto = -9809 - errSSLInternal = -9810 - errSSLCertExpired = -9814 - errSSLCertNotYetValid = -9815 - errSSLUnknownRootCert = -9812 - errSSLNoRootCert = -9813 - errSSLHostNameMismatch = -9843 - errSSLPeerHandshakeFail = -9824 - errSSLPeerUserCancelled = -9839 - errSSLWeakPeerEphemeralDHKey = -9850 - errSSLServerAuthCompleted = -9841 - errSSLRecordOverflow = -9847 - - errSecVerifyFailed = -67808 - errSecNoTrustSettings = -25263 - errSecItemNotFound = -25300 - errSecInvalidTrustSettings = -25262 - - # Cipher suites. We only pick the ones our default cipher string allows. - # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8 - TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D - TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C - TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 - TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F - TLS_AES_128_GCM_SHA256 = 0x1301 - TLS_AES_256_GCM_SHA384 = 0x1302 - TLS_AES_128_CCM_8_SHA256 = 0x1305 - TLS_AES_128_CCM_SHA256 = 0x1304 diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/low_level.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/low_level.py deleted file mode 100644 index fa0b245d279..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/_securetransport/low_level.py +++ /dev/null @@ -1,397 +0,0 @@ -""" -Low-level helpers for the SecureTransport bindings. - -These are Python functions that are not directly related to the high-level APIs -but are necessary to get them to work. They include a whole bunch of low-level -CoreFoundation messing about and memory management. The concerns in this module -are almost entirely about trying to avoid memory leaks and providing -appropriate and useful assistance to the higher-level code. -""" -import base64 -import ctypes -import itertools -import os -import re -import ssl -import struct -import tempfile - -from .bindings import CFConst, CoreFoundation, Security - -# This regular expression is used to grab PEM data out of a PEM bundle. -_PEM_CERTS_RE = re.compile( - b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL -) - - -def _cf_data_from_bytes(bytestring): - """ - Given a bytestring, create a CFData object from it. This CFData object must - be CFReleased by the caller. - """ - return CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring) - ) - - -def _cf_dictionary_from_tuples(tuples): - """ - Given a list of Python tuples, create an associated CFDictionary. - """ - dictionary_size = len(tuples) - - # We need to get the dictionary keys and values out in the same order. - keys = (t[0] for t in tuples) - values = (t[1] for t in tuples) - cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys) - cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values) - - return CoreFoundation.CFDictionaryCreate( - CoreFoundation.kCFAllocatorDefault, - cf_keys, - cf_values, - dictionary_size, - CoreFoundation.kCFTypeDictionaryKeyCallBacks, - CoreFoundation.kCFTypeDictionaryValueCallBacks, - ) - - -def _cfstr(py_bstr): - """ - Given a Python binary data, create a CFString. - The string must be CFReleased by the caller. - """ - c_str = ctypes.c_char_p(py_bstr) - cf_str = CoreFoundation.CFStringCreateWithCString( - CoreFoundation.kCFAllocatorDefault, - c_str, - CFConst.kCFStringEncodingUTF8, - ) - return cf_str - - -def _create_cfstring_array(lst): - """ - Given a list of Python binary data, create an associated CFMutableArray. - The array must be CFReleased by the caller. - - Raises an ssl.SSLError on failure. - """ - cf_arr = None - try: - cf_arr = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cf_arr: - raise MemoryError("Unable to allocate memory!") - for item in lst: - cf_str = _cfstr(item) - if not cf_str: - raise MemoryError("Unable to allocate memory!") - try: - CoreFoundation.CFArrayAppendValue(cf_arr, cf_str) - finally: - CoreFoundation.CFRelease(cf_str) - except BaseException as e: - if cf_arr: - CoreFoundation.CFRelease(cf_arr) - raise ssl.SSLError("Unable to allocate array: %s" % (e,)) - return cf_arr - - -def _cf_string_to_unicode(value): - """ - Creates a Unicode string from a CFString object. Used entirely for error - reporting. - - Yes, it annoys me quite a lot that this function is this complex. - """ - value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) - - string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, CFConst.kCFStringEncodingUTF8 - ) - if string is None: - buffer = ctypes.create_string_buffer(1024) - result = CoreFoundation.CFStringGetCString( - value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 - ) - if not result: - raise OSError("Error copying C string from CFStringRef") - string = buffer.value - if string is not None: - string = string.decode("utf-8") - return string - - -def _assert_no_error(error, exception_class=None): - """ - Checks the return code and throws an exception if there is an error to - report - """ - if error == 0: - return - - cf_error_string = Security.SecCopyErrorMessageString(error, None) - output = _cf_string_to_unicode(cf_error_string) - CoreFoundation.CFRelease(cf_error_string) - - if output is None or output == u"": - output = u"OSStatus %s" % error - - if exception_class is None: - exception_class = ssl.SSLError - - raise exception_class(output) - - -def _cert_array_from_pem(pem_bundle): - """ - Given a bundle of certs in PEM format, turns them into a CFArray of certs - that can be used to validate a cert chain. - """ - # Normalize the PEM bundle's line endings. - pem_bundle = pem_bundle.replace(b"\r\n", b"\n") - - der_certs = [ - base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) - ] - if not der_certs: - raise ssl.SSLError("No root certificates specified") - - cert_array = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cert_array: - raise ssl.SSLError("Unable to allocate memory!") - - try: - for der_bytes in der_certs: - certdata = _cf_data_from_bytes(der_bytes) - if not certdata: - raise ssl.SSLError("Unable to allocate memory!") - cert = Security.SecCertificateCreateWithData( - CoreFoundation.kCFAllocatorDefault, certdata - ) - CoreFoundation.CFRelease(certdata) - if not cert: - raise ssl.SSLError("Unable to build cert object!") - - CoreFoundation.CFArrayAppendValue(cert_array, cert) - CoreFoundation.CFRelease(cert) - except Exception: - # We need to free the array before the exception bubbles further. - # We only want to do that if an error occurs: otherwise, the caller - # should free. - CoreFoundation.CFRelease(cert_array) - raise - - return cert_array - - -def _is_cert(item): - """ - Returns True if a given CFTypeRef is a certificate. - """ - expected = Security.SecCertificateGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected - - -def _is_identity(item): - """ - Returns True if a given CFTypeRef is an identity. - """ - expected = Security.SecIdentityGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected - - -def _temporary_keychain(): - """ - This function creates a temporary Mac keychain that we can use to work with - credentials. This keychain uses a one-time password and a temporary file to - store the data. We expect to have one keychain per socket. The returned - SecKeychainRef must be freed by the caller, including calling - SecKeychainDelete. - - Returns a tuple of the SecKeychainRef and the path to the temporary - directory that contains it. - """ - # Unfortunately, SecKeychainCreate requires a path to a keychain. This - # means we cannot use mkstemp to use a generic temporary file. Instead, - # we're going to create a temporary directory and a filename to use there. - # This filename will be 8 random bytes expanded into base64. We also need - # some random bytes to password-protect the keychain we're creating, so we - # ask for 40 random bytes. - random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode("utf-8") - password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 - tempdirectory = tempfile.mkdtemp() - - keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") - - # We now want to create the keychain itself. - keychain = Security.SecKeychainRef() - status = Security.SecKeychainCreate( - keychain_path, len(password), password, False, None, ctypes.byref(keychain) - ) - _assert_no_error(status) - - # Having created the keychain, we want to pass it off to the caller. - return keychain, tempdirectory - - -def _load_items_from_file(keychain, path): - """ - Given a single file, loads all the trust objects from it into arrays and - the keychain. - Returns a tuple of lists: the first list is a list of identities, the - second a list of certs. - """ - certificates = [] - identities = [] - result_array = None - - with open(path, "rb") as f: - raw_filedata = f.read() - - try: - filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) - ) - result_array = CoreFoundation.CFArrayRef() - result = Security.SecItemImport( - filedata, # cert data - None, # Filename, leaving it out for now - None, # What the type of the file is, we don't care - None, # what's in the file, we don't care - 0, # import flags - None, # key params, can include passphrase in the future - keychain, # The keychain to insert into - ctypes.byref(result_array), # Results - ) - _assert_no_error(result) - - # A CFArray is not very useful to us as an intermediary - # representation, so we are going to extract the objects we want - # and then free the array. We don't need to keep hold of keys: the - # keychain already has them! - result_count = CoreFoundation.CFArrayGetCount(result_array) - for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) - item = ctypes.cast(item, CoreFoundation.CFTypeRef) - - if _is_cert(item): - CoreFoundation.CFRetain(item) - certificates.append(item) - elif _is_identity(item): - CoreFoundation.CFRetain(item) - identities.append(item) - finally: - if result_array: - CoreFoundation.CFRelease(result_array) - - CoreFoundation.CFRelease(filedata) - - return (identities, certificates) - - -def _load_client_cert_chain(keychain, *paths): - """ - Load certificates and maybe keys from a number of files. Has the end goal - of returning a CFArray containing one SecIdentityRef, and then zero or more - SecCertificateRef objects, suitable for use as a client certificate trust - chain. - """ - # Ok, the strategy. - # - # This relies on knowing that macOS will not give you a SecIdentityRef - # unless you have imported a key into a keychain. This is a somewhat - # artificial limitation of macOS (for example, it doesn't necessarily - # affect iOS), but there is nothing inside Security.framework that lets you - # get a SecIdentityRef without having a key in a keychain. - # - # So the policy here is we take all the files and iterate them in order. - # Each one will use SecItemImport to have one or more objects loaded from - # it. We will also point at a keychain that macOS can use to work with the - # private key. - # - # Once we have all the objects, we'll check what we actually have. If we - # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise, - # we'll take the first certificate (which we assume to be our leaf) and - # ask the keychain to give us a SecIdentityRef with that cert's associated - # key. - # - # We'll then return a CFArray containing the trust chain: one - # SecIdentityRef and then zero-or-more SecCertificateRef objects. The - # responsibility for freeing this CFArray will be with the caller. This - # CFArray must remain alive for the entire connection, so in practice it - # will be stored with a single SSLSocket, along with the reference to the - # keychain. - certificates = [] - identities = [] - - # Filter out bad paths. - paths = (path for path in paths if path) - - try: - for file_path in paths: - new_identities, new_certs = _load_items_from_file(keychain, file_path) - identities.extend(new_identities) - certificates.extend(new_certs) - - # Ok, we have everything. The question is: do we have an identity? If - # not, we want to grab one from the first cert we have. - if not identities: - new_identity = Security.SecIdentityRef() - status = Security.SecIdentityCreateWithCertificate( - keychain, certificates[0], ctypes.byref(new_identity) - ) - _assert_no_error(status) - identities.append(new_identity) - - # We now want to release the original certificate, as we no longer - # need it. - CoreFoundation.CFRelease(certificates.pop(0)) - - # We now need to build a new CFArray that holds the trust chain. - trust_chain = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - for item in itertools.chain(identities, certificates): - # ArrayAppendValue does a CFRetain on the item. That's fine, - # because the finally block will release our other refs to them. - CoreFoundation.CFArrayAppendValue(trust_chain, item) - - return trust_chain - finally: - for obj in itertools.chain(identities, certificates): - CoreFoundation.CFRelease(obj) - - -TLS_PROTOCOL_VERSIONS = { - "SSLv2": (0, 2), - "SSLv3": (3, 0), - "TLSv1": (3, 1), - "TLSv1.1": (3, 2), - "TLSv1.2": (3, 3), -} - - -def _build_tls_unknown_ca_alert(version): - """ - Builds a TLS alert record for an unknown CA. - """ - ver_maj, ver_min = TLS_PROTOCOL_VERSIONS[version] - severity_fatal = 0x02 - description_unknown_ca = 0x30 - msg = struct.pack(">BB", severity_fatal, description_unknown_ca) - msg_len = len(msg) - record_type_alert = 0x15 - record = struct.pack(">BBBH", record_type_alert, ver_maj, ver_min, msg_len) + msg - return record diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/appengine.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/appengine.py deleted file mode 100644 index 1717ee22cdf..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/appengine.py +++ /dev/null @@ -1,314 +0,0 @@ -""" -This module provides a pool manager that uses Google App Engine's -`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. - -Example usage:: - - from pip._vendor.urllib3 import PoolManager - from pip._vendor.urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox - - if is_appengine_sandbox(): - # AppEngineManager uses AppEngine's URLFetch API behind the scenes - http = AppEngineManager() - else: - # PoolManager uses a socket-level API behind the scenes - http = PoolManager() - - r = http.request('GET', 'https://google.com/') - -There are `limitations <https://cloud.google.com/appengine/docs/python/\ -urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be -the best choice for your application. There are three options for using -urllib3 on Google App Engine: - -1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is - cost-effective in many circumstances as long as your usage is within the - limitations. -2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. - Sockets also have `limitations and restrictions - <https://cloud.google.com/appengine/docs/python/sockets/\ - #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. - To use sockets, be sure to specify the following in your ``app.yaml``:: - - env_variables: - GAE_USE_SOCKETS_HTTPLIB : 'true' - -3. If you are using `App Engine Flexible -<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard -:class:`PoolManager` without any configuration or special environment variables. -""" - -from __future__ import absolute_import - -import io -import logging -import warnings - -from ..exceptions import ( - HTTPError, - HTTPWarning, - MaxRetryError, - ProtocolError, - SSLError, - TimeoutError, -) -from ..packages.six.moves.urllib.parse import urljoin -from ..request import RequestMethods -from ..response import HTTPResponse -from ..util.retry import Retry -from ..util.timeout import Timeout -from . import _appengine_environ - -try: - from google.appengine.api import urlfetch -except ImportError: - urlfetch = None - - -log = logging.getLogger(__name__) - - -class AppEnginePlatformWarning(HTTPWarning): - pass - - -class AppEnginePlatformError(HTTPError): - pass - - -class AppEngineManager(RequestMethods): - """ - Connection manager for Google App Engine sandbox applications. - - This manager uses the URLFetch service directly instead of using the - emulated httplib, and is subject to URLFetch limitations as described in - the App Engine documentation `here - <https://cloud.google.com/appengine/docs/python/urlfetch>`_. - - Notably it will raise an :class:`AppEnginePlatformError` if: - * URLFetch is not available. - * If you attempt to use this on App Engine Flexible, as full socket - support is available. - * If a request size is more than 10 megabytes. - * If a response size is more than 32 megabytes. - * If you use an unsupported request method such as OPTIONS. - - Beyond those cases, it will raise normal urllib3 errors. - """ - - def __init__( - self, - headers=None, - retries=None, - validate_certificate=True, - urlfetch_retries=True, - ): - if not urlfetch: - raise AppEnginePlatformError( - "URLFetch is not available in this environment." - ) - - warnings.warn( - "urllib3 is using URLFetch on Google App Engine sandbox instead " - "of sockets. To use sockets directly instead of URLFetch see " - "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", - AppEnginePlatformWarning, - ) - - RequestMethods.__init__(self, headers) - self.validate_certificate = validate_certificate - self.urlfetch_retries = urlfetch_retries - - self.retries = retries or Retry.DEFAULT - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # Return False to re-raise any potential exceptions - return False - - def urlopen( - self, - method, - url, - body=None, - headers=None, - retries=None, - redirect=True, - timeout=Timeout.DEFAULT_TIMEOUT, - **response_kw - ): - - retries = self._get_retries(retries, redirect) - - try: - follow_redirects = redirect and retries.redirect != 0 and retries.total - response = urlfetch.fetch( - url, - payload=body, - method=method, - headers=headers or {}, - allow_truncated=False, - follow_redirects=self.urlfetch_retries and follow_redirects, - deadline=self._get_absolute_timeout(timeout), - validate_certificate=self.validate_certificate, - ) - except urlfetch.DeadlineExceededError as e: - raise TimeoutError(self, e) - - except urlfetch.InvalidURLError as e: - if "too large" in str(e): - raise AppEnginePlatformError( - "URLFetch request too large, URLFetch only " - "supports requests up to 10mb in size.", - e, - ) - raise ProtocolError(e) - - except urlfetch.DownloadError as e: - if "Too many redirects" in str(e): - raise MaxRetryError(self, url, reason=e) - raise ProtocolError(e) - - except urlfetch.ResponseTooLargeError as e: - raise AppEnginePlatformError( - "URLFetch response too large, URLFetch only supports" - "responses up to 32mb in size.", - e, - ) - - except urlfetch.SSLCertificateError as e: - raise SSLError(e) - - except urlfetch.InvalidMethodError as e: - raise AppEnginePlatformError( - "URLFetch does not support method: %s" % method, e - ) - - http_response = self._urlfetch_response_to_http_response( - response, retries=retries, **response_kw - ) - - # Handle redirect? - redirect_location = redirect and http_response.get_redirect_location() - if redirect_location: - # Check for redirect response - if self.urlfetch_retries and retries.raise_on_redirect: - raise MaxRetryError(self, url, "too many redirects") - else: - if http_response.status == 303: - method = "GET" - - try: - retries = retries.increment( - method, url, response=http_response, _pool=self - ) - except MaxRetryError: - if retries.raise_on_redirect: - raise MaxRetryError(self, url, "too many redirects") - return http_response - - retries.sleep_for_retry(http_response) - log.debug("Redirecting %s -> %s", url, redirect_location) - redirect_url = urljoin(url, redirect_location) - return self.urlopen( - method, - redirect_url, - body, - headers, - retries=retries, - redirect=redirect, - timeout=timeout, - **response_kw - ) - - # Check if we should retry the HTTP response. - has_retry_after = bool(http_response.headers.get("Retry-After")) - if retries.is_retry(method, http_response.status, has_retry_after): - retries = retries.increment(method, url, response=http_response, _pool=self) - log.debug("Retry: %s", url) - retries.sleep(http_response) - return self.urlopen( - method, - url, - body=body, - headers=headers, - retries=retries, - redirect=redirect, - timeout=timeout, - **response_kw - ) - - return http_response - - def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): - - if is_prod_appengine(): - # Production GAE handles deflate encoding automatically, but does - # not remove the encoding header. - content_encoding = urlfetch_resp.headers.get("content-encoding") - - if content_encoding == "deflate": - del urlfetch_resp.headers["content-encoding"] - - transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") - # We have a full response's content, - # so let's make sure we don't report ourselves as chunked data. - if transfer_encoding == "chunked": - encodings = transfer_encoding.split(",") - encodings.remove("chunked") - urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) - - original_response = HTTPResponse( - # In order for decoding to work, we must present the content as - # a file-like object. - body=io.BytesIO(urlfetch_resp.content), - msg=urlfetch_resp.header_msg, - headers=urlfetch_resp.headers, - status=urlfetch_resp.status_code, - **response_kw - ) - - return HTTPResponse( - body=io.BytesIO(urlfetch_resp.content), - headers=urlfetch_resp.headers, - status=urlfetch_resp.status_code, - original_response=original_response, - **response_kw - ) - - def _get_absolute_timeout(self, timeout): - if timeout is Timeout.DEFAULT_TIMEOUT: - return None # Defer to URLFetch's default. - if isinstance(timeout, Timeout): - if timeout._read is not None or timeout._connect is not None: - warnings.warn( - "URLFetch does not support granular timeout settings, " - "reverting to total or default URLFetch timeout.", - AppEnginePlatformWarning, - ) - return timeout.total - return timeout - - def _get_retries(self, retries, redirect): - if not isinstance(retries, Retry): - retries = Retry.from_int(retries, redirect=redirect, default=self.retries) - - if retries.connect or retries.read or retries.redirect: - warnings.warn( - "URLFetch only supports total retries and does not " - "recognize connect, read, or redirect retry parameters.", - AppEnginePlatformWarning, - ) - - return retries - - -# Alias methods from _appengine_environ to maintain public API interface. - -is_appengine = _appengine_environ.is_appengine -is_appengine_sandbox = _appengine_environ.is_appengine_sandbox -is_local_appengine = _appengine_environ.is_local_appengine -is_prod_appengine = _appengine_environ.is_prod_appengine -is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/__init__.py new file mode 100644 index 00000000000..c0538059ec8 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/__init__.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import pip._vendor.urllib3.connection as urllib3_connection + +from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection + + +def inject_into_urllib3() -> None: + # override connection classes to use emscripten specific classes + # n.b. mypy complains about the overriding of classes below + # if it isn't ignored + HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection + HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection + urllib3_connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment] + urllib3_connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment] + urllib3_connection.VerifiedHTTPSConnection = EmscriptenHTTPSConnection # type: ignore[assignment] diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/connection.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/connection.py new file mode 100644 index 00000000000..63f79dd3be8 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/connection.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import os +import typing + +# use http.client.HTTPException for consistency with non-emscripten +from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import ResponseNotReady + +from ..._base_connection import _TYPE_BODY +from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...exceptions import TimeoutError +from ...response import BaseHTTPResponse +from ...util.connection import _TYPE_SOCKET_OPTIONS +from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from ...util.url import Url +from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request +from .request import EmscriptenRequest +from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + + +class EmscriptenHTTPConnection: + default_port: typing.ClassVar[int] = port_by_scheme["http"] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + timeout: None | (float) + + host: str + port: int + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool = False + proxy_is_verified: bool | None = None + + response_class: type[BaseHTTPResponse] = EmscriptenHttpResponseWrapper + _response: EmscriptenResponse | None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = None, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + self.host = host + self.port = port + self.timeout = timeout if isinstance(timeout, float) else 0.0 + self.scheme = "http" + self._closed = True + self._response = None + # ignore these things because we don't + # have control over that stuff + self.proxy = None + self.proxy_config = None + self.blocksize = blocksize + self.source_address = None + self.socket_options = None + self.is_verified = False + + def set_tunnel( + self, + host: str, + port: int | None = 0, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + pass + + def connect(self) -> None: + pass + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: + self._closed = False + if url.startswith("/"): + if self.port is not None: + port = f":{self.port}" + else: + port = "" + # no scheme / host / port included, make a full url + url = f"{self.scheme}://{self.host}{port}{url}" + request = EmscriptenRequest( + url=url, + method=method, + timeout=self.timeout if self.timeout else 0, + decode_content=decode_content, + ) + request.set_body(body) + if headers: + for k, v in headers.items(): + request.set_header(k, v) + self._response = None + try: + if not preload_content: + self._response = send_streaming_request(request) + if self._response is None: + self._response = send_request(request) + except _TimeoutError as e: + raise TimeoutError(e.message) from e + except _RequestError as e: + raise HTTPException(e.message) from e + + def getresponse(self) -> BaseHTTPResponse: + if self._response is not None: + return EmscriptenHttpResponseWrapper( + internal_response=self._response, + url=self._response.request.url, + connection=self, + ) + else: + raise ResponseNotReady() + + def close(self) -> None: + self._closed = True + self._response = None + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + return self._closed + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + return True + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + return False + + +class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): + default_port = port_by_scheme["https"] + # all this is basically ignored, as browser handles https + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + cert_file: str | None + key_file: str | None + key_password: str | None + ssl_context: typing.Any | None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None = None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: ( + None | _TYPE_SOCKET_OPTIONS + ) = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: typing.Any | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) + self.scheme = "https" + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data + + self.cert_reqs = None + + # The browser will automatically verify all requests. + # We have no control over that setting. + self.is_verified = True + + def set_cert( + self, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: + pass + + +# verify that this class implements BaseHTTP(s) connection correctly +if typing.TYPE_CHECKING: + _supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0) + _supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0) diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/emscripten_fetch_worker.js new file mode 100644 index 00000000000..faf141e1fa4 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/emscripten_fetch_worker.js @@ -0,0 +1,110 @@ +let Status = { + SUCCESS_HEADER: -1, + SUCCESS_EOF: -2, + ERROR_TIMEOUT: -3, + ERROR_EXCEPTION: -4, +}; + +let connections = new Map(); +let nextConnectionID = 1; +const encoder = new TextEncoder(); + +self.addEventListener("message", async function (event) { + if (event.data.close) { + let connectionID = event.data.close; + connections.delete(connectionID); + return; + } else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader, intBuffer, byteBuffer } = + connections.get(connectionID); + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try { + let readResponse = await reader.read(); + + if (readResponse.done) { + // read everything - clear connection and return + connections.delete(connectionID); + Atomics.store(intBuffer, 0, Status.SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; + } + curOffset = 0; + connections.get(connectionID).value = readResponse.value; + value = readResponse.value; + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } + + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0); + + Atomics.store(intBuffer, 0, curLen); // store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset += curLen; + connections.get(connectionID).curOffset = curOffset; + + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8); + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + let headerObj = { + headers: headers, + status: response.status, + connectionID, + }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections.set(connectionID, { + reader: response.body.getReader(), + intBuffer: intBuffer, + byteBuffer: byteBuffer, + value: undefined, + curOffset: 0, + }); + // set header ready + Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } +}); +self.postMessage({ inited: true }); diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/fetch.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/fetch.py new file mode 100644 index 00000000000..612cfddc4c2 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/fetch.py @@ -0,0 +1,726 @@ +""" +Support for streaming http requests in emscripten. + +A few caveats - + +If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled +https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md +*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the +JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case +timeouts and streaming should just work. + +Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming. + +This approach has several caveats: + +Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. +Streaming only works if you're running pyodide in a web worker. + +Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch +operation, so it requires that you have crossOriginIsolation enabled, by serving over https +(or from localhost) with the two headers below set: + + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp + +You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in +JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole +request into a buffer and then returning it. it shows a warning in the JavaScript console in this case. + +Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once +control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. + +NB: in this code, there are a lot of JavaScript objects. They are named js_* +to make it clear what type of object they are. +""" + +from __future__ import annotations + +import io +import json +from email.parser import Parser +from importlib.resources import files +from typing import TYPE_CHECKING, Any + +import js # type: ignore[import-not-found] +from pyodide.ffi import ( # type: ignore[import-not-found] + JsArray, + JsException, + JsProxy, + to_js, +) + +if TYPE_CHECKING: + from typing_extensions import Buffer + +from .request import EmscriptenRequest +from .response import EmscriptenResponse + +""" +There are some headers that trigger unintended CORS preflight requests. +See also https://github.com/koenvo/pyodide-http/issues/22 +""" +HEADERS_TO_IGNORE = ("user-agent",) + +SUCCESS_HEADER = -1 +SUCCESS_EOF = -2 +ERROR_TIMEOUT = -3 +ERROR_EXCEPTION = -4 + + +class _RequestError(Exception): + def __init__( + self, + message: str | None = None, + *, + request: EmscriptenRequest | None = None, + response: EmscriptenResponse | None = None, + ): + self.request = request + self.response = response + self.message = message + super().__init__(self.message) + + +class _StreamingError(_RequestError): + pass + + +class _TimeoutError(_RequestError): + pass + + +def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy: + return to_js(dict_val, dict_converter=js.Object.fromEntries) + + +class _ReadStream(io.RawIOBase): + def __init__( + self, + int_buffer: JsArray, + byte_buffer: JsArray, + timeout: float, + worker: JsProxy, + connection_id: int, + request: EmscriptenRequest, + ): + self.int_buffer = int_buffer + self.byte_buffer = byte_buffer + self.read_pos = 0 + self.read_len = 0 + self.connection_id = connection_id + self.worker = worker + self.timeout = int(1000 * timeout) if timeout > 0 else None + self.is_live = True + self._is_closed = False + self.request: EmscriptenRequest | None = request + + def __del__(self) -> None: + self.close() + + # this is compatible with _base_connection + def is_closed(self) -> bool: + return self._is_closed + + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + + def close(self) -> None: + if self.is_closed(): + return + self.read_len = 0 + self.read_pos = 0 + self.int_buffer = None + self.byte_buffer = None + self._is_closed = True + self.request = None + if self.is_live: + self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) + self.is_live = False + super().close() + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def readinto(self, byte_obj: Buffer) -> int: + if not self.int_buffer: + raise _StreamingError( + "No buffer for stream in _ReadStream.readinto", + request=self.request, + response=None, + ) + if self.read_len == 0: + # wait for the worker to send something + js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT) + self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) + if ( + js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout) + == "timed-out" + ): + raise _TimeoutError + data_len = self.int_buffer[0] + if data_len > 0: + self.read_len = data_len + self.read_pos = 0 + elif data_len == ERROR_EXCEPTION: + string_len = self.int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", + request=self.request, + response=None, + ) + else: + # EOF, free the buffers and return zero + # and free the request + self.is_live = False + self.close() + return 0 + # copy from int32array to python bytes + ret_length = min(self.read_len, len(memoryview(byte_obj))) + subarray = self.byte_buffer.subarray( + self.read_pos, self.read_pos + ret_length + ).to_py() + memoryview(byte_obj)[0:ret_length] = subarray + self.read_len -= ret_length + self.read_pos += ret_length + return ret_length + + +class _StreamingFetcher: + def __init__(self) -> None: + # make web-worker and data buffer on startup + self.streaming_ready = False + streaming_worker_code = ( + files(__package__) + .joinpath("emscripten_fetch_worker.js") + .read_text(encoding="utf-8") + ) + js_data_blob = js.Blob.new( + to_js([streaming_worker_code], create_pyproxies=False), + _obj_from_dict({"type": "application/javascript"}), + ) + + def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None: + def onMsg(e: JsProxy) -> None: + self.streaming_ready = True + js_resolve_fn(e) + + def onErr(e: JsProxy) -> None: + js_reject_fn(e) # Defensive: never happens in ci + + self.js_worker.onmessage = onMsg + self.js_worker.onerror = onErr + + js_data_url = js.URL.createObjectURL(js_data_blob) + self.js_worker = js.globalThis.Worker.new(js_data_url) + self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver) + + def send(self, request: EmscriptenRequest) -> EmscriptenResponse: + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + + body = request.body + fetch_data = {"headers": headers, "body": to_js(body), "method": request.method} + # start the request off in the worker + timeout = int(1000 * request.timeout) if request.timeout > 0 else None + js_shared_buffer = js.SharedArrayBuffer.new(1048576) + js_int_buffer = js.Int32Array.new(js_shared_buffer) + js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8) + + js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT) + js.Atomics.notify(js_int_buffer, 0) + js_absolute_url = js.URL.new(request.url, js.location).href + self.js_worker.postMessage( + _obj_from_dict( + { + "buffer": js_shared_buffer, + "url": js_absolute_url, + "fetchParams": fetch_data, + } + ) + ) + # wait for the worker to send something + js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout) + if js_int_buffer[0] == ERROR_TIMEOUT: + raise _TimeoutError( + "Timeout connecting to streaming request", + request=request, + response=None, + ) + elif js_int_buffer[0] == SUCCESS_HEADER: + # got response + # header length is in second int of intBuffer + string_len = js_int_buffer[1] + # decode the rest to a JSON string + js_decoder = js.TextDecoder.new() + # this does a copy (the slice) because decode can't work on shared array + # for some silly reason + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + # get it as an object + response_obj = json.loads(json_str) + return EmscriptenResponse( + request=request, + status_code=response_obj["status"], + headers=response_obj["headers"], + body=_ReadStream( + js_int_buffer, + js_byte_buffer, + request.timeout, + self.js_worker, + response_obj["connectionID"], + request, + ), + ) + elif js_int_buffer[0] == ERROR_EXCEPTION: + string_len = js_int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", request=request, response=None + ) + else: + raise _StreamingError( + f"Unknown status from worker in fetch: {js_int_buffer[0]}", + request=request, + response=None, + ) + + +class _JSPIReadStream(io.RawIOBase): + """ + A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch + response. This requires support for WebAssembly JavaScript Promise Integration + in the containing browser, and for pyodide to be launched via runPythonAsync. + + :param js_read_stream: + The JavaScript stream reader + + :param timeout: + Timeout in seconds + + :param request: + The request we're handling + + :param response: + The response this stream relates to + + :param js_abort_controller: + A JavaScript AbortController object, used for timeouts + """ + + def __init__( + self, + js_read_stream: Any, + timeout: float, + request: EmscriptenRequest, + response: EmscriptenResponse, + js_abort_controller: Any, # JavaScript AbortController for timeouts + ): + self.js_read_stream = js_read_stream + self.timeout = timeout + self._is_closed = False + self._is_done = False + self.request: EmscriptenRequest | None = request + self.response: EmscriptenResponse | None = response + self.current_buffer = None + self.current_buffer_pos = 0 + self.js_abort_controller = js_abort_controller + + def __del__(self) -> None: + self.close() + + # this is compatible with _base_connection + def is_closed(self) -> bool: + return self._is_closed + + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + + def close(self) -> None: + if self.is_closed(): + return + self.read_len = 0 + self.read_pos = 0 + self.js_read_stream.cancel() + self.js_read_stream = None + self._is_closed = True + self._is_done = True + self.request = None + self.response = None + super().close() + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def _get_next_buffer(self) -> bool: + result_js = _run_sync_with_timeout( + self.js_read_stream.read(), + self.timeout, + self.js_abort_controller, + request=self.request, + response=self.response, + ) + if result_js.done: + self._is_done = True + return False + else: + self.current_buffer = result_js.value.to_py() + self.current_buffer_pos = 0 + return True + + def readinto(self, byte_obj: Buffer) -> int: + if self.current_buffer is None: + if not self._get_next_buffer() or self.current_buffer is None: + self.close() + return 0 + ret_length = min( + len(byte_obj), len(self.current_buffer) - self.current_buffer_pos + ) + byte_obj[0:ret_length] = self.current_buffer[ + self.current_buffer_pos : self.current_buffer_pos + ret_length + ] + self.current_buffer_pos += ret_length + if self.current_buffer_pos == len(self.current_buffer): + self.current_buffer = None + return ret_length + + +# check if we are in a worker or not +def is_in_browser_main_thread() -> bool: + return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + + +def is_cross_origin_isolated() -> bool: + return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated + + +def is_in_node() -> bool: + return ( + hasattr(js, "process") + and hasattr(js.process, "release") + and hasattr(js.process.release, "name") + and js.process.release.name == "node" + ) + + +def is_worker_available() -> bool: + return hasattr(js, "Worker") and hasattr(js, "Blob") + + +_fetcher: _StreamingFetcher | None = None + +if is_worker_available() and ( + (is_cross_origin_isolated() and not is_in_browser_main_thread()) + and (not is_in_node()) +): + _fetcher = _StreamingFetcher() +else: + _fetcher = None + + +NODE_JSPI_ERROR = ( + "urllib3 only works in Node.js with pyodide.runPythonAsync" + " and requires the flag --experimental-wasm-stack-switching in " + " versions of node <24." +) + + +def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None: + if has_jspi(): + return send_jspi_request(request, True) + elif is_in_node(): + raise _RequestError( + message=NODE_JSPI_ERROR, + request=request, + response=None, + ) + + if _fetcher and streaming_ready(): + return _fetcher.send(request) + else: + _show_streaming_warning() + return None + + +_SHOWN_TIMEOUT_WARNING = False + + +def _show_timeout_warning() -> None: + global _SHOWN_TIMEOUT_WARNING + if not _SHOWN_TIMEOUT_WARNING: + _SHOWN_TIMEOUT_WARNING = True + message = "Warning: Timeout is not available on main browser thread" + js.console.warn(message) + + +_SHOWN_STREAMING_WARNING = False + + +def _show_streaming_warning() -> None: + global _SHOWN_STREAMING_WARNING + if not _SHOWN_STREAMING_WARNING: + _SHOWN_STREAMING_WARNING = True + message = "Can't stream HTTP requests because: \n" + if not is_cross_origin_isolated(): + message += " Page is not cross-origin isolated\n" + if is_in_browser_main_thread(): + message += " Python is running in main browser thread\n" + if not is_worker_available(): + message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in + if streaming_ready() is False: + message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch +is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" + from js import console + + console.warn(message) + + +def send_request(request: EmscriptenRequest) -> EmscriptenResponse: + if has_jspi(): + return send_jspi_request(request, False) + elif is_in_node(): + raise _RequestError( + message=NODE_JSPI_ERROR, + request=request, + response=None, + ) + try: + js_xhr = js.XMLHttpRequest.new() + + if not is_in_browser_main_thread(): + js_xhr.responseType = "arraybuffer" + if request.timeout: + js_xhr.timeout = int(request.timeout * 1000) + else: + js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + if request.timeout: + # timeout isn't available on the main thread - show a warning in console + # if it is set + _show_timeout_warning() + + js_xhr.open(request.method, request.url, False) + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + js_xhr.setRequestHeader(name, value) + + js_xhr.send(to_js(request.body)) + + headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders())) + + if not is_in_browser_main_thread(): + body = js_xhr.response.to_py().tobytes() + else: + body = js_xhr.response.encode("ISO-8859-15") + return EmscriptenResponse( + status_code=js_xhr.status, headers=headers, body=body, request=request + ) + except JsException as err: + if err.name == "TimeoutError": + raise _TimeoutError(err.message, request=request) + elif err.name == "NetworkError": + raise _RequestError(err.message, request=request) + else: + # general http error + raise _RequestError(err.message, request=request) + + +def send_jspi_request( + request: EmscriptenRequest, streaming: bool +) -> EmscriptenResponse: + """ + Send a request using WebAssembly JavaScript Promise Integration + to wrap the asynchronous JavaScript fetch api (experimental). + + :param request: + Request to send + + :param streaming: + Whether to stream the response + + :return: The response object + :rtype: EmscriptenResponse + """ + timeout = request.timeout + js_abort_controller = js.AbortController.new() + headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE} + req_body = request.body + fetch_data = { + "headers": headers, + "body": to_js(req_body), + "method": request.method, + "signal": js_abort_controller.signal, + } + # Node.js returns the whole response (unlike opaqueredirect in browsers), + # so urllib3 can set `redirect: manual` to control redirects itself. + # https://stackoverflow.com/a/78524615 + if _is_node_js(): + fetch_data["redirect"] = "manual" + # Call JavaScript fetch (async api, returns a promise) + fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data)) + # Now suspend WebAssembly until we resolve that promise + # or time out. + response_js = _run_sync_with_timeout( + fetcher_promise_js, + timeout, + js_abort_controller, + request=request, + response=None, + ) + headers = {} + header_iter = response_js.headers.entries() + while True: + iter_value_js = header_iter.next() + if getattr(iter_value_js, "done", False): + break + else: + headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1]) + status_code = response_js.status + body: bytes | io.RawIOBase = b"" + + response = EmscriptenResponse( + status_code=status_code, headers=headers, body=b"", request=request + ) + if streaming: + # get via inputstream + if response_js.body is not None: + # get a reader from the fetch response + body_stream_js = response_js.body.getReader() + body = _JSPIReadStream( + body_stream_js, timeout, request, response, js_abort_controller + ) + else: + # get directly via arraybuffer + # n.b. this is another async JavaScript call. + body = _run_sync_with_timeout( + response_js.arrayBuffer(), + timeout, + js_abort_controller, + request=request, + response=response, + ).to_py() + response.body = body + return response + + +def _run_sync_with_timeout( + promise: Any, + timeout: float, + js_abort_controller: Any, + request: EmscriptenRequest | None, + response: EmscriptenResponse | None, +) -> Any: + """ + Await a JavaScript promise synchronously with a timeout which is implemented + via the AbortController + + :param promise: + Javascript promise to await + + :param timeout: + Timeout in seconds + + :param js_abort_controller: + A JavaScript AbortController object, used on timeout + + :param request: + The request being handled + + :param response: + The response being handled (if it exists yet) + + :raises _TimeoutError: If the request times out + :raises _RequestError: If the request raises a JavaScript exception + + :return: The result of awaiting the promise. + """ + timer_id = None + if timeout > 0: + timer_id = js.setTimeout( + js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000) + ) + try: + from pyodide.ffi import run_sync + + # run_sync here uses WebAssembly JavaScript Promise Integration to + # suspend python until the JavaScript promise resolves. + return run_sync(promise) + except JsException as err: + if err.name == "AbortError": + raise _TimeoutError( + message="Request timed out", request=request, response=response + ) + else: + raise _RequestError(message=err.message, request=request, response=response) + finally: + if timer_id is not None: + js.clearTimeout(timer_id) + + +def has_jspi() -> bool: + """ + Return true if jspi can be used. + + This requires both browser support and also WebAssembly + to be in the correct state - i.e. that the javascript + call into python was async not sync. + + :return: True if jspi can be used. + :rtype: bool + """ + try: + from pyodide.ffi import can_run_sync, run_sync # noqa: F401 + + return bool(can_run_sync()) + except ImportError: + return False + + +def _is_node_js() -> bool: + """ + Check if we are in Node.js. + + :return: True if we are in Node.js. + :rtype: bool + """ + return ( + hasattr(js, "process") + and hasattr(js.process, "release") + # According to the Node.js documentation, the release name is always "node". + and js.process.release.name == "node" + ) + + +def streaming_ready() -> bool | None: + if _fetcher: + return _fetcher.streaming_ready + else: + return None # no fetcher, return None to signify that + + +async def wait_for_streaming_ready() -> bool: + if _fetcher: + await _fetcher.js_worker_ready_promise + return True + else: + return False diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/request.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/request.py new file mode 100644 index 00000000000..e692e692bd0 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/request.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..._base_connection import _TYPE_BODY + + +@dataclass +class EmscriptenRequest: + method: str + url: str + params: dict[str, str] | None = None + body: _TYPE_BODY | None = None + headers: dict[str, str] = field(default_factory=dict) + timeout: float = 0 + decode_content: bool = True + + def set_header(self, name: str, value: str) -> None: + self.headers[name.capitalize()] = value + + def set_body(self, body: _TYPE_BODY | None) -> None: + self.body = body diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/response.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/response.py new file mode 100644 index 00000000000..cb1088a1826 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/emscripten/response.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import json as _json +import logging +import typing +from contextlib import contextmanager +from dataclasses import dataclass +from http.client import HTTPException as HTTPException +from io import BytesIO, IOBase + +from ...exceptions import InvalidHeader, TimeoutError +from ...response import BaseHTTPResponse +from ...util.retry import Retry +from .request import EmscriptenRequest + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + +log = logging.getLogger(__name__) + + +@dataclass +class EmscriptenResponse: + status_code: int + headers: dict[str, str] + body: IOBase | bytes + request: EmscriptenRequest + + +class EmscriptenHttpResponseWrapper(BaseHTTPResponse): + def __init__( + self, + internal_response: EmscriptenResponse, + url: str | None = None, + connection: BaseHTTPConnection | BaseHTTPSConnection | None = None, + ): + self._pool = None # set by pool class + self._body = None + self._response = internal_response + self._url = url + self._connection = connection + self._closed = False + super().__init__( + headers=internal_response.headers, + status=internal_response.status_code, + request_url=url, + version=0, + version_string="HTTP/?", + reason="", + decode_content=True, + ) + self.length_remaining = self._init_length(self._response.request.method) + self.length_is_certain = False + + @property + def url(self) -> str | None: + return self._url + + @url.setter + def url(self, url: str | None) -> None: + self._url = url + + @property + def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None: + return self._connection + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + self._retries = retries + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes]: + """ + A generator wrapper for the read() method. A call will block until + ``amt`` bytes have been read from the connection or until the + connection is closed. + + :param amt: + How much of the content to read. The generator will return up to + much data per iteration, but may return less. This is particularly + likely when using compressed data. However, the empty string will + never be returned. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + while True: + data = self.read(amt=amt, decode_content=decode_content) + + if data: + yield data + else: + break + + def _init_length(self, request_method: str | None) -> int | None: + length: int | None + content_length: str | None = self.headers.get("content-length") + + if content_length is not None: + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = {int(val) for val in content_length.split(",")} + if len(lengths) > 1: + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % content_length + ) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + else: # if content_length is None + length = None + + # Check for responses that shouldn't include a body + if ( + self.status in (204, 304) + or 100 <= self.status < 200 + or request_method == "HEAD" + ): + length = 0 + + return length + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, # ignored because browser decodes always + cache_content: bool = False, + ) -> bytes: + if ( + self._closed + or self._response is None + or (isinstance(self._response.body, IOBase) and self._response.body.closed) + ): + return b"" + + with self._error_catcher(): + # body has been preloaded as a string by XmlHttpRequest + if not isinstance(self._response.body, IOBase): + self.length_remaining = len(self._response.body) + self.length_is_certain = True + # wrap body in IOStream + self._response.body = BytesIO(self._response.body) + if amt is not None and amt >= 0: + # don't cache partial content + cache_content = False + data = self._response.body.read(amt) + else: # read all we can (and cache it) + data = self._response.body.read() + if cache_content: + self._body = data + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if len(data) == 0 or ( + self.length_is_certain and self.length_remaining == 0 + ): + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Generator[bytes]: + # chunked is handled by browser + while True: + bytes = self.read(amt, decode_content) + if not bytes: + break + yield bytes + + def release_conn(self) -> None: + if not self._pool or not self._connection: + return None + + self._pool._put_conn(self._connection) + self._connection = None + + def drain_conn(self) -> None: + self.close() + + @property + def data(self) -> bytes: + if self._body: + return self._body + else: + return self.read(cache_content=True) + + def json(self) -> typing.Any: + """ + Deserializes the body of the HTTP response as a Python object. + + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. + + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here <json_content>`. + + :returns: The body of the HTTP response as a Python object. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + + def close(self) -> None: + if not self._closed: + if isinstance(self._response.body, IOBase): + self._response.body.close() + if self._connection: + self._connection.close() + self._connection = None + self._closed = True + + @contextmanager + def _error_catcher(self) -> typing.Generator[None]: + """ + Catch Emscripten specific exceptions thrown by fetch.py, + instead re-raising urllib3 variants, so that low-level exceptions + are not leaked in the high-level api. + + On exit, release the connection back to the pool. + """ + from .fetch import _RequestError, _TimeoutError # avoid circular import + + clean_exit = False + + try: + yield + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + except _TimeoutError as e: + raise TimeoutError(str(e)) + except _RequestError as e: + raise HTTPException(str(e)) + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now + if ( + isinstance(self._response.body, IOBase) + and not self._response.body.closed + ): + self._response.body.close() + # release the connection back to the pool + self.release_conn() + else: + # If we have read everything from the response stream, + # return the connection back to the pool. + if ( + isinstance(self._response.body, IOBase) + and self._response.body.closed + ): + self.release_conn() diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/ntlmpool.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/ntlmpool.py deleted file mode 100644 index 471665754e9..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/ntlmpool.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -NTLM authenticating pool, contributed by erikcederstran - -Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 -""" -from __future__ import absolute_import - -import warnings -from logging import getLogger - -from ntlm import ntlm - -from .. import HTTPSConnectionPool -from ..packages.six.moves.http_client import HTTPSConnection - -warnings.warn( - "The 'urllib3.contrib.ntlmpool' module is deprecated and will be removed " - "in urllib3 v2.0 release, urllib3 is not able to support it properly due " - "to reasons listed in issue: https://github.com/urllib3/urllib3/issues/2282. " - "If you are a user of this module please comment in the mentioned issue.", - DeprecationWarning, -) - -log = getLogger(__name__) - - -class NTLMConnectionPool(HTTPSConnectionPool): - """ - Implements an NTLM authentication version of an urllib3 connection pool - """ - - scheme = "https" - - def __init__(self, user, pw, authurl, *args, **kwargs): - """ - authurl is a random URL on the server that is protected by NTLM. - user is the Windows user, probably in the DOMAIN\\username format. - pw is the password for the user. - """ - super(NTLMConnectionPool, self).__init__(*args, **kwargs) - self.authurl = authurl - self.rawuser = user - user_parts = user.split("\\", 1) - self.domain = user_parts[0].upper() - self.user = user_parts[1] - self.pw = pw - - def _new_conn(self): - # Performs the NTLM handshake that secures the connection. The socket - # must be kept open while requests are performed. - self.num_connections += 1 - log.debug( - "Starting NTLM HTTPS connection no. %d: https://%s%s", - self.num_connections, - self.host, - self.authurl, - ) - - headers = {"Connection": "Keep-Alive"} - req_header = "Authorization" - resp_header = "www-authenticate" - - conn = HTTPSConnection(host=self.host, port=self.port) - - # Send negotiation message - headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( - self.rawuser - ) - log.debug("Request headers: %s", headers) - conn.request("GET", self.authurl, None, headers) - res = conn.getresponse() - reshdr = dict(res.headers) - log.debug("Response status: %s %s", res.status, res.reason) - log.debug("Response headers: %s", reshdr) - log.debug("Response data: %s [...]", res.read(100)) - - # Remove the reference to the socket, so that it can not be closed by - # the response object (we want to keep the socket open) - res.fp = None - - # Server should respond with a challenge message - auth_header_values = reshdr[resp_header].split(", ") - auth_header_value = None - for s in auth_header_values: - if s[:5] == "NTLM ": - auth_header_value = s[5:] - if auth_header_value is None: - raise Exception( - "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) - ) - - # Send authentication message - ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( - auth_header_value - ) - auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( - ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags - ) - headers[req_header] = "NTLM %s" % auth_msg - log.debug("Request headers: %s", headers) - conn.request("GET", self.authurl, None, headers) - res = conn.getresponse() - log.debug("Response status: %s %s", res.status, res.reason) - log.debug("Response headers: %s", dict(res.headers)) - log.debug("Response data: %s [...]", res.read()[:100]) - if res.status != 200: - if res.status == 401: - raise Exception("Server rejected request: wrong username or password") - raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) - - res.fp = None - log.debug("Connection established") - return conn - - def urlopen( - self, - method, - url, - body=None, - headers=None, - retries=3, - redirect=True, - assert_same_host=True, - ): - if headers is None: - headers = {} - headers["Connection"] = "Keep-Alive" - return super(NTLMConnectionPool, self).urlopen( - method, url, body, headers, retries, redirect, assert_same_host - ) diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/pyopenssl.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/pyopenssl.py index 19e4aa97cc1..a7b16a02bac 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -1,17 +1,17 @@ """ -TLS with SNI_-support for Python 2. Follow these instructions if you would -like to verify TLS certificates in Python 2. Note, the default libraries do -*not* do certificate checking; you need to do additional work to validate -certificates yourself. +Module for using pyOpenSSL as a TLS backend. This module was relevant before +the standard library ``ssl`` module supported SNI, but now that we've dropped +support for Python 2.7 all relevant Python versions support SNI so +**this module is no longer recommended**. This needs the following packages installed: * `pyOpenSSL`_ (tested with 16.0.0) * `cryptography`_ (minimum 1.3.4, from pyopenssl) -* `idna`_ (minimum 2.0, from cryptography) +* `idna`_ (minimum 2.0) -However, pyopenssl depends on cryptography, which depends on idna, so while we -use all three directly here we end up having relatively few packages required. +However, pyOpenSSL depends on cryptography, so while we use all three directly here we +end up having relatively few packages required. You can install them with the following command: @@ -33,75 +33,46 @@ like this: except ImportError: pass -Now you can use :mod:`urllib3` as you normally would, and it will support SNI -when the required modules are installed. - -Activating this module also has the positive side effect of disabling SSL/TLS -compression in Python 2 (see `CRIME attack`_). - -.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication -.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) .. _pyopenssl: https://www.pyopenssl.org .. _cryptography: https://cryptography.io .. _idna: https://github.com/kjd/idna """ -from __future__ import absolute_import -import OpenSSL.crypto -import OpenSSL.SSL +from __future__ import annotations + +import OpenSSL.SSL # type: ignore[import-not-found] from cryptography import x509 -from cryptography.hazmat.backends.openssl import backend as openssl_backend try: - from cryptography.x509 import UnsupportedExtension + from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined] except ImportError: # UnsupportedExtension is gone in cryptography >= 2.1.0 - class UnsupportedExtension(Exception): + class UnsupportedExtension(Exception): # type: ignore[no-redef] pass -from io import BytesIO -from socket import error as SocketError -from socket import timeout - -try: # Platform-specific: Python 2 - from socket import _fileobject -except ImportError: # Platform-specific: Python 3 - _fileobject = None - from ..packages.backports.makefile import backport_makefile - import logging import ssl -import sys -import warnings +import typing +from io import BytesIO +from socket import socket as socket_cls +from socket import timeout from .. import util -from ..packages import six -from ..util.ssl_ import PROTOCOL_TLS_CLIENT -warnings.warn( - "'urllib3.contrib.pyopenssl' module is deprecated and will be removed " - "in a future release of urllib3 2.x. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2680", - category=DeprecationWarning, - stacklevel=2, -) +if typing.TYPE_CHECKING: + from OpenSSL.crypto import X509 # type: ignore[import-not-found] -__all__ = ["inject_into_urllib3", "extract_from_urllib3"] -# SNI always works. -HAS_SNI = True +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # Map from urllib3 to PyOpenSSL compatible parameter-values. -_openssl_versions = { - util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, - PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, +_openssl_versions: dict[int, int] = { + util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined] + util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined] ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } -if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): - _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD - if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD @@ -115,43 +86,77 @@ _stdlib_to_openssl_verify = { ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) +_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()} + +# The SSLvX values are the most likely to be missing in the future +# but we check them all just to be sure. +_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr( + OpenSSL.SSL, "OP_NO_SSLv3", 0 +) +_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0) +_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0) +_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0) +_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0) + +_openssl_to_ssl_minimum_version: dict[int, int] = { + ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1, + ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1, + ssl.TLSVersion.TLSv1_3: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 + ), + ssl.TLSVersion.MAXIMUM_SUPPORTED: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 + ), +} +_openssl_to_ssl_maximum_version: dict[int, int] = { + ssl.TLSVersion.MINIMUM_SUPPORTED: ( + _OP_NO_SSLv2_OR_SSLv3 + | _OP_NO_TLSv1 + | _OP_NO_TLSv1_1 + | _OP_NO_TLSv1_2 + | _OP_NO_TLSv1_3 + ), + ssl.TLSVersion.TLSv1: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3 + ), + ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3, + ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3, + ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3, +} # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 -orig_util_HAS_SNI = util.HAS_SNI orig_util_SSLContext = util.ssl_.SSLContext log = logging.getLogger(__name__) -def inject_into_urllib3(): +def inject_into_urllib3() -> None: "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." _validate_dependencies_met() - util.SSLContext = PyOpenSSLContext - util.ssl_.SSLContext = PyOpenSSLContext - util.HAS_SNI = HAS_SNI - util.ssl_.HAS_SNI = HAS_SNI + util.SSLContext = PyOpenSSLContext # type: ignore[assignment] + util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment] util.IS_PYOPENSSL = True util.ssl_.IS_PYOPENSSL = True -def extract_from_urllib3(): +def extract_from_urllib3() -> None: "Undo monkey-patching by :func:`inject_into_urllib3`." util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext - util.HAS_SNI = orig_util_HAS_SNI - util.ssl_.HAS_SNI = orig_util_HAS_SNI util.IS_PYOPENSSL = False util.ssl_.IS_PYOPENSSL = False -def _validate_dependencies_met(): +def _validate_dependencies_met() -> None: """ Verifies that PyOpenSSL's package-level dependencies have been met. Throws `ImportError` if they are not met. @@ -177,7 +182,7 @@ def _validate_dependencies_met(): ) -def _dnsname_to_stdlib(name): +def _dnsname_to_stdlib(name: str) -> str | None: """ Converts a dNSName SubjectAlternativeName field to the form used by the standard library on the given Python version. @@ -191,7 +196,7 @@ def _dnsname_to_stdlib(name): the name given should be skipped. """ - def idna_encode(name): + def idna_encode(name: str) -> bytes | None: """ Borrowed wholesale from the Python Cryptography Project. It turns out that we can't just safely call `idna.encode`: it can explode for @@ -200,7 +205,7 @@ def _dnsname_to_stdlib(name): from pip._vendor import idna try: - for prefix in [u"*.", u"."]: + for prefix in ["*.", "."]: if name.startswith(prefix): name = name[len(prefix) :] return prefix.encode("ascii") + idna.encode(name) @@ -212,24 +217,17 @@ def _dnsname_to_stdlib(name): if ":" in name: return name - name = idna_encode(name) - if name is None: + encoded_name = idna_encode(name) + if encoded_name is None: return None - elif sys.version_info >= (3, 0): - name = name.decode("utf-8") - return name + return encoded_name.decode("utf-8") -def get_subj_alt_name(peer_cert): +def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]: """ Given an PyOpenSSL certificate, provides all the subject alternative names. """ - # Pass the cert to cryptography, which has much better APIs for this. - if hasattr(peer_cert, "to_cryptography"): - cert = peer_cert.to_cryptography() - else: - der = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, peer_cert) - cert = x509.load_der_x509_certificate(der, openssl_backend) + cert = peer_cert.to_cryptography() # We want to find the SAN extension. Ask Cryptography to locate it (it's # faster than looping in Python) @@ -273,93 +271,94 @@ def get_subj_alt_name(peer_cert): return names -class WrappedSocket(object): - """API-compatibility wrapper for Python OpenSSL's Connection-class. - - Note: _makefile_refs, _drop() and _reuse() are needed for the garbage - collector of pypy. - """ +class WrappedSocket: + """API-compatibility wrapper for Python OpenSSL's Connection-class.""" - def __init__(self, connection, socket, suppress_ragged_eofs=True): + def __init__( + self, + connection: OpenSSL.SSL.Connection, + socket: socket_cls, + suppress_ragged_eofs: bool = True, + ) -> None: self.connection = connection self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs - self._makefile_refs = 0 + self._io_refs = 0 self._closed = False - def fileno(self): + def fileno(self) -> int: return self.socket.fileno() # Copy-pasted from Python 3.5 source code - def _decref_socketios(self): - if self._makefile_refs > 0: - self._makefile_refs -= 1 + def _decref_socketios(self) -> None: + if self._io_refs > 0: + self._io_refs -= 1 if self._closed: self.close() - def recv(self, *args, **kwargs): + def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes: try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return b"" else: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b"" else: raise - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout("The read operation timed out") + raise timeout("The read operation timed out") from e else: return self.recv(*args, **kwargs) # TLS 1.3 post-handshake authentication except OpenSSL.SSL.Error as e: - raise ssl.SSLError("read error: %r" % e) + raise ssl.SSLError(f"read error: {e!r}") from e else: - return data + return data # type: ignore[no-any-return] - def recv_into(self, *args, **kwargs): + def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int: try: - return self.connection.recv_into(*args, **kwargs) + return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return] except OpenSSL.SSL.SysCallError as e: if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return 0 else: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return 0 else: raise - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout("The read operation timed out") + raise timeout("The read operation timed out") from e else: return self.recv_into(*args, **kwargs) # TLS 1.3 post-handshake authentication except OpenSSL.SSL.Error as e: - raise ssl.SSLError("read error: %r" % e) + raise ssl.SSLError(f"read error: {e!r}") from e - def settimeout(self, timeout): + def settimeout(self, timeout: float) -> None: return self.socket.settimeout(timeout) - def _send_until_done(self, data): + def _send_until_done(self, data: bytes) -> int: while True: try: - return self.connection.send(data) - except OpenSSL.SSL.WantWriteError: + return self.connection.send(data) # type: ignore[no-any-return] + except OpenSSL.SSL.WantWriteError as e: if not util.wait_for_write(self.socket, self.socket.gettimeout()): - raise timeout() + raise timeout() from e continue except OpenSSL.SSL.SysCallError as e: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e - def sendall(self, data): + def sendall(self, data: bytes) -> None: total_sent = 0 while total_sent < len(data): sent = self._send_until_done( @@ -367,135 +366,151 @@ class WrappedSocket(object): ) total_sent += sent - def shutdown(self): - # FIXME rethrow compatible exceptions should we ever use this - self.connection.shutdown() + def shutdown(self, how: int) -> None: + try: + self.connection.shutdown() + except OpenSSL.SSL.Error as e: + raise ssl.SSLError(f"shutdown error: {e!r}") from e - def close(self): - if self._makefile_refs < 1: - try: - self._closed = True - return self.connection.close() - except OpenSSL.SSL.Error: - return - else: - self._makefile_refs -= 1 + def close(self) -> None: + self._closed = True + if self._io_refs <= 0: + self._real_close() - def getpeercert(self, binary_form=False): + def _real_close(self) -> None: + try: + return self.connection.close() # type: ignore[no-any-return] + except OpenSSL.SSL.Error: + return + + def getpeercert( + self, binary_form: bool = False + ) -> dict[str, list[typing.Any]] | None: x509 = self.connection.get_peer_certificate() if not x509: - return x509 + return x509 # type: ignore[no-any-return] if binary_form: - return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) + return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return] return { - "subject": ((("commonName", x509.get_subject().CN),),), + "subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item] "subjectAltName": get_subj_alt_name(x509), } - def version(self): - return self.connection.get_protocol_version_name() - - def _reuse(self): - self._makefile_refs += 1 - - def _drop(self): - if self._makefile_refs < 1: - self.close() - else: - self._makefile_refs -= 1 - - -if _fileobject: # Platform-specific: Python 2 + def version(self) -> str: + return self.connection.get_protocol_version_name() # type: ignore[no-any-return] - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) + def selected_alpn_protocol(self) -> str | None: + alpn_proto = self.connection.get_alpn_proto_negotiated() + return alpn_proto.decode() if alpn_proto else None -else: # Platform-specific: Python 3 - makefile = backport_makefile -WrappedSocket.makefile = makefile +WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined] -class PyOpenSSLContext(object): +class PyOpenSSLContext: """ I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible for translating the interface of the standard library ``SSLContext`` object to calls into PyOpenSSL. """ - def __init__(self, protocol): + def __init__(self, protocol: int) -> None: self.protocol = _openssl_versions[protocol] self._ctx = OpenSSL.SSL.Context(self.protocol) self._options = 0 self.check_hostname = False + self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED + self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED + self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST @property - def options(self): + def options(self) -> int: return self._options @options.setter - def options(self, value): + def options(self, value: int) -> None: self._options = value - self._ctx.set_options(value) + self._set_ctx_options() + + @property + def verify_flags(self) -> int: + return self._verify_flags + + @verify_flags.setter + def verify_flags(self, value: int) -> None: + self._verify_flags = value + self._ctx.get_cert_store().set_flags(self._verify_flags) @property - def verify_mode(self): + def verify_mode(self) -> int: return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()] @verify_mode.setter - def verify_mode(self, value): + def verify_mode(self, value: ssl.VerifyMode) -> None: self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) - def set_default_verify_paths(self): + def set_default_verify_paths(self) -> None: self._ctx.set_default_verify_paths() - def set_ciphers(self, ciphers): - if isinstance(ciphers, six.text_type): + def set_ciphers(self, ciphers: bytes | str) -> None: + if isinstance(ciphers, str): ciphers = ciphers.encode("utf-8") self._ctx.set_cipher_list(ciphers) - def load_verify_locations(self, cafile=None, capath=None, cadata=None): + def load_verify_locations( + self, + cafile: str | None = None, + capath: str | None = None, + cadata: bytes | None = None, + ) -> None: if cafile is not None: - cafile = cafile.encode("utf-8") + cafile = cafile.encode("utf-8") # type: ignore[assignment] if capath is not None: - capath = capath.encode("utf-8") + capath = capath.encode("utf-8") # type: ignore[assignment] try: self._ctx.load_verify_locations(cafile, capath) if cadata is not None: self._ctx.load_verify_locations(BytesIO(cadata)) except OpenSSL.SSL.Error as e: - raise ssl.SSLError("unable to load trusted certificates: %r" % e) + raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e - def load_cert_chain(self, certfile, keyfile=None, password=None): - self._ctx.use_certificate_chain_file(certfile) - if password is not None: - if not isinstance(password, six.binary_type): - password = password.encode("utf-8") - self._ctx.set_passwd_cb(lambda *_: password) - self._ctx.use_privatekey_file(keyfile or certfile) + def load_cert_chain( + self, + certfile: str, + keyfile: str | None = None, + password: str | None = None, + ) -> None: + try: + self._ctx.use_certificate_chain_file(certfile) + if password is not None: + if not isinstance(password, bytes): + password = password.encode("utf-8") # type: ignore[assignment] + self._ctx.set_passwd_cb(lambda *_: password) + self._ctx.use_privatekey_file(keyfile or certfile) + except OpenSSL.SSL.Error as e: + raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e - def set_alpn_protocols(self, protocols): - protocols = [six.ensure_binary(p) for p in protocols] - return self._ctx.set_alpn_protos(protocols) + def set_alpn_protocols(self, protocols: list[bytes | str]) -> None: + protocols = [util.util.to_bytes(p, "ascii") for p in protocols] + return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return] def wrap_socket( self, - sock, - server_side=False, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - server_hostname=None, - ): + sock: socket_cls, + server_side: bool = False, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + server_hostname: bytes | str | None = None, + ) -> WrappedSocket: cnx = OpenSSL.SSL.Connection(self._ctx, sock) - if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode("utf-8") - - if server_hostname is not None: + # If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3 + if server_hostname and not util.ssl_.is_ipaddress(server_hostname): + if isinstance(server_hostname, str): + server_hostname = server_hostname.encode("utf-8") cnx.set_tlsext_host_name(server_hostname) cnx.set_connect_state() @@ -503,16 +518,47 @@ class PyOpenSSLContext(object): while True: try: cnx.do_handshake() - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(sock, sock.gettimeout()): - raise timeout("select timed out") + raise timeout("select timed out") from e continue except OpenSSL.SSL.Error as e: - raise ssl.SSLError("bad handshake: %r" % e) + raise ssl.SSLError(f"bad handshake: {e!r}") from e break return WrappedSocket(cnx, sock) + def _set_ctx_options(self) -> None: + self._ctx.set_options( + self._options + | _openssl_to_ssl_minimum_version[self._minimum_version] + | _openssl_to_ssl_maximum_version[self._maximum_version] + ) + + @property + def minimum_version(self) -> int: + return self._minimum_version + + @minimum_version.setter + def minimum_version(self, minimum_version: int) -> None: + self._minimum_version = minimum_version + self._set_ctx_options() + + @property + def maximum_version(self) -> int: + return self._maximum_version + + @maximum_version.setter + def maximum_version(self, maximum_version: int) -> None: + self._maximum_version = maximum_version + self._set_ctx_options() + -def _verify_callback(cnx, x509, err_no, err_depth, return_code): +def _verify_callback( + cnx: OpenSSL.SSL.Connection, + x509: X509, + err_no: int, + err_depth: int, + return_code: int, +) -> bool: return err_no == 0 diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/securetransport.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/securetransport.py deleted file mode 100644 index 722ee4e1242..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/securetransport.py +++ /dev/null @@ -1,920 +0,0 @@ -""" -SecureTranport support for urllib3 via ctypes. - -This makes platform-native TLS available to urllib3 users on macOS without the -use of a compiler. This is an important feature because the Python Package -Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL -that ships with macOS is not capable of doing TLSv1.2. The only way to resolve -this is to give macOS users an alternative solution to the problem, and that -solution is to use SecureTransport. - -We use ctypes here because this solution must not require a compiler. That's -because pip is not allowed to require a compiler either. - -This is not intended to be a seriously long-term solution to this problem. -The hope is that PEP 543 will eventually solve this issue for us, at which -point we can retire this contrib module. But in the short term, we need to -solve the impending tire fire that is Python on Mac without this kind of -contrib module. So...here we are. - -To use this module, simply import and inject it:: - - import pip._vendor.urllib3.contrib.securetransport as securetransport - securetransport.inject_into_urllib3() - -Happy TLSing! - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - -.. code-block:: - - Copyright (c) 2015-2016 Will Bond <[email protected]> - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" -from __future__ import absolute_import - -import contextlib -import ctypes -import errno -import os.path -import shutil -import socket -import ssl -import struct -import threading -import weakref - -from .. import util -from ..packages import six -from ..util.ssl_ import PROTOCOL_TLS_CLIENT -from ._securetransport.bindings import CoreFoundation, Security, SecurityConst -from ._securetransport.low_level import ( - _assert_no_error, - _build_tls_unknown_ca_alert, - _cert_array_from_pem, - _create_cfstring_array, - _load_client_cert_chain, - _temporary_keychain, -) - -try: # Platform-specific: Python 2 - from socket import _fileobject -except ImportError: # Platform-specific: Python 3 - _fileobject = None - from ..packages.backports.makefile import backport_makefile - -__all__ = ["inject_into_urllib3", "extract_from_urllib3"] - -# SNI always works -HAS_SNI = True - -orig_util_HAS_SNI = util.HAS_SNI -orig_util_SSLContext = util.ssl_.SSLContext - -# This dictionary is used by the read callback to obtain a handle to the -# calling wrapped socket. This is a pretty silly approach, but for now it'll -# do. I feel like I should be able to smuggle a handle to the wrapped socket -# directly in the SSLConnectionRef, but for now this approach will work I -# guess. -# -# We need to lock around this structure for inserts, but we don't do it for -# reads/writes in the callbacks. The reasoning here goes as follows: -# -# 1. It is not possible to call into the callbacks before the dictionary is -# populated, so once in the callback the id must be in the dictionary. -# 2. The callbacks don't mutate the dictionary, they only read from it, and -# so cannot conflict with any of the insertions. -# -# This is good: if we had to lock in the callbacks we'd drastically slow down -# the performance of this code. -_connection_refs = weakref.WeakValueDictionary() -_connection_ref_lock = threading.Lock() - -# Limit writes to 16kB. This is OpenSSL's limit, but we'll cargo-cult it over -# for no better reason than we need *a* limit, and this one is right there. -SSL_WRITE_BLOCKSIZE = 16384 - -# This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to -# individual cipher suites. We need to do this because this is how -# SecureTransport wants them. -CIPHER_SUITES = [ - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_AES_256_GCM_SHA384, - SecurityConst.TLS_AES_128_GCM_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_AES_128_CCM_8_SHA256, - SecurityConst.TLS_AES_128_CCM_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA, -] - -# Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of -# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. -# TLSv1 to 1.2 are supported on macOS 10.8+ -_protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), - PROTOCOL_TLS_CLIENT: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), -} - -if hasattr(ssl, "PROTOCOL_SSLv2"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, - SecurityConst.kSSLProtocol2, - ) -if hasattr(ssl, "PROTOCOL_SSLv3"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, - SecurityConst.kSSLProtocol3, - ) -if hasattr(ssl, "PROTOCOL_TLSv1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, - SecurityConst.kTLSProtocol1, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, - SecurityConst.kTLSProtocol11, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_2"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, - SecurityConst.kTLSProtocol12, - ) - - -def inject_into_urllib3(): - """ - Monkey-patch urllib3 with SecureTransport-backed SSL-support. - """ - util.SSLContext = SecureTransportContext - util.ssl_.SSLContext = SecureTransportContext - util.HAS_SNI = HAS_SNI - util.ssl_.HAS_SNI = HAS_SNI - util.IS_SECURETRANSPORT = True - util.ssl_.IS_SECURETRANSPORT = True - - -def extract_from_urllib3(): - """ - Undo monkey-patching by :func:`inject_into_urllib3`. - """ - util.SSLContext = orig_util_SSLContext - util.ssl_.SSLContext = orig_util_SSLContext - util.HAS_SNI = orig_util_HAS_SNI - util.ssl_.HAS_SNI = orig_util_HAS_SNI - util.IS_SECURETRANSPORT = False - util.ssl_.IS_SECURETRANSPORT = False - - -def _read_callback(connection_id, data_buffer, data_length_pointer): - """ - SecureTransport read callback. This is called by ST to request that data - be returned from the socket. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - requested_length = data_length_pointer[0] - - timeout = wrapped_socket.gettimeout() - error = None - read_count = 0 - - try: - while read_count < requested_length: - if timeout is None or timeout >= 0: - if not util.wait_for_read(base_socket, timeout): - raise socket.error(errno.EAGAIN, "timed out") - - remaining = requested_length - read_count - buffer = (ctypes.c_char * remaining).from_address( - data_buffer + read_count - ) - chunk_size = base_socket.recv_into(buffer, remaining) - read_count += chunk_size - if not chunk_size: - if not read_count: - return SecurityConst.errSSLClosedGraceful - break - except (socket.error) as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = read_count - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = read_count - - if read_count != requested_length: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -def _write_callback(connection_id, data_buffer, data_length_pointer): - """ - SecureTransport write callback. This is called by ST to request that data - actually be sent on the network. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - bytes_to_write = data_length_pointer[0] - data = ctypes.string_at(data_buffer, bytes_to_write) - - timeout = wrapped_socket.gettimeout() - error = None - sent = 0 - - try: - while sent < bytes_to_write: - if timeout is None or timeout >= 0: - if not util.wait_for_write(base_socket, timeout): - raise socket.error(errno.EAGAIN, "timed out") - chunk_sent = base_socket.send(data) - sent += chunk_sent - - # This has some needless copying here, but I'm not sure there's - # much value in optimising this data path. - data = data[chunk_sent:] - except (socket.error) as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = sent - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = sent - - if sent != bytes_to_write: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -# We need to keep these two objects references alive: if they get GC'd while -# in use then SecureTransport could attempt to call a function that is in freed -# memory. That would be...uh...bad. Yeah, that's the word. Bad. -_read_callback_pointer = Security.SSLReadFunc(_read_callback) -_write_callback_pointer = Security.SSLWriteFunc(_write_callback) - - -class WrappedSocket(object): - """ - API-compatibility wrapper for Python's OpenSSL wrapped socket object. - - Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage - collector of PyPy. - """ - - def __init__(self, socket): - self.socket = socket - self.context = None - self._makefile_refs = 0 - self._closed = False - self._exception = None - self._keychain = None - self._keychain_dir = None - self._client_cert_chain = None - - # We save off the previously-configured timeout and then set it to - # zero. This is done because we use select and friends to handle the - # timeouts, but if we leave the timeout set on the lower socket then - # Python will "kindly" call select on that socket again for us. Avoid - # that by forcing the timeout to zero. - self._timeout = self.socket.gettimeout() - self.socket.settimeout(0) - - @contextlib.contextmanager - def _raise_on_error(self): - """ - A context manager that can be used to wrap calls that do I/O from - SecureTransport. If any of the I/O callbacks hit an exception, this - context manager will correctly propagate the exception after the fact. - This avoids silently swallowing those exceptions. - - It also correctly forces the socket closed. - """ - self._exception = None - - # We explicitly don't catch around this yield because in the unlikely - # event that an exception was hit in the block we don't want to swallow - # it. - yield - if self._exception is not None: - exception, self._exception = self._exception, None - self.close() - raise exception - - def _set_ciphers(self): - """ - Sets up the allowed ciphers. By default this matches the set in - util.ssl_.DEFAULT_CIPHERS, at least as supported by macOS. This is done - custom and doesn't allow changing at this time, mostly because parsing - OpenSSL cipher strings is going to be a freaking nightmare. - """ - ciphers = (Security.SSLCipherSuite * len(CIPHER_SUITES))(*CIPHER_SUITES) - result = Security.SSLSetEnabledCiphers( - self.context, ciphers, len(CIPHER_SUITES) - ) - _assert_no_error(result) - - def _set_alpn_protocols(self, protocols): - """ - Sets up the ALPN protocols on the context. - """ - if not protocols: - return - protocols_arr = _create_cfstring_array(protocols) - try: - result = Security.SSLSetALPNProtocols(self.context, protocols_arr) - _assert_no_error(result) - finally: - CoreFoundation.CFRelease(protocols_arr) - - def _custom_validate(self, verify, trust_bundle): - """ - Called when we have set custom validation. We do this in two cases: - first, when cert validation is entirely disabled; and second, when - using a custom trust DB. - Raises an SSLError if the connection is not trusted. - """ - # If we disabled cert validation, just say: cool. - if not verify: - return - - successes = ( - SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed, - ) - try: - trust_result = self._evaluate_trust(trust_bundle) - if trust_result in successes: - return - reason = "error code: %d" % (trust_result,) - except Exception as e: - # Do not trust on error - reason = "exception: %r" % (e,) - - # SecureTransport does not send an alert nor shuts down the connection. - rec = _build_tls_unknown_ca_alert(self.version()) - self.socket.sendall(rec) - # close the connection immediately - # l_onoff = 1, activate linger - # l_linger = 0, linger for 0 seoncds - opts = struct.pack("ii", 1, 0) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, opts) - self.close() - raise ssl.SSLError("certificate verify failed, %s" % reason) - - def _evaluate_trust(self, trust_bundle): - # We want data in memory, so load it up. - if os.path.isfile(trust_bundle): - with open(trust_bundle, "rb") as f: - trust_bundle = f.read() - - cert_array = None - trust = Security.SecTrustRef() - - try: - # Get a CFArray that contains the certs we want. - cert_array = _cert_array_from_pem(trust_bundle) - - # Ok, now the hard part. We want to get the SecTrustRef that ST has - # created for this connection, shove our CAs into it, tell ST to - # ignore everything else it knows, and then ask if it can build a - # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - raise ssl.SSLError("Failed to copy trust reference") - - result = Security.SecTrustSetAnchorCertificates(trust, cert_array) - _assert_no_error(result) - - result = Security.SecTrustSetAnchorCertificatesOnly(trust, True) - _assert_no_error(result) - - trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) - _assert_no_error(result) - finally: - if trust: - CoreFoundation.CFRelease(trust) - - if cert_array is not None: - CoreFoundation.CFRelease(cert_array) - - return trust_result.value - - def handshake( - self, - server_hostname, - verify, - trust_bundle, - min_version, - max_version, - client_cert, - client_key, - client_key_passphrase, - alpn_protocols, - ): - """ - Actually performs the TLS handshake. This is run automatically by - wrapped socket, and shouldn't be needed in user code. - """ - # First, we do the initial bits of connection setup. We need to create - # a context, set its I/O funcs, and set the connection reference. - self.context = Security.SSLCreateContext( - None, SecurityConst.kSSLClientSide, SecurityConst.kSSLStreamType - ) - result = Security.SSLSetIOFuncs( - self.context, _read_callback_pointer, _write_callback_pointer - ) - _assert_no_error(result) - - # Here we need to compute the handle to use. We do this by taking the - # id of self modulo 2**31 - 1. If this is already in the dictionary, we - # just keep incrementing by one until we find a free space. - with _connection_ref_lock: - handle = id(self) % 2147483647 - while handle in _connection_refs: - handle = (handle + 1) % 2147483647 - _connection_refs[handle] = self - - result = Security.SSLSetConnection(self.context, handle) - _assert_no_error(result) - - # If we have a server hostname, we should set that too. - if server_hostname: - if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode("utf-8") - - result = Security.SSLSetPeerDomainName( - self.context, server_hostname, len(server_hostname) - ) - _assert_no_error(result) - - # Setup the ciphers. - self._set_ciphers() - - # Setup the ALPN protocols. - self._set_alpn_protocols(alpn_protocols) - - # Set the minimum and maximum TLS versions. - result = Security.SSLSetProtocolVersionMin(self.context, min_version) - _assert_no_error(result) - - result = Security.SSLSetProtocolVersionMax(self.context, max_version) - _assert_no_error(result) - - # If there's a trust DB, we need to use it. We do that by telling - # SecureTransport to break on server auth. We also do that if we don't - # want to validate the certs at all: we just won't actually do any - # authing in that case. - if not verify or trust_bundle is not None: - result = Security.SSLSetSessionOption( - self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True - ) - _assert_no_error(result) - - # If there's a client cert, we need to use it. - if client_cert: - self._keychain, self._keychain_dir = _temporary_keychain() - self._client_cert_chain = _load_client_cert_chain( - self._keychain, client_cert, client_key - ) - result = Security.SSLSetCertificate(self.context, self._client_cert_chain) - _assert_no_error(result) - - while True: - with self._raise_on_error(): - result = Security.SSLHandshake(self.context) - - if result == SecurityConst.errSSLWouldBlock: - raise socket.timeout("handshake timed out") - elif result == SecurityConst.errSSLServerAuthCompleted: - self._custom_validate(verify, trust_bundle) - continue - else: - _assert_no_error(result) - break - - def fileno(self): - return self.socket.fileno() - - # Copy-pasted from Python 3.5 source code - def _decref_socketios(self): - if self._makefile_refs > 0: - self._makefile_refs -= 1 - if self._closed: - self.close() - - def recv(self, bufsiz): - buffer = ctypes.create_string_buffer(bufsiz) - bytes_read = self.recv_into(buffer, bufsiz) - data = buffer[:bytes_read] - return data - - def recv_into(self, buffer, nbytes=None): - # Read short on EOF. - if self._closed: - return 0 - - if nbytes is None: - nbytes = len(buffer) - - buffer = (ctypes.c_char * nbytes).from_buffer(buffer) - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLRead( - self.context, buffer, nbytes, ctypes.byref(processed_bytes) - ) - - # There are some result codes that we want to treat as "not always - # errors". Specifically, those are errSSLWouldBlock, - # errSSLClosedGraceful, and errSSLClosedNoNotify. - if result == SecurityConst.errSSLWouldBlock: - # If we didn't process any bytes, then this was just a time out. - # However, we can get errSSLWouldBlock in situations when we *did* - # read some data, and in those cases we should just read "short" - # and return. - if processed_bytes.value == 0: - # Timed out, no data read. - raise socket.timeout("recv timed out") - elif result in ( - SecurityConst.errSSLClosedGraceful, - SecurityConst.errSSLClosedNoNotify, - ): - # The remote peer has closed this connection. We should do so as - # well. Note that we don't actually return here because in - # principle this could actually be fired along with return data. - # It's unlikely though. - self.close() - else: - _assert_no_error(result) - - # Ok, we read and probably succeeded. We should return whatever data - # was actually read. - return processed_bytes.value - - def settimeout(self, timeout): - self._timeout = timeout - - def gettimeout(self): - return self._timeout - - def send(self, data): - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLWrite( - self.context, data, len(data), ctypes.byref(processed_bytes) - ) - - if result == SecurityConst.errSSLWouldBlock and processed_bytes.value == 0: - # Timed out - raise socket.timeout("send timed out") - else: - _assert_no_error(result) - - # We sent, and probably succeeded. Tell them how much we sent. - return processed_bytes.value - - def sendall(self, data): - total_sent = 0 - while total_sent < len(data): - sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) - total_sent += sent - - def shutdown(self): - with self._raise_on_error(): - Security.SSLClose(self.context) - - def close(self): - # TODO: should I do clean shutdown here? Do I have to? - if self._makefile_refs < 1: - self._closed = True - if self.context: - CoreFoundation.CFRelease(self.context) - self.context = None - if self._client_cert_chain: - CoreFoundation.CFRelease(self._client_cert_chain) - self._client_cert_chain = None - if self._keychain: - Security.SecKeychainDelete(self._keychain) - CoreFoundation.CFRelease(self._keychain) - shutil.rmtree(self._keychain_dir) - self._keychain = self._keychain_dir = None - return self.socket.close() - else: - self._makefile_refs -= 1 - - def getpeercert(self, binary_form=False): - # Urgh, annoying. - # - # Here's how we do this: - # - # 1. Call SSLCopyPeerTrust to get hold of the trust object for this - # connection. - # 2. Call SecTrustGetCertificateAtIndex for index 0 to get the leaf. - # 3. To get the CN, call SecCertificateCopyCommonName and process that - # string so that it's of the appropriate type. - # 4. To get the SAN, we need to do something a bit more complex: - # a. Call SecCertificateCopyValues to get the data, requesting - # kSecOIDSubjectAltName. - # b. Mess about with this dictionary to try to get the SANs out. - # - # This is gross. Really gross. It's going to be a few hundred LoC extra - # just to repeat something that SecureTransport can *already do*. So my - # operating assumption at this time is that what we want to do is - # instead to just flag to urllib3 that it shouldn't do its own hostname - # validation when using SecureTransport. - if not binary_form: - raise ValueError("SecureTransport only supports dumping binary certs") - trust = Security.SecTrustRef() - certdata = None - der_bytes = None - - try: - # Grab the trust store. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - # Probably we haven't done the handshake yet. No biggie. - return None - - cert_count = Security.SecTrustGetCertificateCount(trust) - if not cert_count: - # Also a case that might happen if we haven't handshaked. - # Handshook? Handshaken? - return None - - leaf = Security.SecTrustGetCertificateAtIndex(trust, 0) - assert leaf - - # Ok, now we want the DER bytes. - certdata = Security.SecCertificateCopyData(leaf) - assert certdata - - data_length = CoreFoundation.CFDataGetLength(certdata) - data_buffer = CoreFoundation.CFDataGetBytePtr(certdata) - der_bytes = ctypes.string_at(data_buffer, data_length) - finally: - if certdata: - CoreFoundation.CFRelease(certdata) - if trust: - CoreFoundation.CFRelease(trust) - - return der_bytes - - def version(self): - protocol = Security.SSLProtocol() - result = Security.SSLGetNegotiatedProtocolVersion( - self.context, ctypes.byref(protocol) - ) - _assert_no_error(result) - if protocol.value == SecurityConst.kTLSProtocol13: - raise ssl.SSLError("SecureTransport does not support TLS 1.3") - elif protocol.value == SecurityConst.kTLSProtocol12: - return "TLSv1.2" - elif protocol.value == SecurityConst.kTLSProtocol11: - return "TLSv1.1" - elif protocol.value == SecurityConst.kTLSProtocol1: - return "TLSv1" - elif protocol.value == SecurityConst.kSSLProtocol3: - return "SSLv3" - elif protocol.value == SecurityConst.kSSLProtocol2: - return "SSLv2" - else: - raise ssl.SSLError("Unknown TLS version: %r" % protocol) - - def _reuse(self): - self._makefile_refs += 1 - - def _drop(self): - if self._makefile_refs < 1: - self.close() - else: - self._makefile_refs -= 1 - - -if _fileobject: # Platform-specific: Python 2 - - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) - -else: # Platform-specific: Python 3 - - def makefile(self, mode="r", buffering=None, *args, **kwargs): - # We disable buffering with SecureTransport because it conflicts with - # the buffering that ST does internally (see issue #1153 for more). - buffering = 0 - return backport_makefile(self, mode, buffering, *args, **kwargs) - - -WrappedSocket.makefile = makefile - - -class SecureTransportContext(object): - """ - I am a wrapper class for the SecureTransport library, to translate the - interface of the standard library ``SSLContext`` object to calls into - SecureTransport. - """ - - def __init__(self, protocol): - self._min_version, self._max_version = _protocol_to_min_max[protocol] - self._options = 0 - self._verify = False - self._trust_bundle = None - self._client_cert = None - self._client_key = None - self._client_key_passphrase = None - self._alpn_protocols = None - - @property - def check_hostname(self): - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - return True - - @check_hostname.setter - def check_hostname(self, value): - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - pass - - @property - def options(self): - # TODO: Well, crap. - # - # So this is the bit of the code that is the most likely to cause us - # trouble. Essentially we need to enumerate all of the SSL options that - # users might want to use and try to see if we can sensibly translate - # them, or whether we should just ignore them. - return self._options - - @options.setter - def options(self, value): - # TODO: Update in line with above. - self._options = value - - @property - def verify_mode(self): - return ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE - - @verify_mode.setter - def verify_mode(self, value): - self._verify = True if value == ssl.CERT_REQUIRED else False - - def set_default_verify_paths(self): - # So, this has to do something a bit weird. Specifically, what it does - # is nothing. - # - # This means that, if we had previously had load_verify_locations - # called, this does not undo that. We need to do that because it turns - # out that the rest of the urllib3 code will attempt to load the - # default verify paths if it hasn't been told about any paths, even if - # the context itself was sometime earlier. We resolve that by just - # ignoring it. - pass - - def load_default_certs(self): - return self.set_default_verify_paths() - - def set_ciphers(self, ciphers): - # For now, we just require the default cipher string. - if ciphers != util.ssl_.DEFAULT_CIPHERS: - raise ValueError("SecureTransport doesn't support custom cipher strings") - - def load_verify_locations(self, cafile=None, capath=None, cadata=None): - # OK, we only really support cadata and cafile. - if capath is not None: - raise ValueError("SecureTransport does not support cert directories") - - # Raise if cafile does not exist. - if cafile is not None: - with open(cafile): - pass - - self._trust_bundle = cafile or cadata - - def load_cert_chain(self, certfile, keyfile=None, password=None): - self._client_cert = certfile - self._client_key = keyfile - self._client_cert_passphrase = password - - def set_alpn_protocols(self, protocols): - """ - Sets the ALPN protocols that will later be set on the context. - - Raises a NotImplementedError if ALPN is not supported. - """ - if not hasattr(Security, "SSLSetALPNProtocols"): - raise NotImplementedError( - "SecureTransport supports ALPN only in macOS 10.12+" - ) - self._alpn_protocols = [six.ensure_binary(p) for p in protocols] - - def wrap_socket( - self, - sock, - server_side=False, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - server_hostname=None, - ): - # So, what do we do here? Firstly, we assert some properties. This is a - # stripped down shim, so there is some functionality we don't support. - # See PEP 543 for the real deal. - assert not server_side - assert do_handshake_on_connect - assert suppress_ragged_eofs - - # Ok, we're good to go. Now we want to create the wrapped socket object - # and store it in the appropriate place. - wrapped_socket = WrappedSocket(sock) - - # Now we can handshake - wrapped_socket.handshake( - server_hostname, - self._verify, - self._trust_bundle, - self._min_version, - self._max_version, - self._client_cert, - self._client_key, - self._client_key_passphrase, - self._alpn_protocols, - ) - return wrapped_socket diff --git a/contrib/python/pip/pip/_vendor/urllib3/contrib/socks.py b/contrib/python/pip/pip/_vendor/urllib3/contrib/socks.py index c326e80dd11..e3239b569d9 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/contrib/socks.py +++ b/contrib/python/pip/pip/_vendor/urllib3/contrib/socks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This module contains provisional support for SOCKS proxies from within urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and @@ -38,10 +37,11 @@ with the proxy: proxy_url="socks5h://<username>:<password>@proxy-host" """ -from __future__ import absolute_import + +from __future__ import annotations try: - import socks + import socks # type: ignore[import-untyped] except ImportError: import warnings @@ -51,13 +51,13 @@ except ImportError: ( "SOCKS support in urllib3 requires the installation of optional " "dependencies: specifically, PySocks. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies" + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies" ), DependencyWarning, ) raise -from socket import error as SocketError +import typing from socket import timeout as SocketTimeout from ..connection import HTTPConnection, HTTPSConnection @@ -69,7 +69,16 @@ from ..util.url import parse_url try: import ssl except ImportError: - ssl = None + ssl = None # type: ignore[assignment] + + +class _TYPE_SOCKS_OPTIONS(typing.TypedDict): + socks_version: int + proxy_host: str | None + proxy_port: str | None + username: str | None + password: str | None + rdns: bool class SOCKSConnection(HTTPConnection): @@ -77,15 +86,20 @@ class SOCKSConnection(HTTPConnection): A plain-text HTTP connection that connects via a SOCKS proxy. """ - def __init__(self, *args, **kwargs): - self._socks_options = kwargs.pop("_socks_options") - super(SOCKSConnection, self).__init__(*args, **kwargs) + def __init__( + self, + _socks_options: _TYPE_SOCKS_OPTIONS, + *args: typing.Any, + **kwargs: typing.Any, + ) -> None: + self._socks_options = _socks_options + super().__init__(*args, **kwargs) - def _new_conn(self): + def _new_conn(self) -> socks.socksocket: """ Establish a new connection via the SOCKS proxy. """ - extra_kw = {} + extra_kw: dict[str, typing.Any] = {} if self.source_address: extra_kw["source_address"] = self.source_address @@ -102,15 +116,14 @@ class SOCKSConnection(HTTPConnection): proxy_password=self._socks_options["password"], proxy_rdns=self._socks_options["rdns"], timeout=self.timeout, - **extra_kw + **extra_kw, ) - except SocketTimeout: + except SocketTimeout as e: raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e except socks.ProxyError as e: # This is fragile as hell, but it seems to be the only way to raise @@ -120,22 +133,23 @@ class SOCKSConnection(HTTPConnection): if isinstance(error, SocketTimeout): raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e else: + # Adding `from e` messes with coverage somehow, so it's omitted. + # See #2386. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % error + self, f"Failed to establish a new connection: {error}" ) else: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e - ) + self, f"Failed to establish a new connection: {e}" + ) from e - except SocketError as e: # Defensive: PySocks should catch all these. + except OSError as e: # Defensive: PySocks should catch all these. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e - ) + self, f"Failed to establish a new connection: {e}" + ) from e return conn @@ -169,12 +183,12 @@ class SOCKSProxyManager(PoolManager): def __init__( self, - proxy_url, - username=None, - password=None, - num_pools=10, - headers=None, - **connection_pool_kw + proxy_url: str, + username: str | None = None, + password: str | None = None, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + **connection_pool_kw: typing.Any, ): parsed = parse_url(proxy_url) @@ -195,7 +209,7 @@ class SOCKSProxyManager(PoolManager): socks_version = socks.PROXY_TYPE_SOCKS4 rdns = True else: - raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) + raise ValueError(f"Unable to determine SOCKS version from {proxy_url}") self.proxy_url = proxy_url @@ -209,8 +223,6 @@ class SOCKSProxyManager(PoolManager): } connection_pool_kw["_socks_options"] = socks_options - super(SOCKSProxyManager, self).__init__( - num_pools, headers, **connection_pool_kw - ) + super().__init__(num_pools, headers, **connection_pool_kw) self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/contrib/python/pip/pip/_vendor/urllib3/exceptions.py b/contrib/python/pip/pip/_vendor/urllib3/exceptions.py index cba6f3f560f..58723faeb0c 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/exceptions.py +++ b/contrib/python/pip/pip/_vendor/urllib3/exceptions.py @@ -1,6 +1,16 @@ -from __future__ import absolute_import +from __future__ import annotations -from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead +import socket +import typing +import warnings +from email.errors import MessageDefect +from http.client import IncompleteRead as httplib_IncompleteRead + +if typing.TYPE_CHECKING: + from .connection import HTTPConnection + from .connectionpool import ConnectionPool + from .response import HTTPResponse + from .util.retry import Retry # Base Exceptions @@ -8,64 +18,61 @@ from .packages.six.moves.http_client import IncompleteRead as httplib_Incomplete class HTTPError(Exception): """Base exception used by this module.""" - pass - class HTTPWarning(Warning): """Base warning used by this module.""" - pass + +_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]] class PoolError(HTTPError): """Base exception for errors caused within a pool.""" - def __init__(self, pool, message): + def __init__(self, pool: ConnectionPool, message: str) -> None: self.pool = pool - HTTPError.__init__(self, "%s: %s" % (pool, message)) + self._message = message + super().__init__(f"{pool}: {message}") - def __reduce__(self): + def __reduce__(self) -> _TYPE_REDUCE_RESULT: # For pickling purposes. - return self.__class__, (None, None) + return self.__class__, (None, self._message) class RequestError(PoolError): """Base exception for PoolErrors that have associated URLs.""" - def __init__(self, pool, url, message): + def __init__(self, pool: ConnectionPool, url: str | None, message: str) -> None: self.url = url - PoolError.__init__(self, pool, message) + super().__init__(pool, message) - def __reduce__(self): + def __reduce__(self) -> _TYPE_REDUCE_RESULT: # For pickling purposes. - return self.__class__, (None, self.url, None) + return self.__class__, (None, self.url, self._message) class SSLError(HTTPError): """Raised when SSL certificate fails in an HTTPS connection.""" - pass - class ProxyError(HTTPError): """Raised when the connection to a proxy fails.""" - def __init__(self, message, error, *args): - super(ProxyError, self).__init__(message, error, *args) + # The original error is also available as __cause__. + original_error: Exception + + def __init__(self, message: str, error: Exception) -> None: + super().__init__(message, error) self.original_error = error class DecodeError(HTTPError): """Raised when automatic decoding based on Content-Type fails.""" - pass - class ProtocolError(HTTPError): """Raised when something unexpected happens mid-request/response.""" - pass - #: Renamed to ProtocolError but aliased for backwards compatibility. ConnectionError = ProtocolError @@ -79,33 +86,40 @@ class MaxRetryError(RequestError): :param pool: The connection pool :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` - :param string url: The requested Url - :param exceptions.Exception reason: The underlying error + :param str url: The requested Url + :param reason: The underlying error + :type reason: :class:`Exception` """ - def __init__(self, pool, url, reason=None): + def __init__( + self, pool: ConnectionPool, url: str | None, reason: Exception | None = None + ) -> None: self.reason = reason - message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) + message = f"Max retries exceeded with url: {url} (Caused by {reason!r})" + + super().__init__(pool, url, message) - RequestError.__init__(self, pool, url, message) + def __reduce__(self) -> _TYPE_REDUCE_RESULT: + # For pickling purposes. + return self.__class__, (None, self.url, self.reason) class HostChangedError(RequestError): """Raised when an existing pool gets a request for a foreign host.""" - def __init__(self, pool, url, retries=3): - message = "Tried to open a foreign host with url: %s" % url - RequestError.__init__(self, pool, url, message) + def __init__( + self, pool: ConnectionPool, url: str, retries: Retry | int = 3 + ) -> None: + message = f"Tried to open a foreign host with url: {url}" + super().__init__(pool, url, message) self.retries = retries class TimeoutStateError(HTTPError): """Raised when passing an invalid state to a timeout""" - pass - class TimeoutError(HTTPError): """Raised when a socket timeout error occurs. @@ -114,53 +128,77 @@ class TimeoutError(HTTPError): <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`. """ - pass - class ReadTimeoutError(TimeoutError, RequestError): """Raised when a socket timeout occurs while receiving data from a server""" - pass - # This timeout error does not have a URL attached and needs to inherit from the # base HTTPError class ConnectTimeoutError(TimeoutError): """Raised when a socket timeout occurs while connecting to a server""" - pass - -class NewConnectionError(ConnectTimeoutError, PoolError): +class NewConnectionError(ConnectTimeoutError, HTTPError): """Raised when we fail to establish a new connection. Usually ECONNREFUSED.""" - pass + def __init__(self, conn: HTTPConnection, message: str) -> None: + self.conn = conn + self._message = message + super().__init__(f"{conn}: {message}") + + def __reduce__(self) -> _TYPE_REDUCE_RESULT: + # For pickling purposes. + return self.__class__, (None, self._message) + + @property + def pool(self) -> HTTPConnection: + warnings.warn( + "The 'pool' property is deprecated and will be removed " + "in urllib3 v2.1.0. Use 'conn' instead.", + DeprecationWarning, + stacklevel=2, + ) + + return self.conn + + +class NameResolutionError(NewConnectionError): + """Raised when host name resolution fails.""" + + def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror): + message = f"Failed to resolve '{host}' ({reason})" + self._host = host + self._reason = reason + super().__init__(conn, message) + + def __reduce__(self) -> _TYPE_REDUCE_RESULT: + # For pickling purposes. + return self.__class__, (self._host, None, self._reason) class EmptyPoolError(PoolError): """Raised when a pool runs out of connections and no more are allowed.""" - pass + +class FullPoolError(PoolError): + """Raised when we try to add a connection to a full pool in blocking mode.""" class ClosedPoolError(PoolError): """Raised when a request enters a pool after the pool has been closed.""" - pass - class LocationValueError(ValueError, HTTPError): """Raised when there is something wrong with a given URL input.""" - pass - class LocationParseError(LocationValueError): """Raised when get_host or similar fails to parse the URL input.""" - def __init__(self, location): - message = "Failed to parse: %s" % location - HTTPError.__init__(self, message) + def __init__(self, location: str) -> None: + message = f"Failed to parse: {location}" + super().__init__(message) self.location = location @@ -168,9 +206,9 @@ class LocationParseError(LocationValueError): class URLSchemeUnknown(LocationValueError): """Raised when a URL input has an unsupported scheme.""" - def __init__(self, scheme): - message = "Not supported URL scheme %s" % scheme - super(URLSchemeUnknown, self).__init__(message) + def __init__(self, scheme: str): + message = f"Not supported URL scheme {scheme}" + super().__init__(message) self.scheme = scheme @@ -185,38 +223,22 @@ class ResponseError(HTTPError): class SecurityWarning(HTTPWarning): """Warned when performing security reducing actions""" - pass - - -class SubjectAltNameWarning(SecurityWarning): - """Warned when connecting to a host with a certificate missing a SAN.""" - - pass - class InsecureRequestWarning(SecurityWarning): """Warned when making an unverified HTTPS request.""" - pass + +class NotOpenSSLWarning(SecurityWarning): + """Warned when using unsupported SSL library""" class SystemTimeWarning(SecurityWarning): """Warned when system time is suspected to be wrong""" - pass - class InsecurePlatformWarning(SecurityWarning): """Warned when certain TLS/SSL configuration is not available on a platform.""" - pass - - -class SNIMissingWarning(HTTPWarning): - """Warned when making a HTTPS request without SNI available.""" - - pass - class DependencyWarning(HTTPWarning): """ @@ -224,14 +246,10 @@ class DependencyWarning(HTTPWarning): dependencies. """ - pass - class ResponseNotChunked(ProtocolError, ValueError): """Response needs to be chunked in order to read it as chunks.""" - pass - class BodyNotHttplibCompatible(HTTPError): """ @@ -239,8 +257,6 @@ class BodyNotHttplibCompatible(HTTPError): (have an fp attribute which returns raw chunks) for read_chunked(). """ - pass - class IncompleteRead(HTTPError, httplib_IncompleteRead): """ @@ -250,10 +266,14 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for ``partial`` to avoid creating large objects on streamed reads. """ - def __init__(self, partial, expected): - super(IncompleteRead, self).__init__(partial, expected) + partial: int # type: ignore[assignment] + expected: int - def __repr__(self): + def __init__(self, partial: int, expected: int) -> None: + self.partial = partial + self.expected = expected + + def __repr__(self) -> str: return "IncompleteRead(%i bytes read, %i more expected)" % ( self.partial, self.expected, @@ -263,14 +283,13 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): class InvalidChunkLength(HTTPError, httplib_IncompleteRead): """Invalid chunk length in a chunked response.""" - def __init__(self, response, length): - super(InvalidChunkLength, self).__init__( - response.tell(), response.length_remaining - ) + def __init__(self, response: HTTPResponse, length: bytes) -> None: + self.partial: int = response.tell() # type: ignore[assignment] + self.expected: int | None = response.length_remaining self.response = response self.length = length - def __repr__(self): + def __repr__(self) -> str: return "InvalidChunkLength(got length %r, %i bytes read)" % ( self.length, self.partial, @@ -280,15 +299,13 @@ class InvalidChunkLength(HTTPError, httplib_IncompleteRead): class InvalidHeader(HTTPError): """The header provided was somehow invalid.""" - pass - class ProxySchemeUnknown(AssertionError, URLSchemeUnknown): """ProxyManager does not support the supplied scheme""" # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. - def __init__(self, scheme): + def __init__(self, scheme: str | None) -> None: # 'localhost' is here because our URL parser parses # localhost:8080 -> scheme=localhost, remove if we fix this. if scheme == "localhost": @@ -296,28 +313,23 @@ class ProxySchemeUnknown(AssertionError, URLSchemeUnknown): if scheme is None: message = "Proxy URL had no scheme, should start with http:// or https://" else: - message = ( - "Proxy URL had unsupported scheme %s, should use http:// or https://" - % scheme - ) - super(ProxySchemeUnknown, self).__init__(message) + message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://" + super().__init__(message) class ProxySchemeUnsupported(ValueError): """Fetching HTTPS resources through HTTPS proxies is unsupported""" - pass - class HeaderParsingError(HTTPError): """Raised by assert_header_parsing, but we convert it to a log.warning statement.""" - def __init__(self, defects, unparsed_data): - message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) - super(HeaderParsingError, self).__init__(message) + def __init__( + self, defects: list[MessageDefect], unparsed_data: bytes | str | None + ) -> None: + message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}" + super().__init__(message) class UnrewindableBodyError(HTTPError): """urllib3 encountered an error when trying to rewind a body""" - - pass diff --git a/contrib/python/pip/pip/_vendor/urllib3/fields.py b/contrib/python/pip/pip/_vendor/urllib3/fields.py index 9d630f491d9..97c4730cff0 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/fields.py +++ b/contrib/python/pip/pip/_vendor/urllib3/fields.py @@ -1,13 +1,20 @@ -from __future__ import absolute_import +from __future__ import annotations import email.utils import mimetypes -import re +import typing -from .packages import six +_TYPE_FIELD_VALUE = typing.Union[str, bytes] +_TYPE_FIELD_VALUE_TUPLE = typing.Union[ + _TYPE_FIELD_VALUE, + tuple[str, _TYPE_FIELD_VALUE], + tuple[str, _TYPE_FIELD_VALUE, str], +] -def guess_content_type(filename, default="application/octet-stream"): +def guess_content_type( + filename: str | None, default: str = "application/octet-stream" +) -> str: """ Guess the "Content-Type" of a file. @@ -21,7 +28,7 @@ def guess_content_type(filename, default="application/octet-stream"): return default -def format_header_param_rfc2231(name, value): +def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str: """ Helper function to format and quote a single header parameter using the strategy defined in RFC 2231. @@ -34,14 +41,28 @@ def format_header_param_rfc2231(name, value): The name of the parameter, a string expected to be ASCII only. :param value: The value of the parameter, provided as ``bytes`` or `str``. - :ret: + :returns: An RFC-2231-formatted unicode string. + + .. deprecated:: 2.0.0 + Will be removed in urllib3 v2.1.0. This is not valid for + ``multipart/form-data`` header parameters. """ - if isinstance(value, six.binary_type): + import warnings + + warnings.warn( + "'format_header_param_rfc2231' is deprecated and will be " + "removed in urllib3 v2.1.0. This is not valid for " + "multipart/form-data header parameters.", + DeprecationWarning, + stacklevel=2, + ) + + if isinstance(value, bytes): value = value.decode("utf-8") if not any(ch in value for ch in '"\\\r\n'): - result = u'%s="%s"' % (name, value) + result = f'{name}="{value}"' try: result.encode("ascii") except (UnicodeEncodeError, UnicodeDecodeError): @@ -49,81 +70,87 @@ def format_header_param_rfc2231(name, value): else: return result - if six.PY2: # Python 2: - value = value.encode("utf-8") - - # encode_rfc2231 accepts an encoded string and returns an ascii-encoded - # string in Python 2 but accepts and returns unicode strings in Python 3 value = email.utils.encode_rfc2231(value, "utf-8") - value = "%s*=%s" % (name, value) - - if six.PY2: # Python 2: - value = value.decode("utf-8") + value = f"{name}*={value}" return value -_HTML5_REPLACEMENTS = { - u"\u0022": u"%22", - # Replace "\" with "\\". - u"\u005C": u"\u005C\u005C", -} - -# All control characters from 0x00 to 0x1F *except* 0x1B. -_HTML5_REPLACEMENTS.update( - { - six.unichr(cc): u"%{:02X}".format(cc) - for cc in range(0x00, 0x1F + 1) - if cc not in (0x1B,) - } -) - - -def _replace_multiple(value, needles_and_replacements): - def replacer(match): - return needles_and_replacements[match.group(0)] +def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str: + """ + Format and quote a single multipart header parameter. - pattern = re.compile( - r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) - ) + This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching + the behavior of current browser and curl versions. Values are + assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are + percent encoded. - result = pattern.sub(replacer, value) + .. _WHATWG HTML Standard: + https://html.spec.whatwg.org/multipage/ + form-control-infrastructure.html#multipart-form-data - return result + :param name: + The name of the parameter, an ASCII-only ``str``. + :param value: + The value of the parameter, a ``str`` or UTF-8 encoded + ``bytes``. + :returns: + A string ``name="value"`` with the escaped value. + .. versionchanged:: 2.0.0 + Matches the WHATWG HTML Standard as of 2021/06/10. Control + characters are no longer percent encoded. -def format_header_param_html5(name, value): + .. versionchanged:: 2.0.0 + Renamed from ``format_header_param_html5`` and + ``format_header_param``. The old names will be removed in + urllib3 v2.1.0. """ - Helper function to format and quote a single header parameter using the - HTML5 strategy. + if isinstance(value, bytes): + value = value.decode("utf-8") - Particularly useful for header parameters which might contain - non-ASCII values, like file names. This follows the `HTML5 Working Draft - Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. + # percent encode \n \r " + value = value.translate({10: "%0A", 13: "%0D", 34: "%22"}) + return f'{name}="{value}"' - .. _HTML5 Working Draft Section 4.10.22.7: - https://w3c.github.io/html/sec-forms.html#multipart-form-data - :param name: - The name of the parameter, a string expected to be ASCII only. - :param value: - The value of the parameter, provided as ``bytes`` or `str``. - :ret: - A unicode string, stripped of troublesome characters. +def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str: """ - if isinstance(value, six.binary_type): - value = value.decode("utf-8") + .. deprecated:: 2.0.0 + Renamed to :func:`format_multipart_header_param`. Will be + removed in urllib3 v2.1.0. + """ + import warnings - value = _replace_multiple(value, _HTML5_REPLACEMENTS) + warnings.warn( + "'format_header_param_html5' has been renamed to " + "'format_multipart_header_param'. The old name will be " + "removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + return format_multipart_header_param(name, value) - return u'%s="%s"' % (name, value) +def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str: + """ + .. deprecated:: 2.0.0 + Renamed to :func:`format_multipart_header_param`. Will be + removed in urllib3 v2.1.0. + """ + import warnings -# For backwards-compatibility. -format_header_param = format_header_param_html5 + warnings.warn( + "'format_header_param' has been renamed to " + "'format_multipart_header_param'. The old name will be " + "removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + return format_multipart_header_param(name, value) -class RequestField(object): +class RequestField: """ A data container for request body parameters. @@ -135,29 +162,47 @@ class RequestField(object): An optional filename of the request field. Must be unicode. :param headers: An optional dict-like object of headers to initially use for the field. - :param header_formatter: - An optional callable that is used to encode and format the headers. By - default, this is :func:`format_header_param_html5`. + + .. versionchanged:: 2.0.0 + The ``header_formatter`` parameter is deprecated and will + be removed in urllib3 v2.1.0. """ def __init__( self, - name, - data, - filename=None, - headers=None, - header_formatter=format_header_param_html5, + name: str, + data: _TYPE_FIELD_VALUE, + filename: str | None = None, + headers: typing.Mapping[str, str] | None = None, + header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None, ): self._name = name self._filename = filename self.data = data - self.headers = {} + self.headers: dict[str, str | None] = {} if headers: self.headers = dict(headers) - self.header_formatter = header_formatter + + if header_formatter is not None: + import warnings + + warnings.warn( + "The 'header_formatter' parameter is deprecated and " + "will be removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + self.header_formatter = header_formatter + else: + self.header_formatter = format_multipart_header_param @classmethod - def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): + def from_tuples( + cls, + fieldname: str, + value: _TYPE_FIELD_VALUE_TUPLE, + header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None, + ) -> RequestField: """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -174,6 +219,10 @@ class RequestField(object): Field names and filenames must be unicode. """ + filename: str | None + content_type: str | None + data: _TYPE_FIELD_VALUE + if isinstance(value, tuple): if len(value) == 3: filename, data, content_type = value @@ -192,20 +241,29 @@ class RequestField(object): return request_param - def _render_part(self, name, value): + def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str: """ - Overridable helper function to format a single header parameter. By - default, this calls ``self.header_formatter``. + Override this method to change how each multipart header + parameter is formatted. By default, this calls + :func:`format_multipart_header_param`. :param name: - The name of the parameter, a string expected to be ASCII only. + The name of the parameter, an ASCII-only ``str``. :param value: - The value of the parameter, provided as a unicode string. - """ + The value of the parameter, a ``str`` or UTF-8 encoded + ``bytes``. + :meta public: + """ return self.header_formatter(name, value) - def _render_parts(self, header_parts): + def _render_parts( + self, + header_parts: ( + dict[str, _TYPE_FIELD_VALUE | None] + | typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]] + ), + ) -> str: """ Helper function to format and quote a single header. @@ -216,18 +274,21 @@ class RequestField(object): A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format as `k1="v1"; k2="v2"; ...`. """ + iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]] + parts = [] - iterable = header_parts if isinstance(header_parts, dict): iterable = header_parts.items() + else: + iterable = header_parts for name, value in iterable: if value is not None: parts.append(self._render_part(name, value)) - return u"; ".join(parts) + return "; ".join(parts) - def render_headers(self): + def render_headers(self) -> str: """ Renders the headers for this request field. """ @@ -236,39 +297,45 @@ class RequestField(object): sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) + lines.append(f"{sort_key}: {self.headers[sort_key]}") for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append(u"%s: %s" % (header_name, header_value)) + lines.append(f"{header_name}: {header_value}") - lines.append(u"\r\n") - return u"\r\n".join(lines) + lines.append("\r\n") + return "\r\n".join(lines) def make_multipart( - self, content_disposition=None, content_type=None, content_location=None - ): + self, + content_disposition: str | None = None, + content_type: str | None = None, + content_location: str | None = None, + ) -> None: """ Makes this request field into a multipart request field. This method overrides "Content-Disposition", "Content-Type" and "Content-Location" headers to the request parameter. + :param content_disposition: + The 'Content-Disposition' of the request body. Defaults to 'form-data' :param content_type: The 'Content-Type' of the request body. :param content_location: The 'Content-Location' of the request body. """ - self.headers["Content-Disposition"] = content_disposition or u"form-data" - self.headers["Content-Disposition"] += u"; ".join( + content_disposition = (content_disposition or "form-data") + "; ".join( [ - u"", + "", self._render_parts( - ((u"name", self._name), (u"filename", self._filename)) + (("name", self._name), ("filename", self._filename)) ), ] ) + + self.headers["Content-Disposition"] = content_disposition self.headers["Content-Type"] = content_type self.headers["Content-Location"] = content_location diff --git a/contrib/python/pip/pip/_vendor/urllib3/filepost.py b/contrib/python/pip/pip/_vendor/urllib3/filepost.py index 36c9252c647..14f70b05b47 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/filepost.py +++ b/contrib/python/pip/pip/_vendor/urllib3/filepost.py @@ -1,28 +1,32 @@ -from __future__ import absolute_import +from __future__ import annotations import binascii import codecs import os +import typing from io import BytesIO -from .fields import RequestField -from .packages import six -from .packages.six import b +from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField writer = codecs.lookup("utf-8")[3] +_TYPE_FIELDS_SEQUENCE = typing.Sequence[ + typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField] +] +_TYPE_FIELDS = typing.Union[ + _TYPE_FIELDS_SEQUENCE, + typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE], +] -def choose_boundary(): + +def choose_boundary() -> str: """ Our embarrassingly-simple replacement for mimetools.choose_boundary. """ - boundary = binascii.hexlify(os.urandom(16)) - if not six.PY2: - boundary = boundary.decode("ascii") - return boundary + return binascii.hexlify(os.urandom(16)).decode() -def iter_field_objects(fields): +def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]: """ Iterate over fields. @@ -30,42 +34,29 @@ def iter_field_objects(fields): :class:`~urllib3.fields.RequestField`. """ - if isinstance(fields, dict): - i = six.iteritems(fields) + iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]] + + if isinstance(fields, typing.Mapping): + iterable = fields.items() else: - i = iter(fields) + iterable = fields - for field in i: + for field in iterable: if isinstance(field, RequestField): yield field else: yield RequestField.from_tuples(*field) -def iter_fields(fields): - """ - .. deprecated:: 1.6 - - Iterate over fields. - - The addition of :class:`~urllib3.fields.RequestField` makes this function - obsolete. Instead, use :func:`iter_field_objects`, which returns - :class:`~urllib3.fields.RequestField` objects. - - Supports list of (k, v) tuples and dicts. - """ - if isinstance(fields, dict): - return ((k, v) for k, v in six.iteritems(fields)) - - return ((k, v) for k, v in fields) - - -def encode_multipart_formdata(fields, boundary=None): +def encode_multipart_formdata( + fields: _TYPE_FIELDS, boundary: str | None = None +) -> tuple[bytes, str]: """ Encode a dictionary of ``fields`` using the multipart/form-data MIME format. :param fields: Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). + Values are processed by :func:`urllib3.fields.RequestField.from_tuples`. :param boundary: If not specified, then a random boundary will be generated using @@ -76,7 +67,7 @@ def encode_multipart_formdata(fields, boundary=None): boundary = choose_boundary() for field in iter_field_objects(fields): - body.write(b("--%s\r\n" % (boundary))) + body.write(f"--{boundary}\r\n".encode("latin-1")) writer(body).write(field.render_headers()) data = field.data @@ -84,15 +75,15 @@ def encode_multipart_formdata(fields, boundary=None): if isinstance(data, int): data = str(data) # Backwards compatibility - if isinstance(data, six.text_type): + if isinstance(data, str): writer(body).write(data) else: body.write(data) body.write(b"\r\n") - body.write(b("--%s--\r\n" % (boundary))) + body.write(f"--{boundary}--\r\n".encode("latin-1")) - content_type = str("multipart/form-data; boundary=%s" % boundary) + content_type = f"multipart/form-data; boundary={boundary}" return body.getvalue(), content_type diff --git a/contrib/python/pip/pip/_vendor/urllib3/http2/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/http2/__init__.py new file mode 100644 index 00000000000..133e1d8f237 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/http2/__init__.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from importlib.metadata import version + +__all__ = [ + "inject_into_urllib3", + "extract_from_urllib3", +] + +import typing + +orig_HTTPSConnection: typing.Any = None + + +def inject_into_urllib3() -> None: + # First check if h2 version is valid + h2_version = version("h2") + if not h2_version.startswith("4."): + raise ImportError( + "urllib3 v2 supports h2 version 4.x.x, currently " + f"the 'h2' module is compiled with {h2_version!r}. " + "See: https://github.com/urllib3/urllib3/issues/3290" + ) + + # Import here to avoid circular dependencies. + from .. import connection as urllib3_connection + from .. import util as urllib3_util + from ..connectionpool import HTTPSConnectionPool + from ..util import ssl_ as urllib3_util_ssl + from .connection import HTTP2Connection + + global orig_HTTPSConnection + orig_HTTPSConnection = urllib3_connection.HTTPSConnection + + HTTPSConnectionPool.ConnectionCls = HTTP2Connection + urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc] + + # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy. + urllib3_util.ALPN_PROTOCOLS = ["h2"] + urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"] + + +def extract_from_urllib3() -> None: + from .. import connection as urllib3_connection + from .. import util as urllib3_util + from ..connectionpool import HTTPSConnectionPool + from ..util import ssl_ as urllib3_util_ssl + + HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection + urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc] + + urllib3_util.ALPN_PROTOCOLS = ["http/1.1"] + urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"] diff --git a/contrib/python/pip/pip/_vendor/urllib3/http2/connection.py b/contrib/python/pip/pip/_vendor/urllib3/http2/connection.py new file mode 100644 index 00000000000..0a026da0a83 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/http2/connection.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import logging +import re +import threading +import types +import typing + +import h2.config +import h2.connection +import h2.events + +from .._base_connection import _TYPE_BODY +from .._collections import HTTPHeaderDict +from ..connection import HTTPSConnection, _get_default_user_agent +from ..exceptions import ConnectionError +from ..response import BaseHTTPResponse + +orig_HTTPSConnection = HTTPSConnection + +T = typing.TypeVar("T") + +log = logging.getLogger(__name__) + +RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$") +RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$") + + +def _is_legal_header_name(name: bytes) -> bool: + """ + "An implementation that validates fields according to the definitions in Sections + 5.1 and 5.5 of [HTTP] only needs an additional check that field names do not + include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity) + + `http.client._is_legal_header_name` does not validate the field name according to the + HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters. + + This does not allow for the `:` character in the header name, so should not + be used to validate pseudo-headers. + """ + return bool(RE_IS_LEGAL_HEADER_NAME.match(name)) + + +def _is_illegal_header_value(value: bytes) -> bool: + """ + "A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed + (ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field + value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB, + 0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity) + """ + return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value)) + + +class _LockedObject(typing.Generic[T]): + """ + A wrapper class that hides a specific object behind a lock. + The goal here is to provide a simple way to protect access to an object + that cannot safely be simultaneously accessed from multiple threads. The + intended use of this class is simple: take hold of it with a context + manager, which returns the protected object. + """ + + __slots__ = ( + "lock", + "_obj", + ) + + def __init__(self, obj: T): + self.lock = threading.RLock() + self._obj = obj + + def __enter__(self) -> T: + self.lock.acquire() + return self._obj + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + self.lock.release() + + +class HTTP2Connection(HTTPSConnection): + def __init__( + self, host: str, port: int | None = None, **kwargs: typing.Any + ) -> None: + self._h2_conn = self._new_h2_conn() + self._h2_stream: int | None = None + self._headers: list[tuple[bytes, bytes]] = [] + + if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive: + raise NotImplementedError("Proxies aren't supported with HTTP/2") + + super().__init__(host, port, **kwargs) + + if self._tunnel_host is not None: + raise NotImplementedError("Tunneling isn't supported with HTTP/2") + + def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]: + config = h2.config.H2Configuration(client_side=True) + return _LockedObject(h2.connection.H2Connection(config=config)) + + def connect(self) -> None: + super().connect() + with self._h2_conn as conn: + conn.initiate_connection() + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + def putrequest( # type: ignore[override] + self, + method: str, + url: str, + **kwargs: typing.Any, + ) -> None: + """putrequest + This deviates from the HTTPConnection method signature since we never need to override + sending accept-encoding headers or the host header. + """ + if "skip_host" in kwargs: + raise NotImplementedError("`skip_host` isn't supported") + if "skip_accept_encoding" in kwargs: + raise NotImplementedError("`skip_accept_encoding` isn't supported") + + self._request_url = url or "/" + self._validate_path(url) # type: ignore[attr-defined] + + if ":" in self.host: + authority = f"[{self.host}]:{self.port or 443}" + else: + authority = f"{self.host}:{self.port or 443}" + + self._headers.append((b":scheme", b"https")) + self._headers.append((b":method", method.encode())) + self._headers.append((b":authority", authority.encode())) + self._headers.append((b":path", url.encode())) + + with self._h2_conn as conn: + self._h2_stream = conn.get_next_available_stream_id() + + def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override] + # TODO SKIPPABLE_HEADERS from urllib3 are ignored. + header = header.encode() if isinstance(header, str) else header + header = header.lower() # A lot of upstream code uses capitalized headers. + if not _is_legal_header_name(header): + raise ValueError(f"Illegal header name {str(header)}") + + for value in values: + value = value.encode() if isinstance(value, str) else value + if _is_illegal_header_value(value): + raise ValueError(f"Illegal header value {str(value)}") + self._headers.append((header, value)) + + def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override] + if self._h2_stream is None: + raise ConnectionError("Must call `putrequest` first.") + + with self._h2_conn as conn: + conn.send_headers( + stream_id=self._h2_stream, + headers=self._headers, + end_stream=(message_body is None), + ) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + self._headers = [] # Reset headers for the next request. + + def send(self, data: typing.Any) -> None: + """Send data to the server. + `data` can be: `str`, `bytes`, an iterable, or file-like objects + that support a .read() method. + """ + if self._h2_stream is None: + raise ConnectionError("Must call `putrequest` first.") + + with self._h2_conn as conn: + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + if hasattr(data, "read"): # file-like objects + while True: + chunk = data.read(self.blocksize) + if not chunk: + break + if isinstance(chunk, str): + chunk = chunk.encode() + conn.send_data(self._h2_stream, chunk, end_stream=False) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + conn.end_stream(self._h2_stream) + return + + if isinstance(data, str): # str -> bytes + data = data.encode() + + try: + if isinstance(data, bytes): + conn.send_data(self._h2_stream, data, end_stream=True) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + else: + for chunk in data: + conn.send_data(self._h2_stream, chunk, end_stream=False) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + conn.end_stream(self._h2_stream) + except TypeError: + raise TypeError( + "`data` should be str, bytes, iterable, or file. got %r" + % type(data) + ) + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + raise NotImplementedError( + "HTTP/2 does not support setting up a tunnel through a proxy" + ) + + def getresponse( # type: ignore[override] + self, + ) -> HTTP2Response: + status = None + data = bytearray() + with self._h2_conn as conn: + end_stream = False + while not end_stream: + # TODO: Arbitrary read value. + if received_data := self.sock.recv(65535): + events = conn.receive_data(received_data) + for event in events: + if isinstance(event, h2.events.ResponseReceived): + headers = HTTPHeaderDict() + for header, value in event.headers: + if header == b":status": + status = int(value.decode()) + else: + headers.add( + header.decode("ascii"), value.decode("ascii") + ) + + elif isinstance(event, h2.events.DataReceived): + data += event.data + conn.acknowledge_received_data( + event.flow_controlled_length, event.stream_id + ) + + elif isinstance(event, h2.events.StreamEnded): + end_stream = True + + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + assert status is not None + return HTTP2Response( + status=status, + headers=headers, + request_url=self._request_url, + data=bytes(data), + ) + + def request( # type: ignore[override] + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + *, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + **kwargs: typing.Any, + ) -> None: + """Send an HTTP/2 request""" + if "chunked" in kwargs: + # TODO this is often present from upstream. + # raise NotImplementedError("`chunked` isn't supported with HTTP/2") + pass + + if self.sock is not None: + self.sock.settimeout(self.timeout) + + self.putrequest(method, url) + + headers = headers or {} + for k, v in headers.items(): + if k.lower() == "transfer-encoding" and v == "chunked": + continue + else: + self.putheader(k, v) + + if b"user-agent" not in dict(self._headers): + self.putheader(b"user-agent", _get_default_user_agent()) + + if body: + self.endheaders(message_body=body) + self.send(body) + else: + self.endheaders() + + def close(self) -> None: + with self._h2_conn as conn: + try: + conn.close_connection() + if data := conn.data_to_send(): + self.sock.sendall(data) + except Exception: + pass + + # Reset all our HTTP/2 connection state. + self._h2_conn = self._new_h2_conn() + self._h2_stream = None + self._headers = [] + + super().close() + + +class HTTP2Response(BaseHTTPResponse): + # TODO: This is a woefully incomplete response object, but works for non-streaming. + def __init__( + self, + status: int, + headers: HTTPHeaderDict, + request_url: str, + data: bytes, + decode_content: bool = False, # TODO: support decoding + ) -> None: + super().__init__( + status=status, + headers=headers, + # Following CPython, we map HTTP versions to major * 10 + minor integers + version=20, + version_string="HTTP/2", + # No reason phrase in HTTP/2 + reason=None, + decode_content=decode_content, + request_url=request_url, + ) + self._data = data + self.length_remaining = 0 + + @property + def data(self) -> bytes: + return self._data + + def get_redirect_location(self) -> None: + return None + + def close(self) -> None: + pass diff --git a/contrib/python/pip/pip/_vendor/urllib3/http2/probe.py b/contrib/python/pip/pip/_vendor/urllib3/http2/probe.py new file mode 100644 index 00000000000..9ea900764f0 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/http2/probe.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import threading + + +class _HTTP2ProbeCache: + __slots__ = ( + "_lock", + "_cache_locks", + "_cache_values", + ) + + def __init__(self) -> None: + self._lock = threading.Lock() + self._cache_locks: dict[tuple[str, int], threading.RLock] = {} + self._cache_values: dict[tuple[str, int], bool | None] = {} + + def acquire_and_get(self, host: str, port: int) -> bool | None: + # By the end of this block we know that + # _cache_[values,locks] is available. + value = None + with self._lock: + key = (host, port) + try: + value = self._cache_values[key] + # If it's a known value we return right away. + if value is not None: + return value + except KeyError: + self._cache_locks[key] = threading.RLock() + self._cache_values[key] = None + + # If the value is unknown, we acquire the lock to signal + # to the requesting thread that the probe is in progress + # or that the current thread needs to return their findings. + key_lock = self._cache_locks[key] + key_lock.acquire() + try: + # If the by the time we get the lock the value has been + # updated we want to return the updated value. + value = self._cache_values[key] + + # In case an exception like KeyboardInterrupt is raised here. + except BaseException as e: # Defensive: + assert not isinstance(e, KeyError) # KeyError shouldn't be possible. + key_lock.release() + raise + + return value + + def set_and_release( + self, host: str, port: int, supports_http2: bool | None + ) -> None: + key = (host, port) + key_lock = self._cache_locks[key] + with key_lock: # Uses an RLock, so can be locked again from same thread. + if supports_http2 is None and self._cache_values[key] is not None: + raise ValueError( + "Cannot reset HTTP/2 support for origin after value has been set." + ) # Defensive: not expected in normal usage + + self._cache_values[key] = supports_http2 + key_lock.release() + + def _values(self) -> dict[tuple[str, int], bool | None]: + """This function is for testing purposes only. Gets the current state of the probe cache""" + with self._lock: + return {k: v for k, v in self._cache_values.items()} + + def _reset(self) -> None: + """This function is for testing purposes only. Reset the cache values""" + with self._lock: + self._cache_locks = {} + self._cache_values = {} + + +_HTTP2_PROBE_CACHE = _HTTP2ProbeCache() + +set_and_release = _HTTP2_PROBE_CACHE.set_and_release +acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get +_values = _HTTP2_PROBE_CACHE._values +_reset = _HTTP2_PROBE_CACHE._reset + +__all__ = [ + "set_and_release", + "acquire_and_get", +] diff --git a/contrib/python/pip/pip/_vendor/urllib3/packages/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/packages/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/packages/__init__.py +++ /dev/null diff --git a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/packages/backports/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/__init__.py +++ /dev/null diff --git a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/makefile.py b/contrib/python/pip/pip/_vendor/urllib3/packages/backports/makefile.py deleted file mode 100644 index b8fb2154b6d..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/makefile.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -""" -backports.makefile -~~~~~~~~~~~~~~~~~~ - -Backports the Python 3 ``socket.makefile`` method for use with anything that -wants to create a "fake" socket object. -""" -import io -from socket import SocketIO - - -def backport_makefile( - self, mode="r", buffering=None, encoding=None, errors=None, newline=None -): - """ - Backport of ``socket.makefile`` from Python 3.5. - """ - if not set(mode) <= {"r", "w", "b"}: - raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) - writing = "w" in mode - reading = "r" in mode or not writing - assert reading or writing - binary = "b" in mode - rawmode = "" - if reading: - rawmode += "r" - if writing: - rawmode += "w" - raw = SocketIO(self, rawmode) - self._makefile_refs += 1 - if buffering is None: - buffering = -1 - if buffering < 0: - buffering = io.DEFAULT_BUFFER_SIZE - if buffering == 0: - if not binary: - raise ValueError("unbuffered streams must be binary") - return raw - if reading and writing: - buffer = io.BufferedRWPair(raw, raw, buffering) - elif reading: - buffer = io.BufferedReader(raw, buffering) - else: - assert writing - buffer = io.BufferedWriter(raw, buffering) - if binary: - return buffer - text = io.TextIOWrapper(buffer, encoding, errors, newline) - text.mode = mode - return text diff --git a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/weakref_finalize.py b/contrib/python/pip/pip/_vendor/urllib3/packages/backports/weakref_finalize.py deleted file mode 100644 index a2f2966e549..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/packages/backports/weakref_finalize.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -""" -backports.weakref_finalize -~~~~~~~~~~~~~~~~~~ - -Backports the Python 3 ``weakref.finalize`` method. -""" -from __future__ import absolute_import - -import itertools -import sys -from weakref import ref - -__all__ = ["weakref_finalize"] - - -class weakref_finalize(object): - """Class for finalization of weakrefable objects - finalize(obj, func, *args, **kwargs) returns a callable finalizer - object which will be called when obj is garbage collected. The - first time the finalizer is called it evaluates func(*arg, **kwargs) - and returns the result. After this the finalizer is dead, and - calling it just returns None. - When the program exits any remaining finalizers for which the - atexit attribute is true will be run in reverse order of creation. - By default atexit is true. - """ - - # Finalizer objects don't have any state of their own. They are - # just used as keys to lookup _Info objects in the registry. This - # ensures that they cannot be part of a ref-cycle. - - __slots__ = () - _registry = {} - _shutdown = False - _index_iter = itertools.count() - _dirty = False - _registered_with_atexit = False - - class _Info(object): - __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") - - def __init__(self, obj, func, *args, **kwargs): - if not self._registered_with_atexit: - # We may register the exit function more than once because - # of a thread race, but that is harmless - import atexit - - atexit.register(self._exitfunc) - weakref_finalize._registered_with_atexit = True - info = self._Info() - info.weakref = ref(obj, self) - info.func = func - info.args = args - info.kwargs = kwargs or None - info.atexit = True - info.index = next(self._index_iter) - self._registry[self] = info - weakref_finalize._dirty = True - - def __call__(self, _=None): - """If alive then mark as dead and return func(*args, **kwargs); - otherwise return None""" - info = self._registry.pop(self, None) - if info and not self._shutdown: - return info.func(*info.args, **(info.kwargs or {})) - - def detach(self): - """If alive then mark as dead and return (obj, func, args, kwargs); - otherwise return None""" - info = self._registry.get(self) - obj = info and info.weakref() - if obj is not None and self._registry.pop(self, None): - return (obj, info.func, info.args, info.kwargs or {}) - - def peek(self): - """If alive then return (obj, func, args, kwargs); - otherwise return None""" - info = self._registry.get(self) - obj = info and info.weakref() - if obj is not None: - return (obj, info.func, info.args, info.kwargs or {}) - - @property - def alive(self): - """Whether finalizer is alive""" - return self in self._registry - - @property - def atexit(self): - """Whether finalizer should be called at exit""" - info = self._registry.get(self) - return bool(info) and info.atexit - - @atexit.setter - def atexit(self, value): - info = self._registry.get(self) - if info: - info.atexit = bool(value) - - def __repr__(self): - info = self._registry.get(self) - obj = info and info.weakref() - if obj is None: - return "<%s object at %#x; dead>" % (type(self).__name__, id(self)) - else: - return "<%s object at %#x; for %r at %#x>" % ( - type(self).__name__, - id(self), - type(obj).__name__, - id(obj), - ) - - @classmethod - def _select_for_exit(cls): - # Return live finalizers marked for exit, oldest first - L = [(f, i) for (f, i) in cls._registry.items() if i.atexit] - L.sort(key=lambda item: item[1].index) - return [f for (f, i) in L] - - @classmethod - def _exitfunc(cls): - # At shutdown invoke finalizers for which atexit is true. - # This is called once all other non-daemonic threads have been - # joined. - reenable_gc = False - try: - if cls._registry: - import gc - - if gc.isenabled(): - reenable_gc = True - gc.disable() - pending = None - while True: - if pending is None or weakref_finalize._dirty: - pending = cls._select_for_exit() - weakref_finalize._dirty = False - if not pending: - break - f = pending.pop() - try: - # gc is disabled, so (assuming no daemonic - # threads) the following is the only line in - # this function which might trigger creation - # of a new finalizer - f() - except Exception: - sys.excepthook(*sys.exc_info()) - assert f not in cls._registry - finally: - # prevent any more finalizers from executing during shutdown - weakref_finalize._shutdown = True - if reenable_gc: - gc.enable() diff --git a/contrib/python/pip/pip/_vendor/urllib3/packages/six.py b/contrib/python/pip/pip/_vendor/urllib3/packages/six.py deleted file mode 100644 index f099a3dcd28..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/packages/six.py +++ /dev/null @@ -1,1076 +0,0 @@ -# Copyright (c) 2010-2020 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Utilities for writing code that runs on Python 2 and 3""" - -from __future__ import absolute_import - -import functools -import itertools -import operator -import sys -import types - -__author__ = "Benjamin Peterson <[email protected]>" -__version__ = "1.16.0" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 -PY34 = sys.version_info[0:2] >= (3, 4) - -if PY3: - string_types = (str,) - integer_types = (int,) - class_types = (type,) - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = (basestring,) - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - -if PY34: - from importlib.util import spec_from_loader -else: - spec_from_loader = None - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) # Invokes __set__. - try: - # This is a bit ugly, but it avoids running this again by - # removing this descriptor. - delattr(obj.__class__, self.name) - except AttributeError: - pass - return result - - -class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - def __getattr__(self, attr): - _module = self._resolve() - value = getattr(_module, attr) - setattr(self, attr, value) - return value - - -class _LazyModule(types.ModuleType): - def __init__(self, name): - super(_LazyModule, self).__init__(name) - self.__doc__ = self.__class__.__doc__ - - def __dir__(self): - attrs = ["__doc__", "__name__"] - attrs += [attr.name for attr in self._moved_attributes] - return attrs - - # Subclasses should override this - _moved_attributes = [] - - -class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - -class _SixMetaPathImporter(object): - - """ - A meta path importer to import six.moves and its submodules. - - This class implements a PEP302 finder and loader. It should be compatible - with Python 2.5 and all existing versions of Python3 - """ - - def __init__(self, six_module_name): - self.name = six_module_name - self.known_modules = {} - - def _add_module(self, mod, *fullnames): - for fullname in fullnames: - self.known_modules[self.name + "." + fullname] = mod - - def _get_module(self, fullname): - return self.known_modules[self.name + "." + fullname] - - def find_module(self, fullname, path=None): - if fullname in self.known_modules: - return self - return None - - def find_spec(self, fullname, path, target=None): - if fullname in self.known_modules: - return spec_from_loader(fullname, self) - return None - - def __get_module(self, fullname): - try: - return self.known_modules[fullname] - except KeyError: - raise ImportError("This loader does not know module " + fullname) - - def load_module(self, fullname): - try: - # in case of a reload - return sys.modules[fullname] - except KeyError: - pass - mod = self.__get_module(fullname) - if isinstance(mod, MovedModule): - mod = mod._resolve() - else: - mod.__loader__ = self - sys.modules[fullname] = mod - return mod - - def is_package(self, fullname): - """ - Return true, if the named module is a package. - - We need this method to get correct spec objects with - Python 3.4 (see PEP451) - """ - return hasattr(self.__get_module(fullname), "__path__") - - def get_code(self, fullname): - """Return None - - Required, if is_package is implemented""" - self.__get_module(fullname) # eventually raises ImportError - return None - - get_source = get_code # same as get_code - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - pass - - -_importer = _SixMetaPathImporter(__name__) - - -class _MovedItems(_LazyModule): - - """Lazy loading of moved objects""" - - __path__ = [] # mark as package - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute( - "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" - ), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("intern", "__builtin__", "sys"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), - MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), - MovedAttribute("getoutput", "commands", "subprocess"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute( - "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" - ), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), - MovedAttribute("UserList", "UserList", "collections"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute( - "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" - ), - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule( - "collections_abc", - "collections", - "collections.abc" if sys.version_info >= (3, 3) else "collections", - ), - MovedModule("copyreg", "copy_reg"), - MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), - MovedModule( - "_dummy_thread", - "dummy_thread", - "_dummy_thread" if sys.version_info < (3, 9) else "_thread", - ), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule( - "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" - ), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("_thread", "thread", "_thread"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), - MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), -] -# Add windows specific modules. -if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] - -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) - if isinstance(attr, MovedModule): - _importer._add_module(attr, "moves." + attr.name) -del attr - -_MovedItems._moved_attributes = _moved_attributes - -moves = _MovedItems(__name__ + ".moves") -_importer._add_module(moves, "moves") - - -class Module_six_moves_urllib_parse(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("SplitResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute( - "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" - ), - MovedAttribute("urlencode", "urllib", "urllib.parse"), - MovedAttribute("splitquery", "urllib", "urllib.parse"), - MovedAttribute("splittag", "urllib", "urllib.parse"), - MovedAttribute("splituser", "urllib", "urllib.parse"), - MovedAttribute("splitvalue", "urllib", "urllib.parse"), - MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), - MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), - MovedAttribute("uses_params", "urlparse", "urllib.parse"), - MovedAttribute("uses_query", "urlparse", "urllib.parse"), - MovedAttribute("uses_relative", "urlparse", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", - "moves.urllib.parse", -) - - -class Module_six_moves_urllib_error(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", - "moves.urllib.error", -) - - -class Module_six_moves_urllib_request(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), - MovedAttribute("proxy_bypass", "urllib", "urllib.request"), - MovedAttribute("parse_http_list", "urllib2", "urllib.request"), - MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", - "moves.urllib.request", -) - - -class Module_six_moves_urllib_response(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", - "moves.urllib.response", -) - - -class Module_six_moves_urllib_robotparser(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -Module_six_moves_urllib_robotparser._moved_attributes = ( - _urllib_robotparser_moved_attributes -) - -_importer._add_module( - Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", - "moves.urllib.robotparser", -) - - -class Module_six_moves_urllib(types.ModuleType): - - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - - __path__ = [] # mark as package - parse = _importer._get_module("moves.urllib_parse") - error = _importer._get_module("moves.urllib_error") - request = _importer._get_module("moves.urllib_request") - response = _importer._get_module("moves.urllib_response") - robotparser = _importer._get_module("moves.urllib_robotparser") - - def __dir__(self): - return ["parse", "error", "request", "response", "robotparser"] - - -_importer._add_module( - Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" -) - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - -try: - advance_iterator = next -except NameError: - - def advance_iterator(it): - return it.next() - - -next = advance_iterator - - -try: - callable = callable -except NameError: - - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - def create_unbound_method(func, cls): - return func - - Iterator = object -else: - - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - def create_unbound_method(func, cls): - return types.MethodType(func, None, cls) - - class Iterator(object): - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc( - get_unbound_function, """Get the function out of a possibly unbound function""" -) - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -if PY3: - - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) - - viewkeys = operator.methodcaller("keys") - - viewvalues = operator.methodcaller("values") - - viewitems = operator.methodcaller("items") -else: - - def iterkeys(d, **kw): - return d.iterkeys(**kw) - - def itervalues(d, **kw): - return d.itervalues(**kw) - - def iteritems(d, **kw): - return d.iteritems(**kw) - - def iterlists(d, **kw): - return d.iterlists(**kw) - - viewkeys = operator.methodcaller("viewkeys") - - viewvalues = operator.methodcaller("viewvalues") - - viewitems = operator.methodcaller("viewitems") - -_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") -_add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc( - iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." -) - - -if PY3: - - def b(s): - return s.encode("latin-1") - - def u(s): - return s - - unichr = chr - import struct - - int2byte = struct.Struct(">B").pack - del struct - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - - StringIO = io.StringIO - BytesIO = io.BytesIO - del io - _assertCountEqual = "assertCountEqual" - if sys.version_info[1] <= 1: - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" - else: - _assertRaisesRegex = "assertRaisesRegex" - _assertRegex = "assertRegex" - _assertNotRegex = "assertNotRegex" -else: - - def b(s): - return s - - # Workaround for standalone backslash - - def u(s): - return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") - - unichr = unichr - int2byte = chr - - def byte2int(bs): - return ord(bs[0]) - - def indexbytes(buf, i): - return ord(buf[i]) - - iterbytes = functools.partial(itertools.imap, ord) - import StringIO - - StringIO = BytesIO = StringIO.StringIO - _assertCountEqual = "assertItemsEqual" - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -def assertCountEqual(self, *args, **kwargs): - return getattr(self, _assertCountEqual)(*args, **kwargs) - - -def assertRaisesRegex(self, *args, **kwargs): - return getattr(self, _assertRaisesRegex)(*args, **kwargs) - - -def assertRegex(self, *args, **kwargs): - return getattr(self, _assertRegex)(*args, **kwargs) - - -def assertNotRegex(self, *args, **kwargs): - return getattr(self, _assertNotRegex)(*args, **kwargs) - - -if PY3: - exec_ = getattr(moves.builtins, "exec") - - def reraise(tp, value, tb=None): - try: - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - finally: - value = None - tb = None - -else: - - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec ("""exec _code_ in _globs_, _locs_""") - - exec_( - """def reraise(tp, value, tb=None): - try: - raise tp, value, tb - finally: - tb = None -""" - ) - - -if sys.version_info[:2] > (3,): - exec_( - """def raise_from(value, from_value): - try: - raise value from from_value - finally: - value = None -""" - ) -else: - - def raise_from(value, from_value): - raise value - - -print_ = getattr(moves.builtins, "print", None) -if print_ is None: - - def print_(*args, **kwargs): - """The new-style print function for Python 2.4 and 2.5.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - - def write(data): - if not isinstance(data, basestring): - data = str(data) - # If the file has an encoding, encode unicode with it. - if ( - isinstance(fp, file) - and isinstance(data, unicode) - and fp.encoding is not None - ): - errors = getattr(fp, "errors", None) - if errors is None: - errors = "strict" - data = data.encode(fp.encoding, errors) - fp.write(data) - - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) - - -if sys.version_info[:2] < (3, 3): - _print = print_ - - def print_(*args, **kwargs): - fp = kwargs.get("file", sys.stdout) - flush = kwargs.pop("flush", False) - _print(*args, **kwargs) - if flush and fp is not None: - fp.flush() - - -_add_doc(reraise, """Reraise an exception.""") - -if sys.version_info[0:2] < (3, 4): - # This does exactly the same what the :func:`py3:functools.update_wrapper` - # function does on Python versions after 3.2. It sets the ``__wrapped__`` - # attribute on ``wrapper`` object and it doesn't raise an error if any of - # the attributes mentioned in ``assigned`` and ``updated`` are missing on - # ``wrapped`` object. - def _update_wrapper( - wrapper, - wrapped, - assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES, - ): - for attr in assigned: - try: - value = getattr(wrapped, attr) - except AttributeError: - continue - else: - setattr(wrapper, attr, value) - for attr in updated: - getattr(wrapper, attr).update(getattr(wrapped, attr, {})) - wrapper.__wrapped__ = wrapped - return wrapper - - _update_wrapper.__doc__ = functools.update_wrapper.__doc__ - - def wraps( - wrapped, - assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES, - ): - return functools.partial( - _update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated - ) - - wraps.__doc__ = functools.wraps.__doc__ - -else: - wraps = functools.wraps - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a dummy - # metaclass for one level of class instantiation that replaces itself with - # the actual metaclass. - class metaclass(type): - def __new__(cls, name, this_bases, d): - if sys.version_info[:2] >= (3, 7): - # This version introduced PEP 560 that requires a bit - # of extra care (we mimic what is done by __build_class__). - resolved_bases = types.resolve_bases(bases) - if resolved_bases is not bases: - d["__orig_bases__"] = bases - else: - resolved_bases = bases - return meta(name, resolved_bases, d) - - @classmethod - def __prepare__(cls, name, this_bases): - return meta.__prepare__(name, bases) - - return type.__new__(metaclass, "temporary_class", (), {}) - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get("__slots__") - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop("__dict__", None) - orig_vars.pop("__weakref__", None) - if hasattr(cls, "__qualname__"): - orig_vars["__qualname__"] = cls.__qualname__ - return metaclass(cls.__name__, cls.__bases__, orig_vars) - - return wrapper - - -def ensure_binary(s, encoding="utf-8", errors="strict"): - """Coerce **s** to six.binary_type. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> encoded to `bytes` - - `bytes` -> `bytes` - """ - if isinstance(s, binary_type): - return s - if isinstance(s, text_type): - return s.encode(encoding, errors) - raise TypeError("not expecting type '%s'" % type(s)) - - -def ensure_str(s, encoding="utf-8", errors="strict"): - """Coerce *s* to `str`. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - # Optimization: Fast return for the common case. - if type(s) is str: - return s - if PY2 and isinstance(s, text_type): - return s.encode(encoding, errors) - elif PY3 and isinstance(s, binary_type): - return s.decode(encoding, errors) - elif not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) - return s - - -def ensure_text(s, encoding="utf-8", errors="strict"): - """Coerce *s* to six.text_type. - - For Python 2: - - `unicode` -> `unicode` - - `str` -> `unicode` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - if isinstance(s, binary_type): - return s.decode(encoding, errors) - elif isinstance(s, text_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) - - -def python_2_unicode_compatible(klass): - """ - A class decorator that defines __unicode__ and __str__ methods under Python 2. - Under Python 3 it does nothing. - - To support Python 2 and 3 with a single code base, define a __str__ method - returning text and apply this decorator to the class. - """ - if PY2: - if "__str__" not in klass.__dict__: - raise ValueError( - "@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % klass.__name__ - ) - klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode("utf-8") - return klass - - -# Complete the moves implementation. -# This code is at the end of this module to speed up module loading. -# Turn this module into a package. -__path__ = [] # required for PEP 302 and PEP 451 -__package__ = __name__ # see PEP 366 @ReservedAssignment -if globals().get("__spec__") is not None: - __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable -# Remove other six meta path importers, since they cause problems. This can -# happen if six is removed from sys.modules and then reloaded. (Setuptools does -# this for some reason.) -if sys.meta_path: - for i, importer in enumerate(sys.meta_path): - # Here's some real nastiness: Another "instance" of the six module might - # be floating around. Therefore, we can't use isinstance() to check for - # the six meta path importer, since the other six instance will have - # inserted an importer with different class. - if ( - type(importer).__name__ == "_SixMetaPathImporter" - and importer.name == __name__ - ): - del sys.meta_path[i] - break - del i, importer -# Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) diff --git a/contrib/python/pip/pip/_vendor/urllib3/poolmanager.py b/contrib/python/pip/pip/_vendor/urllib3/poolmanager.py index fb51bf7d96b..00f9fdced45 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/poolmanager.py +++ b/contrib/python/pip/pip/_vendor/urllib3/poolmanager.py @@ -1,24 +1,33 @@ -from __future__ import absolute_import +from __future__ import annotations -import collections import functools import logging +import typing +import warnings +from types import TracebackType +from urllib.parse import urljoin from ._collections import HTTPHeaderDict, RecentlyUsedContainer +from ._request_methods import RequestMethods +from .connection import ProxyConfig from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, MaxRetryError, ProxySchemeUnknown, - ProxySchemeUnsupported, URLSchemeUnknown, ) -from .packages import six -from .packages.six.moves.urllib.parse import urljoin -from .request import RequestMethods +from .response import BaseHTTPResponse +from .util.connection import _TYPE_SOCKET_OPTIONS from .util.proxy import connection_requires_http_tunnel from .util.retry import Retry -from .util.url import parse_url +from .util.timeout import Timeout +from .util.url import Url, parse_url + +if typing.TYPE_CHECKING: + import ssl + + from typing_extensions import Self __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] @@ -30,53 +39,62 @@ SSL_KEYWORDS = ( "cert_file", "cert_reqs", "ca_certs", + "ca_cert_data", "ssl_version", + "ssl_minimum_version", + "ssl_maximum_version", "ca_cert_dir", "ssl_context", "key_password", "server_hostname", ) +# Default value for `blocksize` - a new parameter introduced to +# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 +_DEFAULT_BLOCKSIZE = 16384 -# All known keyword arguments that could be provided to the pool manager, its -# pools, or the underlying connections. This is used to construct a pool key. -_key_fields = ( - "key_scheme", # str - "key_host", # str - "key_port", # int - "key_timeout", # int or float or Timeout - "key_retries", # int or Retry - "key_strict", # bool - "key_block", # bool - "key_source_address", # str - "key_key_file", # str - "key_key_password", # str - "key_cert_file", # str - "key_cert_reqs", # str - "key_ca_certs", # str - "key_ssl_version", # str - "key_ca_cert_dir", # str - "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext - "key_maxsize", # int - "key_headers", # dict - "key__proxy", # parsed proxy url - "key__proxy_headers", # dict - "key__proxy_config", # class - "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples - "key__socks_options", # dict - "key_assert_hostname", # bool or string - "key_assert_fingerprint", # str - "key_server_hostname", # str -) -#: The namedtuple class used to construct keys for the connection pool. -#: All custom key schemes should include the fields in this key at a minimum. -PoolKey = collections.namedtuple("PoolKey", _key_fields) +class PoolKey(typing.NamedTuple): + """ + All known keyword arguments that could be provided to the pool manager, its + pools, or the underlying connections. + + All custom key schemes should include the fields in this key at a minimum. + """ -_proxy_config_fields = ("ssl_context", "use_forwarding_for_https") -ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields) + key_scheme: str + key_host: str + key_port: int | None + key_timeout: Timeout | float | int | None + key_retries: Retry | bool | int | None + key_block: bool | None + key_source_address: tuple[str, int] | None + key_key_file: str | None + key_key_password: str | None + key_cert_file: str | None + key_cert_reqs: str | None + key_ca_certs: str | None + key_ca_cert_data: str | bytes | None + key_ssl_version: int | str | None + key_ssl_minimum_version: ssl.TLSVersion | None + key_ssl_maximum_version: ssl.TLSVersion | None + key_ca_cert_dir: str | None + key_ssl_context: ssl.SSLContext | None + key_maxsize: int | None + key_headers: frozenset[tuple[str, str]] | None + key__proxy: Url | None + key__proxy_headers: frozenset[tuple[str, str]] | None + key__proxy_config: ProxyConfig | None + key_socket_options: _TYPE_SOCKET_OPTIONS | None + key__socks_options: frozenset[tuple[str, str]] | None + key_assert_hostname: bool | str | None + key_assert_fingerprint: str | None + key_server_hostname: str | None + key_blocksize: int | None -def _default_key_normalizer(key_class, request_context): +def _default_key_normalizer( + key_class: type[PoolKey], request_context: dict[str, typing.Any] +) -> PoolKey: """ Create a pool key out of a request context dictionary. @@ -122,6 +140,10 @@ def _default_key_normalizer(key_class, request_context): if field not in context: context[field] = None + # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context + if context.get("key_blocksize") is None: + context["key_blocksize"] = _DEFAULT_BLOCKSIZE + return key_class(**context) @@ -154,23 +176,50 @@ class PoolManager(RequestMethods): Additional parameters are used to create fresh :class:`urllib3.connectionpool.ConnectionPool` instances. - Example:: + Example: + + .. code-block:: python + + from pip._vendor import urllib3 + + http = urllib3.PoolManager(num_pools=2) - >>> manager = PoolManager(num_pools=2) - >>> r = manager.request('GET', 'http://google.com/') - >>> r = manager.request('GET', 'http://google.com/mail') - >>> r = manager.request('GET', 'http://yahoo.com/') - >>> len(manager.pools) - 2 + resp1 = http.request("GET", "https://google.com/") + resp2 = http.request("GET", "https://google.com/mail") + resp3 = http.request("GET", "https://yahoo.com/") + + print(len(http.pools)) + # 2 """ - proxy = None - proxy_config = None + proxy: Url | None = None + proxy_config: ProxyConfig | None = None - def __init__(self, num_pools=10, headers=None, **connection_pool_kw): - RequestMethods.__init__(self, headers) + def __init__( + self, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + **connection_pool_kw: typing.Any, + ) -> None: + super().__init__(headers) + # PoolManager handles redirects itself in PoolManager.urlopen(). + # It always passes redirect=False to the underlying connection pool to + # suppress per-pool redirect handling. If the user supplied a non-Retry + # value (int/bool/etc) for retries and we let the pool normalize it + # while redirect=False, the resulting Retry object would have redirect + # handling disabled, which can interfere with PoolManager's own + # redirect logic. Normalize here so redirects remain governed solely by + # PoolManager logic. + if "retries" in connection_pool_kw: + retries = connection_pool_kw["retries"] + if not isinstance(retries, Retry): + retries = Retry.from_int(retries) + connection_pool_kw = connection_pool_kw.copy() + connection_pool_kw["retries"] = retries self.connection_pool_kw = connection_pool_kw + + self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool] self.pools = RecentlyUsedContainer(num_pools) # Locally set the pool classes and keys so other PoolManagers can @@ -178,15 +227,26 @@ class PoolManager(RequestMethods): self.pool_classes_by_scheme = pool_classes_by_scheme self.key_fn_by_scheme = key_fn_by_scheme.copy() - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> typing.Literal[False]: self.clear() # Return False to re-raise any potential exceptions return False - def _new_pool(self, scheme, host, port, request_context=None): + def _new_pool( + self, + scheme: str, + host: str, + port: int, + request_context: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: """ Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and any additional pool keyword arguments. @@ -196,10 +256,15 @@ class PoolManager(RequestMethods): connection pools handed out by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = self.pool_classes_by_scheme[scheme] + pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme] if request_context is None: request_context = self.connection_pool_kw.copy() + # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly + # set to 'None' in the request_context. + if request_context.get("blocksize") is None: + request_context["blocksize"] = _DEFAULT_BLOCKSIZE + # Although the context has everything necessary to create the pool, # this function has historically only used the scheme, host, and port # in the positional args. When an API change is acceptable these can @@ -213,7 +278,7 @@ class PoolManager(RequestMethods): return pool_cls(host, port, **request_context) - def clear(self): + def clear(self) -> None: """ Empty our store of pools and direct them all to close. @@ -222,7 +287,13 @@ class PoolManager(RequestMethods): """ self.pools.clear() - def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): + def connection_from_host( + self, + host: str | None, + port: int | None = None, + scheme: str | None = "http", + pool_kwargs: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme. @@ -245,13 +316,23 @@ class PoolManager(RequestMethods): return self.connection_from_context(request_context) - def connection_from_context(self, request_context): + def connection_from_context( + self, request_context: dict[str, typing.Any] + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context. ``request_context`` must at least contain the ``scheme`` key and its value must be a key in ``key_fn_by_scheme`` instance variable. """ + if "strict" in request_context: + warnings.warn( + "The 'strict' parameter is no longer needed on Python 3+. " + "This will raise an error in urllib3 v2.1.0.", + DeprecationWarning, + ) + request_context.pop("strict") + scheme = request_context["scheme"].lower() pool_key_constructor = self.key_fn_by_scheme.get(scheme) if not pool_key_constructor: @@ -260,7 +341,9 @@ class PoolManager(RequestMethods): return self.connection_from_pool_key(pool_key, request_context=request_context) - def connection_from_pool_key(self, pool_key, request_context=None): + def connection_from_pool_key( + self, pool_key: PoolKey, request_context: dict[str, typing.Any] + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key. @@ -284,7 +367,9 @@ class PoolManager(RequestMethods): return pool - def connection_from_url(self, url, pool_kwargs=None): + def connection_from_url( + self, url: str, pool_kwargs: dict[str, typing.Any] | None = None + ) -> HTTPConnectionPool: """ Similar to :func:`urllib3.connectionpool.connection_from_url`. @@ -300,7 +385,9 @@ class PoolManager(RequestMethods): u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs ) - def _merge_pool_kwargs(self, override): + def _merge_pool_kwargs( + self, override: dict[str, typing.Any] | None + ) -> dict[str, typing.Any]: """ Merge a dictionary of override values for self.connection_pool_kw. @@ -320,7 +407,7 @@ class PoolManager(RequestMethods): base_pool_kwargs[key] = value return base_pool_kwargs - def _proxy_requires_url_absolute_form(self, parsed_url): + def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool: """ Indicates if the proxy requires the complete destination URL in the request. Normally this is only needed when not using an HTTP CONNECT @@ -333,24 +420,9 @@ class PoolManager(RequestMethods): self.proxy, self.proxy_config, parsed_url.scheme ) - def _validate_proxy_scheme_url_selection(self, url_scheme): - """ - Validates that were not attempting to do TLS in TLS connections on - Python2 or with unsupported SSL implementations. - """ - if self.proxy is None or url_scheme != "https": - return - - if self.proxy.scheme != "https": - return - - if six.PY2 and not self.proxy_config.use_forwarding_for_https: - raise ProxySchemeUnsupported( - "Contacting HTTPS destinations through HTTPS proxies " - "'via CONNECT tunnels' is not supported in Python 2" - ) - - def urlopen(self, method, url, redirect=True, **kw): + def urlopen( # type: ignore[override] + self, method: str, url: str, redirect: bool = True, **kw: typing.Any + ) -> BaseHTTPResponse: """ Same as :meth:`urllib3.HTTPConnectionPool.urlopen` with custom cross-host redirect logic and only sends the request-uri @@ -360,7 +432,16 @@ class PoolManager(RequestMethods): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ u = parse_url(url) - self._validate_proxy_scheme_url_selection(u.scheme) + + if u.scheme is None: + warnings.warn( + "URLs without a scheme (ie 'https://') are deprecated and will raise an error " + "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs " + "start with 'https://' or 'http://'. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2920", + category=DeprecationWarning, + stacklevel=2, + ) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) @@ -368,7 +449,7 @@ class PoolManager(RequestMethods): kw["redirect"] = False if "headers" not in kw: - kw["headers"] = self.headers.copy() + kw["headers"] = self.headers if self._proxy_requires_url_absolute_form(u): response = conn.urlopen(method, url, **kw) @@ -389,7 +470,7 @@ class PoolManager(RequestMethods): kw["body"] = None kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() - retries = kw.get("retries") + retries = kw.get("retries", response.retries) if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) @@ -399,10 +480,11 @@ class PoolManager(RequestMethods): if retries.remove_headers_on_redirect and not conn.is_same_host( redirect_location ): - headers = list(six.iterkeys(kw["headers"])) - for header in headers: + new_headers = kw["headers"].copy() + for header in kw["headers"]: if header.lower() in retries.remove_headers_on_redirect: - kw["headers"].pop(header, None) + new_headers.pop(header, None) + kw["headers"] = new_headers try: retries = retries.increment(method, url, response=response, _pool=conn) @@ -448,37 +530,51 @@ class ProxyManager(PoolManager): private. IP address, target hostname, SNI, and port are always visible to an HTTPS proxy even when this flag is disabled. + :param proxy_assert_hostname: + The hostname of the certificate to verify against. + + :param proxy_assert_fingerprint: + The fingerprint of the certificate to verify against. + Example: - >>> proxy = urllib3.ProxyManager('http://localhost:3128/') - >>> r1 = proxy.request('GET', 'http://google.com/') - >>> r2 = proxy.request('GET', 'http://httpbin.org/') - >>> len(proxy.pools) - 1 - >>> r3 = proxy.request('GET', 'https://httpbin.org/') - >>> r4 = proxy.request('GET', 'https://twitter.com/') - >>> len(proxy.pools) - 3 + + .. code-block:: python + + from pip._vendor import urllib3 + + proxy = urllib3.ProxyManager("https://localhost:3128/") + + resp1 = proxy.request("GET", "https://google.com/") + resp2 = proxy.request("GET", "https://httpbin.org/") + + print(len(proxy.pools)) + # 1 + + resp3 = proxy.request("GET", "https://httpbin.org/") + resp4 = proxy.request("GET", "https://twitter.com/") + + print(len(proxy.pools)) + # 3 """ def __init__( self, - proxy_url, - num_pools=10, - headers=None, - proxy_headers=None, - proxy_ssl_context=None, - use_forwarding_for_https=False, - **connection_pool_kw - ): - + proxy_url: str, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + proxy_headers: typing.Mapping[str, str] | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + use_forwarding_for_https: bool = False, + proxy_assert_hostname: None | str | typing.Literal[False] = None, + proxy_assert_fingerprint: str | None = None, + **connection_pool_kw: typing.Any, + ) -> None: if isinstance(proxy_url, HTTPConnectionPool): - proxy_url = "%s://%s:%i" % ( - proxy_url.scheme, - proxy_url.host, - proxy_url.port, - ) - proxy = parse_url(proxy_url) + str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}" + else: + str_proxy_url = proxy_url + proxy = parse_url(str_proxy_url) if proxy.scheme not in ("http", "https"): raise ProxySchemeUnknown(proxy.scheme) @@ -490,25 +586,38 @@ class ProxyManager(PoolManager): self.proxy = proxy self.proxy_headers = proxy_headers or {} self.proxy_ssl_context = proxy_ssl_context - self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https) + self.proxy_config = ProxyConfig( + proxy_ssl_context, + use_forwarding_for_https, + proxy_assert_hostname, + proxy_assert_fingerprint, + ) connection_pool_kw["_proxy"] = self.proxy connection_pool_kw["_proxy_headers"] = self.proxy_headers connection_pool_kw["_proxy_config"] = self.proxy_config - super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) + super().__init__(num_pools, headers, **connection_pool_kw) - def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): + def connection_from_host( + self, + host: str | None, + port: int | None = None, + scheme: str | None = "http", + pool_kwargs: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: if scheme == "https": - return super(ProxyManager, self).connection_from_host( + return super().connection_from_host( host, port, scheme, pool_kwargs=pool_kwargs ) - return super(ProxyManager, self).connection_from_host( - self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs + return super().connection_from_host( + self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr] ) - def _set_proxy_headers(self, url, headers=None): + def _set_proxy_headers( + self, url: str, headers: typing.Mapping[str, str] | None = None + ) -> typing.Mapping[str, str]: """ Sets headers needed by proxies: specifically, the Accept and Host headers. Only sets headers not provided by the user. @@ -523,7 +632,9 @@ class ProxyManager(PoolManager): headers_.update(headers) return headers_ - def urlopen(self, method, url, redirect=True, **kw): + def urlopen( # type: ignore[override] + self, method: str, url: str, redirect: bool = True, **kw: typing.Any + ) -> BaseHTTPResponse: "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme): @@ -533,8 +644,8 @@ class ProxyManager(PoolManager): headers = kw.get("headers", self.headers) kw["headers"] = self._set_proxy_headers(url, headers) - return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) + return super().urlopen(method, url, redirect=redirect, **kw) -def proxy_from_url(url, **kw): +def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager: return ProxyManager(proxy_url=url, **kw) diff --git a/contrib/python/pip/pip/_vendor/urllib3/py.typed b/contrib/python/pip/pip/_vendor/urllib3/py.typed new file mode 100644 index 00000000000..5f3ea3d9193 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/py.typed @@ -0,0 +1,2 @@ +# Instruct type checkers to look for inline type annotations in this package. +# See PEP 561. diff --git a/contrib/python/pip/pip/_vendor/urllib3/response.py b/contrib/python/pip/pip/_vendor/urllib3/response.py index 8909f8454e9..68f914dddf5 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/response.py +++ b/contrib/python/pip/pip/_vendor/urllib3/response.py @@ -1,22 +1,32 @@ -from __future__ import absolute_import +from __future__ import annotations +import collections import io +import json as _json import logging +import socket import sys +import typing import warnings import zlib from contextlib import contextmanager -from socket import error as SocketError +from http.client import HTTPMessage as _HttplibHTTPMessage +from http.client import HTTPResponse as _HttplibHTTPResponse from socket import timeout as SocketTimeout +if typing.TYPE_CHECKING: + from ._base_connection import BaseHTTPConnection + brotli = None from . import util +from ._base_connection import _TYPE_BODY from ._collections import HTTPHeaderDict -from .connection import BaseSSLError, HTTPException +from .connection import BaseSSLError, HTTPConnection, HTTPException from .exceptions import ( BodyNotHttplibCompatible, DecodeError, + DependencyWarning, HTTPError, IncompleteRead, InvalidChunkLength, @@ -26,101 +36,262 @@ from .exceptions import ( ResponseNotChunked, SSLError, ) -from .packages import six from .util.response import is_fp_closed, is_response_to_head +from .util.retry import Retry + +if typing.TYPE_CHECKING: + from .connectionpool import HTTPConnectionPool log = logging.getLogger(__name__) -class DeflateDecoder(object): - def __init__(self): +class ContentDecoder: + def decompress(self, data: bytes, max_length: int = -1) -> bytes: + raise NotImplementedError() + + @property + def has_unconsumed_tail(self) -> bool: + raise NotImplementedError() + + def flush(self) -> bytes: + raise NotImplementedError() + + +class DeflateDecoder(ContentDecoder): + def __init__(self) -> None: self._first_try = True - self._data = b"" + self._first_try_data = b"" + self._unfed_data = b"" self._obj = zlib.decompressobj() - def __getattr__(self, name): - return getattr(self._obj, name) - - def decompress(self, data): - if not data: + def decompress(self, data: bytes, max_length: int = -1) -> bytes: + data = self._unfed_data + data + self._unfed_data = b"" + if not data and not self._obj.unconsumed_tail: return data + original_max_length = max_length + if original_max_length < 0: + max_length = 0 + elif original_max_length == 0: + # We should not pass 0 to the zlib decompressor because 0 is + # the default value that will make zlib decompress without a + # length limit. + # Data should be stored for subsequent calls. + self._unfed_data = data + return b"" + # Subsequent calls always reuse `self._obj`. zlib requires + # passing the unconsumed tail if decompression is to continue. if not self._first_try: - return self._obj.decompress(data) + return self._obj.decompress( + self._obj.unconsumed_tail + data, max_length=max_length + ) - self._data += data + # First call tries with RFC 1950 ZLIB format. + self._first_try_data += data try: - decompressed = self._obj.decompress(data) + decompressed = self._obj.decompress(data, max_length=max_length) if decompressed: self._first_try = False - self._data = None + self._first_try_data = b"" return decompressed + # On failure, it falls back to RFC 1951 DEFLATE format. except zlib.error: self._first_try = False self._obj = zlib.decompressobj(-zlib.MAX_WBITS) try: - return self.decompress(self._data) + return self.decompress( + self._first_try_data, max_length=original_max_length + ) finally: - self._data = None + self._first_try_data = b"" + @property + def has_unconsumed_tail(self) -> bool: + return bool(self._unfed_data) or ( + bool(self._obj.unconsumed_tail) and not self._first_try + ) -class GzipDecoderState(object): + def flush(self) -> bytes: + return self._obj.flush() + +class GzipDecoderState: FIRST_MEMBER = 0 OTHER_MEMBERS = 1 SWALLOW_DATA = 2 -class GzipDecoder(object): - def __init__(self): +class GzipDecoder(ContentDecoder): + def __init__(self) -> None: self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER + self._unconsumed_tail = b"" - def __getattr__(self, name): - return getattr(self._obj, name) - - def decompress(self, data): + def decompress(self, data: bytes, max_length: int = -1) -> bytes: ret = bytearray() - if self._state == GzipDecoderState.SWALLOW_DATA or not data: + if self._state == GzipDecoderState.SWALLOW_DATA: + return bytes(ret) + + if max_length == 0: + # We should not pass 0 to the zlib decompressor because 0 is + # the default value that will make zlib decompress without a + # length limit. + # Data should be stored for subsequent calls. + self._unconsumed_tail += data + return b"" + + # zlib requires passing the unconsumed tail to the subsequent + # call if decompression is to continue. + data = self._unconsumed_tail + data + if not data and self._obj.eof: return bytes(ret) + while True: try: - ret += self._obj.decompress(data) + ret += self._obj.decompress( + data, max_length=max(max_length - len(ret), 0) + ) except zlib.error: previous_state = self._state # Ignore data after the first error self._state = GzipDecoderState.SWALLOW_DATA + self._unconsumed_tail = b"" if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients return bytes(ret) raise - data = self._obj.unused_data + + self._unconsumed_tail = data = ( + self._obj.unconsumed_tail or self._obj.unused_data + ) + if max_length > 0 and len(ret) >= max_length: + break + if not data: return bytes(ret) - self._state = GzipDecoderState.OTHER_MEMBERS - self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + # When the end of a gzip member is reached, a new decompressor + # must be created for unused (possibly future) data. + if self._obj.eof: + self._state = GzipDecoderState.OTHER_MEMBERS + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + + return bytes(ret) + + @property + def has_unconsumed_tail(self) -> bool: + return bool(self._unconsumed_tail) + + def flush(self) -> bytes: + return self._obj.flush() if brotli is not None: - class BrotliDecoder(object): + class BrotliDecoder(ContentDecoder): # Supports both 'brotlipy' and 'Brotli' packages # since they share an import name. The top branches # are for 'brotlipy' and bottom branches for 'Brotli' - def __init__(self): + def __init__(self) -> None: self._obj = brotli.Decompressor() if hasattr(self._obj, "decompress"): - self.decompress = self._obj.decompress + setattr(self, "_decompress", self._obj.decompress) else: - self.decompress = self._obj.process + setattr(self, "_decompress", self._obj.process) + + # Requires Brotli >= 1.2.0 for `output_buffer_limit`. + def _decompress(self, data: bytes, output_buffer_limit: int = -1) -> bytes: + raise NotImplementedError() - def flush(self): + def decompress(self, data: bytes, max_length: int = -1) -> bytes: + try: + if max_length > 0: + return self._decompress(data, output_buffer_limit=max_length) + else: + return self._decompress(data) + except TypeError: + # Fallback for Brotli/brotlicffi/brotlipy versions without + # the `output_buffer_limit` parameter. + warnings.warn( + "Brotli >= 1.2.0 is required to prevent decompression bombs.", + DependencyWarning, + ) + return self._decompress(data) + + @property + def has_unconsumed_tail(self) -> bool: + try: + return not self._obj.can_accept_more_data() + except AttributeError: + return False + + def flush(self) -> bytes: if hasattr(self._obj, "flush"): - return self._obj.flush() + return self._obj.flush() # type: ignore[no-any-return] return b"" -class MultiDecoder(object): +try: + if sys.version_info >= (3, 14): + from compression import zstd + else: + from backports import zstd +except ImportError: + HAS_ZSTD = False +else: + HAS_ZSTD = True + + class ZstdDecoder(ContentDecoder): + def __init__(self) -> None: + self._obj = zstd.ZstdDecompressor() + + def decompress(self, data: bytes, max_length: int = -1) -> bytes: + if not data and not self.has_unconsumed_tail: + return b"" + if self._obj.eof: + data = self._obj.unused_data + data + self._obj = zstd.ZstdDecompressor() + part = self._obj.decompress(data, max_length=max_length) + length = len(part) + data_parts = [part] + # Every loop iteration is supposed to read data from a separate frame. + # The loop breaks when: + # - enough data is read; + # - no more unused data is available; + # - end of the last read frame has not been reached (i.e., + # more data has to be fed). + while ( + self._obj.eof + and self._obj.unused_data + and (max_length < 0 or length < max_length) + ): + unused_data = self._obj.unused_data + if not self._obj.needs_input: + self._obj = zstd.ZstdDecompressor() + part = self._obj.decompress( + unused_data, + max_length=(max_length - length) if max_length > 0 else -1, + ) + if part_length := len(part): + data_parts.append(part) + length += part_length + elif self._obj.needs_input: + break + return b"".join(data_parts) + + @property + def has_unconsumed_tail(self) -> bool: + return not (self._obj.needs_input or self._obj.eof) or bool( + self._obj.unused_data + ) + + def flush(self) -> bytes: + if not self._obj.eof: + raise DecodeError("Zstandard data is incomplete") + return b"" + + +class MultiDecoder(ContentDecoder): """ From RFC7231: If one or more encodings have been applied to a representation, the @@ -129,32 +300,387 @@ class MultiDecoder(object): they were applied. """ - def __init__(self, modes): - self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] + # Maximum allowed number of chained HTTP encodings in the + # Content-Encoding header. + max_decode_links = 5 + + def __init__(self, modes: str) -> None: + encodings = [m.strip() for m in modes.split(",")] + if len(encodings) > self.max_decode_links: + raise DecodeError( + "Too many content encodings in the chain: " + f"{len(encodings)} > {self.max_decode_links}" + ) + self._decoders = [_get_decoder(e) for e in encodings] - def flush(self): + def flush(self) -> bytes: return self._decoders[0].flush() - def decompress(self, data): - for d in reversed(self._decoders): - data = d.decompress(data) - return data + def decompress(self, data: bytes, max_length: int = -1) -> bytes: + if max_length <= 0: + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + ret = bytearray() + # Every while loop iteration goes through all decoders once. + # It exits when enough data is read or no more data can be read. + # It is possible that the while loop iteration does not produce + # any data because we retrieve up to `max_length` from every + # decoder, and the amount of bytes may be insufficient for the + # next decoder to produce enough/any output. + while True: + any_data = False + for d in reversed(self._decoders): + data = d.decompress(data, max_length=max_length - len(ret)) + if data: + any_data = True + # We should not break when no data is returned because + # next decoders may produce data even with empty input. + ret += data + if not any_data or len(ret) >= max_length: + return bytes(ret) + data = b"" + + @property + def has_unconsumed_tail(self) -> bool: + return any(d.has_unconsumed_tail for d in self._decoders) -def _get_decoder(mode): +def _get_decoder(mode: str) -> ContentDecoder: if "," in mode: return MultiDecoder(mode) - if mode == "gzip": + # According to RFC 9110 section 8.4.1.3, recipients should + # consider x-gzip equivalent to gzip + if mode in ("gzip", "x-gzip"): return GzipDecoder() if brotli is not None and mode == "br": return BrotliDecoder() + if HAS_ZSTD and mode == "zstd": + return ZstdDecoder() + return DeflateDecoder() -class HTTPResponse(io.IOBase): +class BytesQueueBuffer: + """Memory-efficient bytes buffer + + To return decoded data in read() and still follow the BufferedIOBase API, we need a + buffer to always return the correct amount of bytes. + + This buffer should be filled using calls to put() + + Our maximum memory usage is determined by the sum of the size of: + + * self.buffer, which contains the full data + * the largest chunk that we will copy in get() + """ + + def __init__(self) -> None: + self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque() + self._size: int = 0 + + def __len__(self) -> int: + return self._size + + def put(self, data: bytes) -> None: + self.buffer.append(data) + self._size += len(data) + + def get(self, n: int) -> bytes: + if n == 0: + return b"" + elif not self.buffer: + raise RuntimeError("buffer is empty") + elif n < 0: + raise ValueError("n should be > 0") + + if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes): + self._size -= n + return self.buffer.popleft() + + fetched = 0 + ret = io.BytesIO() + while fetched < n: + remaining = n - fetched + chunk = self.buffer.popleft() + chunk_length = len(chunk) + if remaining < chunk_length: + chunk = memoryview(chunk) + left_chunk, right_chunk = chunk[:remaining], chunk[remaining:] + ret.write(left_chunk) + self.buffer.appendleft(right_chunk) + self._size -= remaining + break + else: + ret.write(chunk) + self._size -= chunk_length + fetched += chunk_length + + if not self.buffer: + break + + return ret.getvalue() + + def get_all(self) -> bytes: + buffer = self.buffer + if not buffer: + assert self._size == 0 + return b"" + if len(buffer) == 1: + result = buffer.pop() + if isinstance(result, memoryview): + result = result.tobytes() + else: + ret = io.BytesIO() + ret.writelines(buffer.popleft() for _ in range(len(buffer))) + result = ret.getvalue() + self._size = 0 + return result + + +class BaseHTTPResponse(io.IOBase): + CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] + if brotli is not None: + CONTENT_DECODERS += ["br"] + if HAS_ZSTD: + CONTENT_DECODERS += ["zstd"] + REDIRECT_STATUSES = [301, 302, 303, 307, 308] + + DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error) + if brotli is not None: + DECODER_ERROR_CLASSES += (brotli.error,) + + if HAS_ZSTD: + DECODER_ERROR_CLASSES += (zstd.ZstdError,) + + def __init__( + self, + *, + headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, + status: int, + version: int, + version_string: str, + reason: str | None, + decode_content: bool, + request_url: str | None, + retries: Retry | None = None, + ) -> None: + if isinstance(headers, HTTPHeaderDict): + self.headers = headers + else: + self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type] + self.status = status + self.version = version + self.version_string = version_string + self.reason = reason + self.decode_content = decode_content + self._has_decoded_content = False + self._request_url: str | None = request_url + self.retries = retries + + self.chunked = False + tr_enc = self.headers.get("transfer-encoding", "").lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: + self.chunked = True + + self._decoder: ContentDecoder | None = None + self.length_remaining: int | None + + def get_redirect_location(self) -> str | None | typing.Literal[False]: + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in self.REDIRECT_STATUSES: + return self.headers.get("location") + return False + + @property + def data(self) -> bytes: + raise NotImplementedError() + + def json(self) -> typing.Any: + """ + Deserializes the body of the HTTP response as a Python object. + + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. + + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here <json_content>`. + + :returns: The body of the HTTP response as a Python object. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + + @property + def url(self) -> str | None: + raise NotImplementedError() + + @url.setter + def url(self, url: str | None) -> None: + raise NotImplementedError() + + @property + def connection(self) -> BaseHTTPConnection | None: + raise NotImplementedError() + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + if retries is not None and retries.history: + self.url = retries.history[-1].redirect_location + self._retries = retries + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Iterator[bytes]: + raise NotImplementedError() + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, + cache_content: bool = False, + ) -> bytes: + raise NotImplementedError() + + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + raise NotImplementedError() + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Iterator[bytes]: + raise NotImplementedError() + + def release_conn(self) -> None: + raise NotImplementedError() + + def drain_conn(self) -> None: + raise NotImplementedError() + + def shutdown(self) -> None: + raise NotImplementedError() + + def close(self) -> None: + raise NotImplementedError() + + def _init_decoder(self) -> None: + """ + Set-up the _decoder attribute if necessary. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get("content-encoding", "").lower() + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif "," in content_encoding: + encodings = [ + e.strip() + for e in content_encoding.split(",") + if e.strip() in self.CONTENT_DECODERS + ] + if encodings: + self._decoder = _get_decoder(content_encoding) + + def _decode( + self, + data: bytes, + decode_content: bool | None, + flush_decoder: bool, + max_length: int | None = None, + ) -> bytes: + """ + Decode the data passed in and potentially flush the decoder. + """ + if not decode_content: + if self._has_decoded_content: + raise RuntimeError( + "Calling read(decode_content=False) is not supported after " + "read(decode_content=True) was called." + ) + return data + + if max_length is None or flush_decoder: + max_length = -1 + + try: + if self._decoder: + data = self._decoder.decompress(data, max_length=max_length) + self._has_decoded_content = True + except self.DECODER_ERROR_CLASSES as e: + content_encoding = self.headers.get("content-encoding", "").lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, + e, + ) from e + if flush_decoder: + data += self._flush_decoder() + + return data + + def _flush_decoder(self) -> bytes: + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + return self._decoder.decompress(b"") + self._decoder.flush() + return b"" + + # Compatibility methods for `io` module + def readinto(self, b: bytearray) -> int: + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + else: + b[: len(temp)] = temp + return len(temp) + + # Methods used by dependent libraries + def getheaders(self) -> HTTPHeaderDict: + return self.headers + + def getheader(self, name: str, default: str | None = None) -> str | None: + return self.headers.get(name, default) + + # Compatibility method for http.cookiejar + def info(self) -> HTTPHeaderDict: + return self.headers + + def geturl(self) -> str | None: + return self.url + + +class HTTPResponse(BaseHTTPResponse): """ HTTP Response container. @@ -187,126 +713,111 @@ class HTTPResponse(io.IOBase): value of Content-Length header, if present. Otherwise, raise error. """ - CONTENT_DECODERS = ["gzip", "deflate"] - if brotli is not None: - CONTENT_DECODERS += ["br"] - REDIRECT_STATUSES = [301, 302, 303, 307, 308] - def __init__( self, - body="", - headers=None, - status=0, - version=0, - reason=None, - strict=0, - preload_content=True, - decode_content=True, - original_response=None, - pool=None, - connection=None, - msg=None, - retries=None, - enforce_content_length=False, - request_method=None, - request_url=None, - auto_close=True, - ): + body: _TYPE_BODY = "", + headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, + status: int = 0, + version: int = 0, + version_string: str = "HTTP/?", + reason: str | None = None, + preload_content: bool = True, + decode_content: bool = True, + original_response: _HttplibHTTPResponse | None = None, + pool: HTTPConnectionPool | None = None, + connection: HTTPConnection | None = None, + msg: _HttplibHTTPMessage | None = None, + retries: Retry | None = None, + enforce_content_length: bool = True, + request_method: str | None = None, + request_url: str | None = None, + auto_close: bool = True, + sock_shutdown: typing.Callable[[int], None] | None = None, + ) -> None: + super().__init__( + headers=headers, + status=status, + version=version, + version_string=version_string, + reason=reason, + decode_content=decode_content, + request_url=request_url, + retries=retries, + ) - if isinstance(headers, HTTPHeaderDict): - self.headers = headers - else: - self.headers = HTTPHeaderDict(headers) - self.status = status - self.version = version - self.reason = reason - self.strict = strict - self.decode_content = decode_content - self.retries = retries self.enforce_content_length = enforce_content_length self.auto_close = auto_close - self._decoder = None self._body = None - self._fp = None + self._fp: _HttplibHTTPResponse | None = None self._original_response = original_response self._fp_bytes_read = 0 self.msg = msg - self._request_url = request_url - if body and isinstance(body, (six.string_types, bytes)): + if body and isinstance(body, (str, bytes)): self._body = body self._pool = pool self._connection = connection if hasattr(body, "read"): - self._fp = body + self._fp = body # type: ignore[assignment] + self._sock_shutdown = sock_shutdown # Are we using the chunked-style of transfer encoding? - self.chunked = False - self.chunk_left = None - tr_enc = self.headers.get("transfer-encoding", "").lower() - # Don't incur the penalty of creating a list and then discarding it - encodings = (enc.strip() for enc in tr_enc.split(",")) - if "chunked" in encodings: - self.chunked = True + self.chunk_left: int | None = None # Determine length of response self.length_remaining = self._init_length(request_method) + # Used to return the correct amount of bytes for partial read()s + self._decoded_buffer = BytesQueueBuffer() + # If requested, preload the body. if preload_content and not self._body: self._body = self.read(decode_content=decode_content) - def get_redirect_location(self): - """ - Should we redirect and where to? - - :returns: Truthy redirect location string if we got a redirect status - code and valid location. ``None`` if redirect status and no - location. ``False`` if not a redirect status code. - """ - if self.status in self.REDIRECT_STATUSES: - return self.headers.get("location") - - return False - - def release_conn(self): + def release_conn(self) -> None: if not self._pool or not self._connection: - return + return None self._pool._put_conn(self._connection) self._connection = None - def drain_conn(self): + def drain_conn(self) -> None: """ Read and discard any remaining HTTP response data in the response connection. Unread data in the HTTPResponse connection blocks the connection from being released back to the pool. """ try: - self.read() - except (HTTPError, SocketError, BaseSSLError, HTTPException): + self.read( + # Do not spend resources decoding the content unless + # decoding has already been initiated. + decode_content=self._has_decoded_content, + ) + except (HTTPError, OSError, BaseSSLError, HTTPException): pass @property - def data(self): + def data(self) -> bytes: # For backwards-compat with earlier urllib3 0.4 and earlier. if self._body: - return self._body + return self._body # type: ignore[return-value] if self._fp: return self.read(cache_content=True) + return None # type: ignore[return-value] + @property - def connection(self): + def connection(self) -> HTTPConnection | None: return self._connection - def isclosed(self): + def isclosed(self) -> bool: return is_fp_closed(self._fp) - def tell(self): + def tell(self) -> int: """ Obtain the number of bytes pulled over the wire so far. May differ from the amount of content returned by :meth:``urllib3.response.HTTPResponse.read`` @@ -314,13 +825,14 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read - def _init_length(self, request_method): + def _init_length(self, request_method: str | None) -> int | None: """ Set initial length value for Response content if available. """ - length = self.headers.get("content-length") + length: int | None + content_length: str | None = self.headers.get("content-length") - if length is not None: + if content_length is not None: if self.chunked: # This Response will fail with an IncompleteRead if it can't be # received as chunked. This method falls back to attempt reading @@ -340,11 +852,11 @@ class HTTPResponse(io.IOBase): # (e.g. Content-Length: 42, 42). This line ensures the values # are all valid ints and that as long as the `set` length is 1, # all values are the same. Otherwise, the header is invalid. - lengths = set([int(val) for val in length.split(",")]) + lengths = {int(val) for val in content_length.split(",")} if len(lengths) > 1: raise InvalidHeader( "Content-Length contained multiple " - "unmatching values (%s)" % length + "unmatching values (%s)" % content_length ) length = lengths.pop() except ValueError: @@ -353,6 +865,9 @@ class HTTPResponse(io.IOBase): if length < 0: length = None + else: # if content_length is None + length = None + # Convert status to int for comparison # In some cases, httplib returns a status of "_UNKNOWN" try: @@ -366,64 +881,8 @@ class HTTPResponse(io.IOBase): return length - def _init_decoder(self): - """ - Set-up the _decoder attribute if necessary. - """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get("content-encoding", "").lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) - elif "," in content_encoding: - encodings = [ - e.strip() - for e in content_encoding.split(",") - if e.strip() in self.CONTENT_DECODERS - ] - if len(encodings): - self._decoder = _get_decoder(content_encoding) - - DECODER_ERROR_CLASSES = (IOError, zlib.error) - if brotli is not None: - DECODER_ERROR_CLASSES += (brotli.error,) - - def _decode(self, data, decode_content, flush_decoder): - """ - Decode the data passed in and potentially flush the decoder. - """ - if not decode_content: - return data - - try: - if self._decoder: - data = self._decoder.decompress(data) - except self.DECODER_ERROR_CLASSES as e: - content_encoding = self.headers.get("content-encoding", "").lower() - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, - e, - ) - if flush_decoder: - data += self._flush_decoder() - - return data - - def _flush_decoder(self): - """ - Flushes the decoder. Should only be called if the decoder is actually - being used. - """ - if self._decoder: - buf = self._decoder.decompress(b"") - return buf + self._decoder.flush() - - return b"" - @contextmanager - def _error_catcher(self): + def _error_catcher(self) -> typing.Generator[None]: """ Catch low-level python exceptions, instead re-raising urllib3 variants, so that low-level exceptions are not leaked in the @@ -437,22 +896,32 @@ class HTTPResponse(io.IOBase): try: yield - except SocketTimeout: + except SocketTimeout as e: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. - raise ReadTimeoutError(self._pool, None, "Read timed out.") + raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? if "read operation timed out" not in str(e): # SSL errors related to framing/MAC get wrapped and reraised here - raise SSLError(e) + raise SSLError(e) from e - raise ReadTimeoutError(self._pool, None, "Read timed out.") + raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] - except (HTTPException, SocketError) as e: - # This includes IncompleteRead. - raise ProtocolError("Connection broken: %r" % e, e) + except IncompleteRead as e: + if ( + e.expected is not None + and e.partial is not None + and e.expected == -e.partial + ): + arg = "Response may not contain content." + else: + arg = f"Connection broken: {e!r}" + raise ProtocolError(arg, e) from e + + except (HTTPException, OSError) as e: + raise ProtocolError(f"Connection broken: {e!r}", e) from e # If no exception is thrown, we should avoid cleaning up # unnecessarily. @@ -478,7 +947,12 @@ class HTTPResponse(io.IOBase): if self._original_response and self._original_response.isclosed(): self.release_conn() - def _fp_read(self, amt): + def _fp_read( + self, + amt: int | None = None, + *, + read1: bool = False, + ) -> bytes: """ Read a response with the thought that reading the number of bytes larger than can fit in a 32-bit int at a time via SSL in some @@ -487,21 +961,23 @@ class HTTPResponse(io.IOBase): happen. The known cases: - * 3.8 <= CPython < 3.9.7 because of a bug + * CPython < 3.9.7 because of a bug https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900. * urllib3 injected with pyOpenSSL-backed SSL-support. * CPython < 3.10 only when `amt` does not fit 32-bit int. """ assert self._fp - c_int_max = 2 ** 31 - 1 + c_int_max = 2**31 - 1 if ( - ( - (amt and amt > c_int_max) - or (self.length_remaining and self.length_remaining > c_int_max) + (amt and amt > c_int_max) + or ( + amt is None + and self.length_remaining + and self.length_remaining > c_int_max ) - and not util.IS_SECURETRANSPORT - and (util.IS_PYOPENSSL or sys.version_info < (3, 10)) - ): + ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)): + if read1: + return self._fp.read1(c_int_max) buffer = io.BytesIO() # Besides `max_chunk_amt` being a maximum chunk size, it # affects memory overhead of reading a response by this @@ -509,7 +985,7 @@ class HTTPResponse(io.IOBase): # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum # chunk size that does not lead to an overflow error, but # 256 MiB is a compromise. - max_chunk_amt = 2 ** 28 + max_chunk_amt = 2**28 while amt is None or amt != 0: if amt is not None: chunk_amt = min(amt, max_chunk_amt) @@ -522,11 +998,70 @@ class HTTPResponse(io.IOBase): buffer.write(data) del data # to reduce peak memory usage by `max_chunk_amt`. return buffer.getvalue() + elif read1: + return self._fp.read1(amt) if amt is not None else self._fp.read1() else: # StringIO doesn't like amt=None return self._fp.read(amt) if amt is not None else self._fp.read() - def read(self, amt=None, decode_content=None, cache_content=False): + def _raw_read( + self, + amt: int | None = None, + *, + read1: bool = False, + ) -> bytes: + """ + Reads `amt` of bytes from the socket. + """ + if self._fp is None: + return None # type: ignore[return-value] + + fp_closed = getattr(self._fp, "closed", False) + + with self._error_catcher(): + data = self._fp_read(amt, read1=read1) if not fp_closed else b"" + if amt is not None and amt != 0 and not data: + # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do + # not properly close the connection in all cases. There is + # no harm in redundantly calling close. + self._fp.close() + if ( + self.enforce_content_length + and self.length_remaining is not None + and self.length_remaining != 0 + ): + # This is an edge case that httplib failed to cover due + # to concerns of backward compatibility. We're + # addressing it here to make sure IncompleteRead is + # raised during streaming, so all calls with incorrect + # Content-Length are caught. + raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + elif read1 and ( + (amt != 0 and not data) or self.length_remaining == len(data) + ): + # All data has been read, but `self._fp.read1` in + # CPython 3.12 and older doesn't always close + # `http.client.HTTPResponse`, so we close it here. + # See https://github.com/python/cpython/issues/113199 + self._fp.close() + + if data: + self._fp_bytes_read += len(data) + if self.length_remaining is not None: + self.length_remaining -= len(data) + return data + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, + cache_content: bool = False, + ) -> bytes: """ Similar to :meth:`http.client.HTTPResponse.read`, but with two additional parameters: ``decode_content`` and ``cache_content``. @@ -551,54 +1086,145 @@ class HTTPResponse(io.IOBase): if decode_content is None: decode_content = self.decode_content - if self._fp is None: - return + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + elif amt is not None: + cache_content = False - flush_decoder = False - fp_closed = getattr(self._fp, "closed", False) + if self._decoder and self._decoder.has_unconsumed_tail: + decoded_data = self._decode( + b"", + decode_content, + flush_decoder=False, + max_length=amt - len(self._decoded_buffer), + ) + self._decoded_buffer.put(decoded_data) + if len(self._decoded_buffer) >= amt: + return self._decoded_buffer.get(amt) - with self._error_catcher(): - data = self._fp_read(amt) if not fp_closed else b"" - if amt is None: - flush_decoder = True - else: - cache_content = False - if ( - amt != 0 and not data - ): # Platform-specific: Buggy versions of Python. - # Close the connection when no data is returned - # - # This is redundant to what httplib/http.client _should_ - # already do. However, versions of python released before - # December 15, 2012 (http://bugs.python.org/issue16298) do - # not properly close the connection in all cases. There is - # no harm in redundantly calling close. - self._fp.close() - flush_decoder = True - if self.enforce_content_length and self.length_remaining not in ( - 0, - None, - ): - # This is an edge case that httplib failed to cover due - # to concerns of backward compatibility. We're - # addressing it here to make sure IncompleteRead is - # raised during streaming, so all calls with incorrect - # Content-Length are caught. - raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + data = self._raw_read(amt) - if data: - self._fp_bytes_read += len(data) - if self.length_remaining is not None: - self.length_remaining -= len(data) + flush_decoder = amt is None or (amt != 0 and not data) - data = self._decode(data, decode_content, flush_decoder) + if ( + not data + and len(self._decoded_buffer) == 0 + and not (self._decoder and self._decoder.has_unconsumed_tail) + ): + return data + if amt is None: + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data + else: + # do not waste memory on buffer when not decoding + if not decode_content: + if self._has_decoded_content: + raise RuntimeError( + "Calling read(decode_content=False) is not supported after " + "read(decode_content=True) was called." + ) + return data + + decoded_data = self._decode( + data, + decode_content, + flush_decoder, + max_length=amt - len(self._decoded_buffer), + ) + self._decoded_buffer.put(decoded_data) + + while len(self._decoded_buffer) < amt and data: + # TODO make sure to initially read enough data to get past the headers + # For example, the GZ file header takes 10 bytes, we don't want to read + # it one byte at a time + data = self._raw_read(amt) + decoded_data = self._decode( + data, + decode_content, + flush_decoder, + max_length=amt - len(self._decoded_buffer), + ) + self._decoded_buffer.put(decoded_data) + data = self._decoded_buffer.get(amt) return data - def stream(self, amt=2 ** 16, decode_content=None): + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + """ + Similar to ``http.client.HTTPResponse.read1`` and documented + in :meth:`io.BufferedReader.read1`, but with an additional parameter: + ``decode_content``. + + :param amt: + How much of the content to read. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + if decode_content is None: + decode_content = self.decode_content + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + # try and respond without going to the network + if self._has_decoded_content: + if not decode_content: + raise RuntimeError( + "Calling read1(decode_content=False) is not supported after " + "read1(decode_content=True) was called." + ) + if ( + self._decoder + and self._decoder.has_unconsumed_tail + and (amt is None or len(self._decoded_buffer) < amt) + ): + decoded_data = self._decode( + b"", + decode_content, + flush_decoder=False, + max_length=( + amt - len(self._decoded_buffer) if amt is not None else None + ), + ) + self._decoded_buffer.put(decoded_data) + if len(self._decoded_buffer) > 0: + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + if amt == 0: + return b"" + + # FIXME, this method's type doesn't say returning None is possible + data = self._raw_read(amt, read1=True) + if not decode_content or data is None: + return data + + self._init_decoder() + while True: + flush_decoder = not data + decoded_data = self._decode( + data, decode_content, flush_decoder, max_length=amt + ) + self._decoded_buffer.put(decoded_data) + if decoded_data or flush_decoder: + break + data = self._raw_read(8192, read1=True) + + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes]: """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -615,73 +1241,35 @@ class HTTPResponse(io.IOBase): 'content-encoding' header. """ if self.chunked and self.supports_chunked_reads(): - for line in self.read_chunked(amt, decode_content=decode_content): - yield line + yield from self.read_chunked(amt, decode_content=decode_content) else: - while not is_fp_closed(self._fp): + while ( + not is_fp_closed(self._fp) + or len(self._decoded_buffer) > 0 + or (self._decoder and self._decoder.has_unconsumed_tail) + ): data = self.read(amt=amt, decode_content=decode_content) if data: yield data - @classmethod - def from_httplib(ResponseCls, r, **response_kw): - """ - Given an :class:`http.client.HTTPResponse` instance ``r``, return a - corresponding :class:`urllib3.response.HTTPResponse` object. - - Remaining parameters are passed to the HTTPResponse constructor, along - with ``original_response=r``. - """ - headers = r.msg - - if not isinstance(headers, HTTPHeaderDict): - if six.PY2: - # Python 2.7 - headers = HTTPHeaderDict.from_httplib(headers) - else: - headers = HTTPHeaderDict(headers.items()) - - # HTTPResponse objects in Python 3 don't have a .strict attribute - strict = getattr(r, "strict", 0) - resp = ResponseCls( - body=r, - headers=headers, - status=r.status, - version=r.version, - reason=r.reason, - strict=strict, - original_response=r, - **response_kw - ) - return resp - - # Backwards-compatibility methods for http.client.HTTPResponse - def getheaders(self): - warnings.warn( - "HTTPResponse.getheaders() is deprecated and will be removed " - "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.", - category=DeprecationWarning, - stacklevel=2, - ) - return self.headers + # Overrides from io.IOBase + def readable(self) -> bool: + return True - def getheader(self, name, default=None): - warnings.warn( - "HTTPResponse.getheader() is deprecated and will be removed " - "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).", - category=DeprecationWarning, - stacklevel=2, - ) - return self.headers.get(name, default) + def shutdown(self) -> None: + if not self._sock_shutdown: + raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set") + if self._connection is None: + raise RuntimeError( + "Cannot shutdown as connection has already been released to the pool" + ) + self._sock_shutdown(socket.SHUT_RD) - # Backwards compatibility for http.cookiejar - def info(self): - return self.headers + def close(self) -> None: + self._sock_shutdown = None - # Overrides from io.IOBase - def close(self): - if not self.closed: + if not self.closed and self._fp: self._fp.close() if self._connection: @@ -691,9 +1279,9 @@ class HTTPResponse(io.IOBase): io.IOBase.close(self) @property - def closed(self): + def closed(self) -> bool: if not self.auto_close: - return io.IOBase.closed.__get__(self) + return io.IOBase.closed.__get__(self) # type: ignore[no-any-return] elif self._fp is None: return True elif hasattr(self._fp, "isclosed"): @@ -703,18 +1291,18 @@ class HTTPResponse(io.IOBase): else: return True - def fileno(self): + def fileno(self) -> int: if self._fp is None: - raise IOError("HTTPResponse has no file to get a fileno from") + raise OSError("HTTPResponse has no file to get a fileno from") elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError( + raise OSError( "The file-like object this HTTPResponse is wrapped " "around has no file descriptor" ) - def flush(self): + def flush(self) -> None: if ( self._fp is not None and hasattr(self._fp, "flush") @@ -722,20 +1310,7 @@ class HTTPResponse(io.IOBase): ): return self._fp.flush() - def readable(self): - # This method is required for `io` module compatibility. - return True - - def readinto(self, b): - # This method is required for `io` module compatibility. - temp = self.read(len(b)) - if len(temp) == 0: - return 0 - else: - b[: len(temp)] = temp - return len(temp) - - def supports_chunked_reads(self): + def supports_chunked_reads(self) -> bool: """ Checks if the underlying file-like object looks like a :class:`http.client.HTTPResponse` object. We do this by testing for @@ -744,43 +1319,49 @@ class HTTPResponse(io.IOBase): """ return hasattr(self._fp, "fp") - def _update_chunk_length(self): + def _update_chunk_length(self) -> None: # First, we'll figure out length of a chunk and then # we'll try to read it from socket. if self.chunk_left is not None: - return - line = self._fp.fp.readline() + return None + line = self._fp.fp.readline() # type: ignore[union-attr] line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) except ValueError: - # Invalid chunked protocol response, abort. self.close() - raise InvalidChunkLength(self, line) + if line: + # Invalid chunked protocol response, abort. + raise InvalidChunkLength(self, line) from None + else: + # Truncated at start of next chunk + raise ProtocolError("Response ended prematurely") from None - def _handle_chunk(self, amt): + def _handle_chunk(self, amt: int | None) -> bytes: returned_chunk = None if amt is None: - chunk = self._fp._safe_read(self.chunk_left) + chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] returned_chunk = chunk - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None - elif amt < self.chunk_left: - value = self._fp._safe_read(amt) + elif self.chunk_left is not None and amt < self.chunk_left: + value = self._fp._safe_read(amt) # type: ignore[union-attr] self.chunk_left = self.chunk_left - amt returned_chunk = value elif amt == self.chunk_left: - value = self._fp._safe_read(amt) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + value = self._fp._safe_read(amt) # type: ignore[union-attr] + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None returned_chunk = value else: # amt > self.chunk_left - returned_chunk = self._fp._safe_read(self.chunk_left) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None - return returned_chunk + return returned_chunk # type: ignore[no-any-return] - def read_chunked(self, amt=None, decode_content=None): + def read_chunked( + self, amt: int | None = None, decode_content: bool | None = None + ) -> typing.Generator[bytes]: """ Similar to :meth:`HTTPResponse.read`, but with an additional parameter: ``decode_content``. @@ -811,20 +1392,32 @@ class HTTPResponse(io.IOBase): # Don't bother reading the body of a HEAD request. if self._original_response and is_response_to_head(self._original_response): self._original_response.close() - return + return None # If a response is already read and closed # then return immediately. - if self._fp.fp is None: - return + if self._fp.fp is None: # type: ignore[union-attr] + return None + + if amt and amt < 0: + # Negative numbers and `None` should be treated the same, + # but httplib handles only `None` correctly. + amt = None while True: - self._update_chunk_length() - if self.chunk_left == 0: - break - chunk = self._handle_chunk(amt) + # First, check if any data is left in the decoder's buffer. + if self._decoder and self._decoder.has_unconsumed_tail: + chunk = b"" + else: + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) decoded = self._decode( - chunk, decode_content=decode_content, flush_decoder=False + chunk, + decode_content=decode_content, + flush_decoder=False, + max_length=amt, ) if decoded: yield decoded @@ -838,7 +1431,7 @@ class HTTPResponse(io.IOBase): yield decoded # Chunk content ends with \r\n: discard it. - while True: + while self._fp is not None: line = self._fp.fp.readline() if not line: # Some sites may not end with '\r\n'. @@ -850,27 +1443,29 @@ class HTTPResponse(io.IOBase): if self._original_response: self._original_response.close() - def geturl(self): + @property + def url(self) -> str | None: """ Returns the URL that was the source of this response. If the request that generated this response redirected, this method will return the final redirect location. """ - if self.retries is not None and len(self.retries.history): - return self.retries.history[-1].redirect_location - else: - return self._request_url + return self._request_url + + @url.setter + def url(self, url: str | None) -> None: + self._request_url = url - def __iter__(self): - buffer = [] + def __iter__(self) -> typing.Iterator[bytes]: + buffer: list[bytes] = [] for chunk in self.stream(decode_content=True): if b"\n" in chunk: - chunk = chunk.split(b"\n") - yield b"".join(buffer) + chunk[0] + b"\n" - for x in chunk[1:-1]: + chunks = chunk.split(b"\n") + yield b"".join(buffer) + chunks[0] + b"\n" + for x in chunks[1:-1]: yield x + b"\n" - if chunk[-1]: - buffer = [chunk[-1]] + if chunks[-1]: + buffer = [chunks[-1]] else: buffer = [] else: diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/__init__.py b/contrib/python/pip/pip/_vendor/urllib3/util/__init__.py index 4547fc522b6..534126033c0 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/__init__.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/__init__.py @@ -1,46 +1,39 @@ -from __future__ import absolute_import - # For backwards compatibility, provide imports that used to be here. +from __future__ import annotations + from .connection import is_connection_dropped from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers from .response import is_fp_closed from .retry import Retry from .ssl_ import ( ALPN_PROTOCOLS, - HAS_SNI, IS_PYOPENSSL, - IS_SECURETRANSPORT, - PROTOCOL_TLS, SSLContext, assert_fingerprint, + create_urllib3_context, resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, ) -from .timeout import Timeout, current_time -from .url import Url, get_host, parse_url, split_first +from .timeout import Timeout +from .url import Url, parse_url from .wait import wait_for_read, wait_for_write __all__ = ( - "HAS_SNI", "IS_PYOPENSSL", - "IS_SECURETRANSPORT", "SSLContext", - "PROTOCOL_TLS", "ALPN_PROTOCOLS", "Retry", "Timeout", "Url", "assert_fingerprint", - "current_time", + "create_urllib3_context", "is_connection_dropped", "is_fp_closed", - "get_host", "parse_url", "make_headers", "resolve_cert_reqs", "resolve_ssl_version", - "split_first", "ssl_wrap_socket", "wait_for_read", "wait_for_write", diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/connection.py b/contrib/python/pip/pip/_vendor/urllib3/util/connection.py index 6af1138f260..f92519ee912 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/connection.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/connection.py @@ -1,33 +1,23 @@ -from __future__ import absolute_import +from __future__ import annotations import socket +import typing -from ..contrib import _appengine_environ from ..exceptions import LocationParseError -from ..packages import six -from .wait import NoWayToWaitForSocketError, wait_for_read +from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]] -def is_connection_dropped(conn): # Platform-specific - """ - Returns True if the connection is dropped and should be closed. +if typing.TYPE_CHECKING: + from .._base_connection import BaseHTTPConnection - :param conn: - :class:`http.client.HTTPConnection` object. - Note: For platforms like AppEngine, this will always return ``False`` to - let the platform handle connection recycling transparently for us. +def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific """ - sock = getattr(conn, "sock", False) - if sock is False: # Platform-specific: AppEngine - return False - if sock is None: # Connection already closed (such as by httplib). - return True - try: - # Returns True if readable, which here means it's been dropped - return wait_for_read(sock, timeout=0.0) - except NoWayToWaitForSocketError: # Platform-specific: AppEngine - return False + Returns True if the connection is dropped and should be closed. + :param conn: :class:`urllib3.connection.HTTPConnection` object. + """ + return not conn.is_connected # This function is copied from socket.py in the Python 2.7 standard @@ -35,11 +25,11 @@ def is_connection_dropped(conn): # Platform-specific # One additional modification is that we avoid binding to IPv6 servers # discovered in DNS if the system doesn't have IPv6 functionality. def create_connection( - address, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, - socket_options=None, -): + address: tuple[str, int], + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + socket_options: _TYPE_SOCKET_OPTIONS | None = None, +) -> socket.socket: """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -65,9 +55,7 @@ def create_connection( try: host.encode("idna") except UnicodeError: - return six.raise_from( - LocationParseError(u"'%s', label empty or too long" % host), None - ) + raise LocationParseError(f"'{host}', label empty or too long") from None for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res @@ -78,26 +66,33 @@ def create_connection( # If provided, set socket level options before connecting. _set_socket_options(sock, socket_options) - if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + if timeout is not _DEFAULT_TIMEOUT: sock.settimeout(timeout) if source_address: sock.bind(source_address) sock.connect(sa) + # Break explicitly a reference cycle + err = None return sock - except socket.error as e: - err = e + except OSError as _: + err = _ if sock is not None: sock.close() - sock = None if err is not None: - raise err - - raise socket.error("getaddrinfo returns an empty list") + try: + raise err + finally: + # Break explicitly a reference cycle + err = None + else: + raise OSError("getaddrinfo returns an empty list") -def _set_socket_options(sock, options): +def _set_socket_options( + sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None +) -> None: if options is None: return @@ -105,7 +100,7 @@ def _set_socket_options(sock, options): sock.setsockopt(*opt) -def allowed_gai_family(): +def allowed_gai_family() -> socket.AddressFamily: """This function is designed to work in the context of getaddrinfo, where family=socket.AF_UNSPEC is the default and will perform a DNS search for both IPv6 and IPv4 records.""" @@ -116,18 +111,11 @@ def allowed_gai_family(): return family -def _has_ipv6(host): +def _has_ipv6(host: str) -> bool: """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False - # App Engine doesn't support IPV6 sockets and actually has a quota on the - # number of sockets that can be used, so just early out here instead of - # creating a socket needlessly. - # See https://github.com/urllib3/urllib3/issues/1446 - if _appengine_environ.is_appengine_sandbox(): - return False - if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/proxy.py b/contrib/python/pip/pip/_vendor/urllib3/util/proxy.py index 2199cc7b7f0..908fc6621d0 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/proxy.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/proxy.py @@ -1,9 +1,18 @@ -from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version +from __future__ import annotations + +import typing + +from .url import Url + +if typing.TYPE_CHECKING: + from ..connection import ProxyConfig def connection_requires_http_tunnel( - proxy_url=None, proxy_config=None, destination_scheme=None -): + proxy_url: Url | None = None, + proxy_config: ProxyConfig | None = None, + destination_scheme: str | None = None, +) -> bool: """ Returns True if the connection requires an HTTP CONNECT through the proxy. @@ -32,26 +41,3 @@ def connection_requires_http_tunnel( # Otherwise always use a tunnel. return True - - -def create_proxy_ssl_context( - ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None -): - """ - Generates a default proxy ssl context if one hasn't been provided by the - user. - """ - ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(ssl_version), - cert_reqs=resolve_cert_reqs(cert_reqs), - ) - - if ( - not ca_certs - and not ca_cert_dir - and not ca_cert_data - and hasattr(ssl_context, "load_default_certs") - ): - ssl_context.load_default_certs() - - return ssl_context diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/queue.py b/contrib/python/pip/pip/_vendor/urllib3/util/queue.py deleted file mode 100644 index 41784104ee4..00000000000 --- a/contrib/python/pip/pip/_vendor/urllib3/util/queue.py +++ /dev/null @@ -1,22 +0,0 @@ -import collections - -from ..packages import six -from ..packages.six.moves import queue - -if six.PY2: - # Queue is imported for side effects on MS Windows. See issue #229. - import Queue as _unused_module_Queue # noqa: F401 - - -class LifoQueue(queue.Queue): - def _init(self, _): - self.queue = collections.deque() - - def _qsize(self, len=len): - return len(self.queue) - - def _put(self, item): - self.queue.append(item) - - def _get(self): - return self.queue.pop() diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/request.py b/contrib/python/pip/pip/_vendor/urllib3/util/request.py index 330766ef4f3..b0d271777bd 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/request.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/request.py @@ -1,9 +1,16 @@ -from __future__ import absolute_import +from __future__ import annotations +import io +import sys +import typing from base64 import b64encode +from enum import Enum from ..exceptions import UnrewindableBodyError -from ..packages.six import b, integer_types +from .util import to_bytes + +if typing.TYPE_CHECKING: + from typing import Final # Pass as a value within ``headers`` to skip # emitting some HTTP headers that are added automatically. @@ -14,17 +21,41 @@ SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"]) ACCEPT_ENCODING = "gzip,deflate" -_FAILEDTELL = object() +try: + if sys.version_info >= (3, 14): + from compression import zstd as _unused_module_zstd # noqa: F401 + else: + from backports import zstd as _unused_module_zstd # noqa: F401 +except ImportError: + pass +else: + ACCEPT_ENCODING += ",zstd" + + +class _TYPE_FAILEDTELL(Enum): + token = 0 + + +_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token + +_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL] + +# When sending a request with these methods we aren't expecting +# a body so don't need to set an explicit 'Content-Length: 0' +# The reason we do this in the negative instead of tracking methods +# which 'should' have a body is because unknown methods should be +# treated as if they were 'POST' which *does* expect a body. +_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"} def make_headers( - keep_alive=None, - accept_encoding=None, - user_agent=None, - basic_auth=None, - proxy_basic_auth=None, - disable_cache=None, -): + keep_alive: bool | None = None, + accept_encoding: bool | list[str] | str | None = None, + user_agent: str | None = None, + basic_auth: str | None = None, + proxy_basic_auth: str | None = None, + disable_cache: bool | None = None, +) -> dict[str, str]: """ Shortcuts for generating request headers. @@ -33,7 +64,11 @@ def make_headers( :param accept_encoding: Can be a boolean, list, or string. - ``True`` translates to 'gzip,deflate'. + ``True`` translates to 'gzip,deflate'. If the dependencies for + Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or + Zstandard (the ``backports.zstd`` package for Python before 3.14) + algorithms are installed, then their encodings are + included in the string ('br' and 'zstd', respectively). List will get joined by comma. String will be used as provided. @@ -52,14 +87,18 @@ def make_headers( :param disable_cache: If ``True``, adds 'cache-control: no-cache' header. - Example:: + Example: - >>> make_headers(keep_alive=True, user_agent="Batman/1.0") - {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} - >>> make_headers(accept_encoding=True) - {'accept-encoding': 'gzip,deflate'} + .. code-block:: python + + from pip._vendor import urllib3 + + print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0")) + # {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + print(urllib3.util.make_headers(accept_encoding=True)) + # {'accept-encoding': 'gzip,deflate'} """ - headers = {} + headers: dict[str, str] = {} if accept_encoding: if isinstance(accept_encoding, str): pass @@ -76,12 +115,14 @@ def make_headers( headers["connection"] = "keep-alive" if basic_auth: - headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") + headers["authorization"] = ( + f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}" + ) if proxy_basic_auth: - headers["proxy-authorization"] = "Basic " + b64encode( - b(proxy_basic_auth) - ).decode("utf-8") + headers["proxy-authorization"] = ( + f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}" + ) if disable_cache: headers["cache-control"] = "no-cache" @@ -89,7 +130,9 @@ def make_headers( return headers -def set_file_position(body, pos): +def set_file_position( + body: typing.Any, pos: _TYPE_BODY_POSITION | None +) -> _TYPE_BODY_POSITION | None: """ If a position is provided, move file to that point. Otherwise, we'll attempt to record a position for future use. @@ -99,7 +142,7 @@ def set_file_position(body, pos): elif getattr(body, "tell", None) is not None: try: pos = body.tell() - except (IOError, OSError): + except OSError: # This differentiates from None, allowing us to catch # a failed `tell()` later when trying to rewind the body. pos = _FAILEDTELL @@ -107,7 +150,7 @@ def set_file_position(body, pos): return pos -def rewind_body(body, body_pos): +def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None: """ Attempt to rewind body to a certain position. Primarily used for request redirects and retries. @@ -119,13 +162,13 @@ def rewind_body(body, body_pos): Position to seek to in file. """ body_seek = getattr(body, "seek", None) - if body_seek is not None and isinstance(body_pos, integer_types): + if body_seek is not None and isinstance(body_pos, int): try: body_seek(body_pos) - except (IOError, OSError): + except OSError as e: raise UnrewindableBodyError( "An error occurred when rewinding request body for redirect/retry." - ) + ) from e elif body_pos is _FAILEDTELL: raise UnrewindableBodyError( "Unable to record file position for rewinding " @@ -133,5 +176,79 @@ def rewind_body(body, body_pos): ) else: raise ValueError( - "body_pos must be of type integer, instead it was %s." % type(body_pos) + f"body_pos must be of type integer, instead it was {type(body_pos)}." ) + + +class ChunksAndContentLength(typing.NamedTuple): + chunks: typing.Iterable[bytes] | None + content_length: int | None + + +def body_to_chunks( + body: typing.Any | None, method: str, blocksize: int +) -> ChunksAndContentLength: + """Takes the HTTP request method, body, and blocksize and + transforms them into an iterable of chunks to pass to + socket.sendall() and an optional 'Content-Length' header. + + A 'Content-Length' of 'None' indicates the length of the body + can't be determined so should use 'Transfer-Encoding: chunked' + for framing instead. + """ + + chunks: typing.Iterable[bytes] | None + content_length: int | None + + # No body, we need to make a recommendation on 'Content-Length' + # based on whether that request method is expected to have + # a body or not. + if body is None: + chunks = None + if method.upper() not in _METHODS_NOT_EXPECTING_BODY: + content_length = 0 + else: + content_length = None + + # Bytes or strings become bytes + elif isinstance(body, (str, bytes)): + chunks = (to_bytes(body),) + content_length = len(chunks[0]) + + # File-like object, TODO: use seek() and tell() for length? + elif hasattr(body, "read"): + + def chunk_readable() -> typing.Iterable[bytes]: + encode = isinstance(body, io.TextIOBase) + while True: + datablock = body.read(blocksize) + if not datablock: + break + if encode: + datablock = datablock.encode("utf-8") + yield datablock + + chunks = chunk_readable() + content_length = None + + # Otherwise we need to start checking via duck-typing. + else: + try: + # Check if the body implements the buffer API. + mv = memoryview(body) + except TypeError: + try: + # Check if the body is an iterable + chunks = iter(body) + content_length = None + except TypeError: + raise TypeError( + f"'body' must be a bytes-like object, file-like " + f"object, or iterable. Instead was {body!r}" + ) from None + else: + # Since it implements the buffer API can be passed directly to socket.sendall() + chunks = (body,) + content_length = mv.nbytes + + return ChunksAndContentLength(chunks=chunks, content_length=content_length) diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/response.py b/contrib/python/pip/pip/_vendor/urllib3/util/response.py index 5ea609ccedf..0f4578696fa 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/response.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/response.py @@ -1,12 +1,12 @@ -from __future__ import absolute_import +from __future__ import annotations +import http.client as httplib from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect from ..exceptions import HeaderParsingError -from ..packages.six.moves import http_client as httplib -def is_fp_closed(obj): +def is_fp_closed(obj: object) -> bool: """ Checks whether a given file-like object is closed. @@ -17,27 +17,27 @@ def is_fp_closed(obj): try: # Check `isclosed()` first, in case Python3 doesn't set `closed`. # GH Issue #928 - return obj.isclosed() + return obj.isclosed() # type: ignore[no-any-return, attr-defined] except AttributeError: pass try: # Check via the official file-like-object way. - return obj.closed + return obj.closed # type: ignore[no-any-return, attr-defined] except AttributeError: pass try: # Check if the object is a container for another file-like object that # gets released on exhaustion (e.g. HTTPResponse). - return obj.fp is None + return obj.fp is None # type: ignore[attr-defined] except AttributeError: pass raise ValueError("Unable to determine whether fp is closed.") -def assert_header_parsing(headers): +def assert_header_parsing(headers: httplib.HTTPMessage) -> None: """ Asserts whether all headers have been successfully parsed. Extracts encountered errors from the result of parsing headers. @@ -53,55 +53,49 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) - - defects = getattr(headers, "defects", None) - get_payload = getattr(headers, "get_payload", None) + raise TypeError(f"expected httplib.Message, got {type(headers)}.") unparsed_data = None - if get_payload: - # get_payload is actually email.message.Message.get_payload; - # we're only interested in the result if it's not a multipart message - if not headers.is_multipart(): - payload = get_payload() - if isinstance(payload, (bytes, str)): - unparsed_data = payload - if defects: - # httplib is assuming a response body is available - # when parsing headers even when httplib only sends - # header data to parse_headers() This results in - # defects on multipart responses in particular. - # See: https://github.com/urllib3/urllib3/issues/800 + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = headers.get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload + + # httplib is assuming a response body is available + # when parsing headers even when httplib only sends + # header data to parse_headers() This results in + # defects on multipart responses in particular. + # See: https://github.com/urllib3/urllib3/issues/800 - # So we ignore the following defects: - # - StartBoundaryNotFoundDefect: - # The claimed start boundary was never found. - # - MultipartInvariantViolationDefect: - # A message claimed to be a multipart but no subparts were found. - defects = [ - defect - for defect in defects - if not isinstance( - defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect) - ) - ] + # So we ignore the following defects: + # - StartBoundaryNotFoundDefect: + # The claimed start boundary was never found. + # - MultipartInvariantViolationDefect: + # A message claimed to be a multipart but no subparts were found. + defects = [ + defect + for defect in headers.defects + if not isinstance( + defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect) + ) + ] if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) -def is_response_to_head(response): +def is_response_to_head(response: httplib.HTTPResponse) -> bool: """ Checks whether the request of a response has been a HEAD-request. - Handles the quirks of AppEngine. :param http.client.HTTPResponse response: Response to check if the originating request used 'HEAD' as a method. """ # FIXME: Can we do this somehow without accessing private httplib _method? - method = response._method - if isinstance(method, int): # Platform-specific: Appengine - return method == 3 - return method.upper() == "HEAD" + method_str = response._method # type: str # type: ignore[attr-defined] + return method_str.upper() == "HEAD" diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/retry.py b/contrib/python/pip/pip/_vendor/urllib3/util/retry.py index 9a1e90d0b23..b21b4b64ebb 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/retry.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/retry.py @@ -1,12 +1,13 @@ -from __future__ import absolute_import +from __future__ import annotations import email import logging +import random import re import time -import warnings -from collections import namedtuple +import typing from itertools import takewhile +from types import TracebackType from ..exceptions import ( ConnectTimeoutError, @@ -17,97 +18,51 @@ from ..exceptions import ( ReadTimeoutError, ResponseError, ) -from ..packages import six - -log = logging.getLogger(__name__) - - -# Data structure for representing the metadata of requests that result in a retry. -RequestHistory = namedtuple( - "RequestHistory", ["method", "url", "error", "status", "redirect_location"] -) +from .util import reraise +if typing.TYPE_CHECKING: + from typing_extensions import Self -# TODO: In v2 we can remove this sentinel and metaclass with deprecated options. -_Default = object() - - -class _RetryMeta(type): - @property - def DEFAULT_METHOD_WHITELIST(cls): - warnings.warn( - "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", - DeprecationWarning, - ) - return cls.DEFAULT_ALLOWED_METHODS - - @DEFAULT_METHOD_WHITELIST.setter - def DEFAULT_METHOD_WHITELIST(cls, value): - warnings.warn( - "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", - DeprecationWarning, - ) - cls.DEFAULT_ALLOWED_METHODS = value + from ..connectionpool import ConnectionPool + from ..response import BaseHTTPResponse - @property - def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls): - warnings.warn( - "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", - DeprecationWarning, - ) - return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT - - @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter - def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value): - warnings.warn( - "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", - DeprecationWarning, - ) - cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value +log = logging.getLogger(__name__) - @property - def BACKOFF_MAX(cls): - warnings.warn( - "Using 'Retry.BACKOFF_MAX' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", - DeprecationWarning, - ) - return cls.DEFAULT_BACKOFF_MAX - @BACKOFF_MAX.setter - def BACKOFF_MAX(cls, value): - warnings.warn( - "Using 'Retry.BACKOFF_MAX' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", - DeprecationWarning, - ) - cls.DEFAULT_BACKOFF_MAX = value +# Data structure for representing the metadata of requests that result in a retry. +class RequestHistory(typing.NamedTuple): + method: str | None + url: str | None + error: Exception | None + status: int | None + redirect_location: str | None [email protected]_metaclass(_RetryMeta) -class Retry(object): +class Retry: """Retry configuration. Each retry attempt will create a new Retry object with updated values, so they can be safely reused. - Retries can be defined as a default for a pool:: + Retries can be defined as a default for a pool: + + .. code-block:: python retries = Retry(connect=5, read=2, redirect=5) http = PoolManager(retries=retries) - response = http.request('GET', 'http://example.com/') + response = http.request("GET", "https://example.com/") + + Or per-request (which overrides the default for the pool): + + .. code-block:: python - Or per-request (which overrides the default for the pool):: + response = http.request("GET", "https://example.com/", retries=Retry(10)) - response = http.request('GET', 'http://example.com/', retries=Retry(10)) + Retries can be disabled by passing ``False``: - Retries can be disabled by passing ``False``:: + .. code-block:: python - response = http.request('GET', 'http://example.com/', retries=False) + response = http.request("GET", "https://example.com/", retries=False) Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless retries are disabled, in which case the causing exception will be raised. @@ -169,21 +124,16 @@ class Retry(object): If ``total`` is not set, it's a good idea to set this to 0 to account for unexpected edge cases and avoid infinite retry loops. - :param iterable allowed_methods: + :param Collection allowed_methods: Set of uppercased HTTP method verbs that we should retry on. By default, we only retry on methods which are considered to be idempotent (multiple requests with the same parameters end with the same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. - Set to a ``False`` value to retry on any verb. + Set to a ``None`` value to retry on any verb. - .. warning:: - - Previously this parameter was named ``method_whitelist``, that - usage is deprecated in v1.26.0 and will be removed in v2.0. - - :param iterable status_forcelist: + :param Collection status_forcelist: A set of integer HTTP status codes that we should force a retry on. A retry is initiated if the request method is in ``allowed_methods`` and the response status code is in ``status_forcelist``. @@ -195,13 +145,17 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ** ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of previous retries})) + + seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: - seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep - for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer - than :attr:`Retry.DEFAULT_BACKOFF_MAX`. + random.uniform(0, {backoff jitter}) - By default, backoff is disabled (set to 0). + seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will + sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever + be longer than `backoff_max`. + + By default, backoff is disabled (factor set to 0). :param bool raise_on_redirect: Whether, if the number of redirects is exhausted, to raise a MaxRetryError, or to return a response with a @@ -220,10 +174,15 @@ class Retry(object): Whether to respect Retry-After header on status codes defined as :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. - :param iterable remove_headers_on_redirect: + :param Collection remove_headers_on_redirect: Sequence of headers to remove from the request when a response indicating a redirect is returned before firing off the redirected request. + + :param int retry_after_max: Number of seconds to allow as the maximum for + Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`. + Any Retry-After headers larger than this value will be limited to this + value. """ #: Default methods to be used for ``allowed_methods`` @@ -239,48 +198,38 @@ class Retry(object): ["Cookie", "Authorization", "Proxy-Authorization"] ) - #: Maximum backoff time. + #: Default maximum backoff time. DEFAULT_BACKOFF_MAX = 120 - def __init__( - self, - total=10, - connect=None, - read=None, - redirect=None, - status=None, - other=None, - allowed_methods=_Default, - status_forcelist=None, - backoff_factor=0, - raise_on_redirect=True, - raise_on_status=True, - history=None, - respect_retry_after_header=True, - remove_headers_on_redirect=_Default, - # TODO: Deprecated, remove in v2.0 - method_whitelist=_Default, - ): + # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries. + #: Default maximum allowed value for Retry-After headers in seconds + DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600 - if method_whitelist is not _Default: - if allowed_methods is not _Default: - raise ValueError( - "Using both 'allowed_methods' and " - "'method_whitelist' together is not allowed. " - "Instead only use 'allowed_methods'" - ) - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - stacklevel=2, - ) - allowed_methods = method_whitelist - if allowed_methods is _Default: - allowed_methods = self.DEFAULT_ALLOWED_METHODS - if remove_headers_on_redirect is _Default: - remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT + # Backward compatibility; assigned outside of the class. + DEFAULT: typing.ClassVar[Retry] + def __init__( + self, + total: bool | int | None = 10, + connect: int | None = None, + read: int | None = None, + redirect: bool | int | None = None, + status: int | None = None, + other: int | None = None, + allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, + status_forcelist: typing.Collection[int] | None = None, + backoff_factor: float = 0, + backoff_max: float = DEFAULT_BACKOFF_MAX, + raise_on_redirect: bool = True, + raise_on_status: bool = True, + history: tuple[RequestHistory, ...] | None = None, + respect_retry_after_header: bool = True, + remove_headers_on_redirect: typing.Collection[ + str + ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, + backoff_jitter: float = 0.0, + retry_after_max: int = DEFAULT_RETRY_AFTER_MAX, + ) -> None: self.total = total self.connect = connect self.read = read @@ -295,15 +244,18 @@ class Retry(object): self.status_forcelist = status_forcelist or set() self.allowed_methods = allowed_methods self.backoff_factor = backoff_factor + self.backoff_max = backoff_max + self.retry_after_max = retry_after_max self.raise_on_redirect = raise_on_redirect self.raise_on_status = raise_on_status - self.history = history or tuple() + self.history = history or () self.respect_retry_after_header = respect_retry_after_header self.remove_headers_on_redirect = frozenset( - [h.lower() for h in remove_headers_on_redirect] + h.lower() for h in remove_headers_on_redirect ) + self.backoff_jitter = backoff_jitter - def new(self, **kw): + def new(self, **kw: typing.Any) -> Self: params = dict( total=self.total, connect=self.connect, @@ -311,36 +263,29 @@ class Retry(object): redirect=self.redirect, status=self.status, other=self.other, + allowed_methods=self.allowed_methods, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, + backoff_max=self.backoff_max, + retry_after_max=self.retry_after_max, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, remove_headers_on_redirect=self.remove_headers_on_redirect, respect_retry_after_header=self.respect_retry_after_header, + backoff_jitter=self.backoff_jitter, ) - # TODO: If already given in **kw we use what's given to us - # If not given we need to figure out what to pass. We decide - # based on whether our class has the 'method_whitelist' property - # and if so we pass the deprecated 'method_whitelist' otherwise - # we use 'allowed_methods'. Remove in v2.0 - if "method_whitelist" not in kw and "allowed_methods" not in kw: - if "method_whitelist" in self.__dict__: - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - params["method_whitelist"] = self.allowed_methods - else: - params["allowed_methods"] = self.allowed_methods - params.update(kw) - return type(self)(**params) + return type(self)(**params) # type: ignore[arg-type] @classmethod - def from_int(cls, retries, redirect=True, default=None): + def from_int( + cls, + retries: Retry | bool | int | None, + redirect: bool | int | None = True, + default: Retry | bool | int | None = None, + ) -> Retry: """Backwards-compatibility for the old retries format.""" if retries is None: retries = default if default is not None else cls.DEFAULT @@ -353,7 +298,7 @@ class Retry(object): log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries - def get_backoff_time(self): + def get_backoff_time(self) -> float: """Formula for computing the current backoff :rtype: float @@ -368,32 +313,32 @@ class Retry(object): return 0 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) - return min(self.DEFAULT_BACKOFF_MAX, backoff_value) + if self.backoff_jitter != 0.0: + backoff_value += random.random() * self.backoff_jitter + return float(max(0, min(self.backoff_max, backoff_value))) - def parse_retry_after(self, retry_after): + def parse_retry_after(self, retry_after: str) -> float: + seconds: float # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 if re.match(r"^\s*[0-9]+\s*$", retry_after): seconds = int(retry_after) else: retry_date_tuple = email.utils.parsedate_tz(retry_after) if retry_date_tuple is None: - raise InvalidHeader("Invalid Retry-After header: %s" % retry_after) - if retry_date_tuple[9] is None: # Python 2 - # Assume UTC if no timezone was specified - # On Python2.7, parsedate_tz returns None for a timezone offset - # instead of 0 if no timezone is given, where mktime_tz treats - # a None timezone offset as local time. - retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:] + raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") retry_date = email.utils.mktime_tz(retry_date_tuple) seconds = retry_date - time.time() - if seconds < 0: - seconds = 0 + seconds = max(seconds, 0) + + # Check the seconds do not exceed the specified maximum + if seconds > self.retry_after_max: + seconds = self.retry_after_max return seconds - def get_retry_after(self, response): + def get_retry_after(self, response: BaseHTTPResponse) -> float | None: """Get the value of Retry-After in seconds.""" retry_after = response.headers.get("Retry-After") @@ -403,7 +348,7 @@ class Retry(object): return self.parse_retry_after(retry_after) - def sleep_for_retry(self, response=None): + def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: retry_after = self.get_retry_after(response) if retry_after: time.sleep(retry_after) @@ -411,13 +356,13 @@ class Retry(object): return False - def _sleep_backoff(self): + def _sleep_backoff(self) -> None: backoff = self.get_backoff_time() if backoff <= 0: return time.sleep(backoff) - def sleep(self, response=None): + def sleep(self, response: BaseHTTPResponse | None = None) -> None: """Sleep between retry attempts. This method will respect a server's ``Retry-After`` response header @@ -433,7 +378,7 @@ class Retry(object): self._sleep_backoff() - def _is_connection_error(self, err): + def _is_connection_error(self, err: Exception) -> bool: """Errors when we're fairly sure that the server did not receive the request, so it should be safe to retry. """ @@ -441,33 +386,23 @@ class Retry(object): err = err.original_error return isinstance(err, ConnectTimeoutError) - def _is_read_error(self, err): + def _is_read_error(self, err: Exception) -> bool: """Errors that occur after the request has been started, so we should assume that the server began processing it. """ return isinstance(err, (ReadTimeoutError, ProtocolError)) - def _is_method_retryable(self, method): + def _is_method_retryable(self, method: str) -> bool: """Checks if a given HTTP method should be retried upon, depending if it is included in the allowed_methods """ - # TODO: For now favor if the Retry implementation sets its own method_whitelist - # property outside of our constructor to avoid breaking custom implementations. - if "method_whitelist" in self.__dict__: - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - allowed_methods = self.method_whitelist - else: - allowed_methods = self.allowed_methods - - if allowed_methods and method.upper() not in allowed_methods: + if self.allowed_methods and method.upper() not in self.allowed_methods: return False return True - def is_retry(self, method, status_code, has_retry_after=False): + def is_retry( + self, method: str, status_code: int, has_retry_after: bool = False + ) -> bool: """Is this method/status code retryable? (Based on allowlists and control variables such as the number of total retries to allow, whether to respect the Retry-After header, whether this header is present, and @@ -480,24 +415,27 @@ class Retry(object): if self.status_forcelist and status_code in self.status_forcelist: return True - return ( + return bool( self.total and self.respect_retry_after_header and has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES) ) - def is_exhausted(self): + def is_exhausted(self) -> bool: """Are we out of retries?""" - retry_counts = ( - self.total, - self.connect, - self.read, - self.redirect, - self.status, - self.other, - ) - retry_counts = list(filter(None, retry_counts)) + retry_counts = [ + x + for x in ( + self.total, + self.connect, + self.read, + self.redirect, + self.status, + self.other, + ) + if x + ] if not retry_counts: return False @@ -505,18 +443,18 @@ class Retry(object): def increment( self, - method=None, - url=None, - response=None, - error=None, - _pool=None, - _stacktrace=None, - ): + method: str | None = None, + url: str | None = None, + response: BaseHTTPResponse | None = None, + error: Exception | None = None, + _pool: ConnectionPool | None = None, + _stacktrace: TracebackType | None = None, + ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. - :type response: :class:`~urllib3.response.HTTPResponse` + :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. @@ -524,7 +462,7 @@ class Retry(object): """ if self.total is False and error: # Disabled, indicate to re-raise the error. - raise six.reraise(type(error), error, _stacktrace) + raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: @@ -542,14 +480,14 @@ class Retry(object): if error and self._is_connection_error(error): # Connect retry? if connect is False: - raise six.reraise(type(error), error, _stacktrace) + raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? - if read is False or not self._is_method_retryable(method): - raise six.reraise(type(error), error, _stacktrace) + if read is False or method is None or not self._is_method_retryable(method): + raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 @@ -563,7 +501,9 @@ class Retry(object): if redirect is not None: redirect -= 1 cause = "too many redirects" - redirect_location = response.get_redirect_location() + response_redirect_location = response.get_redirect_location() + if response_redirect_location: + redirect_location = response_redirect_location status = response.status else: @@ -591,31 +531,18 @@ class Retry(object): ) if new_retry.is_exhausted(): - raise MaxRetryError(_pool, url, error or ResponseError(cause)) + reason = error or ResponseError(cause) + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry - def __repr__(self): + def __repr__(self) -> str: return ( - "{cls.__name__}(total={self.total}, connect={self.connect}, " - "read={self.read}, redirect={self.redirect}, status={self.status})" - ).format(cls=type(self), self=self) - - def __getattr__(self, item): - if item == "method_whitelist": - # TODO: Remove this deprecated alias in v2.0 - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - return self.allowed_methods - try: - return getattr(super(Retry, self), item) - except AttributeError: - return getattr(Retry, item) + f"{type(self).__name__}(total={self.total}, connect={self.connect}, " + f"read={self.read}, redirect={self.redirect}, status={self.status})" + ) # For backwards compatibility (equivalent to pre-v1.9): diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py index 0a6a0e06a0d..56fe9093ada 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py @@ -1,28 +1,25 @@ -from __future__ import absolute_import +from __future__ import annotations import hashlib import hmac import os +import socket import sys +import typing import warnings -from binascii import hexlify, unhexlify +from binascii import unhexlify -from ..exceptions import ( - InsecurePlatformWarning, - ProxySchemeUnsupported, - SNIMissingWarning, - SSLError, -) -from ..packages import six -from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE +from ..exceptions import ProxySchemeUnsupported, SSLError +from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE SSLContext = None SSLTransport = None -HAS_SNI = False +HAS_NEVER_CHECK_COMMON_NAME = False IS_PYOPENSSL = False -IS_SECURETRANSPORT = False ALPN_PROTOCOLS = ["http/1.1"] +_TYPE_VERSION_INFO = tuple[int, int, int, str, int] + # Maps the length of a digest to a possible hash function producing this digest HASHFUNC_MAP = { length: getattr(hashlib, algorithm, None) @@ -30,159 +27,129 @@ HASHFUNC_MAP = { } -def _const_compare_digest_backport(a, b): - """ - Compare two digests of equal length in constant time. +def _is_bpo_43522_fixed( + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, +) -> bool: + """Return True for CPython 3.9.3+ or 3.10+ and PyPy 7.3.8+ where + setting SSLContext.hostname_checks_common_name to False works. + + Outside of CPython and PyPy we don't know which implementations work + or not so we conservatively use our hostname matching as we know that works + on all implementations. - The digests must be of type str/bytes. - Returns True if the digests match, and False otherwise. + https://github.com/urllib3/urllib3/issues/2192#issuecomment-821832963 + https://foss.heptapod.net/pypy/pypy/-/issues/3539 """ - result = abs(len(a) - len(b)) - for left, right in zip(bytearray(a), bytearray(b)): - result |= left ^ right - return result == 0 + if implementation_name == "pypy": + # https://foss.heptapod.net/pypy/pypy/-/issues/3129 + return pypy_version_info >= (7, 3, 8) # type: ignore[operator] + elif implementation_name == "cpython": + major_minor = version_info[:2] + micro = version_info[2] + return (major_minor == (3, 9) and micro >= 3) or major_minor >= (3, 10) + else: # Defensive: + return False -_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) +def _is_has_never_check_common_name_reliable( + openssl_version: str, + openssl_version_number: int, + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, +) -> bool: + # As of May 2023, all released versions of LibreSSL fail to reject certificates with + # only common names, see https://github.com/urllib3/urllib3/pull/3024 + is_openssl = openssl_version.startswith("OpenSSL ") + # Before fixing OpenSSL issue #14579, the SSL_new() API was not copying hostflags + # like X509_CHECK_FLAG_NEVER_CHECK_SUBJECT, which tripped up CPython. + # https://github.com/openssl/openssl/issues/14579 + # This was released in OpenSSL 1.1.1l+ (>=0x101010cf) + is_openssl_issue_14579_fixed = openssl_version_number >= 0x101010CF -try: # Test for SSL features - import ssl - from ssl import CERT_REQUIRED, wrap_socket -except ImportError: - pass - -try: - from ssl import HAS_SNI # Has SNI? -except ImportError: - pass + return is_openssl and ( + is_openssl_issue_14579_fixed + or _is_bpo_43522_fixed(implementation_name, version_info, pypy_version_info) + ) -try: - from .ssltransport import SSLTransport -except ImportError: - pass +if typing.TYPE_CHECKING: + from ssl import VerifyMode + from typing import TypedDict -try: # Platform-specific: Python 3.6 - from ssl import PROTOCOL_TLS + from .ssltransport import SSLTransport as SSLTransportType - PROTOCOL_SSLv23 = PROTOCOL_TLS -except ImportError: - try: - from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS + class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False): + subjectAltName: tuple[tuple[str, str], ...] + subject: tuple[tuple[tuple[str, str], ...], ...] + serialNumber: str - PROTOCOL_SSLv23 = PROTOCOL_TLS - except ImportError: - PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 -try: - from ssl import PROTOCOL_TLS_CLIENT -except ImportError: - PROTOCOL_TLS_CLIENT = PROTOCOL_TLS +# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X' +_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {} +try: # Do we have ssl at all? + import ssl + from ssl import ( # type: ignore[assignment] + CERT_REQUIRED, + HAS_NEVER_CHECK_COMMON_NAME, + OP_NO_COMPRESSION, + OP_NO_TICKET, + OPENSSL_VERSION, + OPENSSL_VERSION_NUMBER, + PROTOCOL_TLS, + PROTOCOL_TLS_CLIENT, + VERIFY_X509_STRICT, + OP_NO_SSLv2, + OP_NO_SSLv3, + SSLContext, + TLSVersion, + ) -try: - from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 -except ImportError: - OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 - OP_NO_COMPRESSION = 0x20000 + PROTOCOL_SSLv23 = PROTOCOL_TLS + # Needed for Python 3.9 which does not define this + VERIFY_X509_PARTIAL_CHAIN = getattr(ssl, "VERIFY_X509_PARTIAL_CHAIN", 0x80000) -try: # OP_NO_TICKET was added in Python 3.6 - from ssl import OP_NO_TICKET -except ImportError: - OP_NO_TICKET = 0x4000 + # Setting SSLContext.hostname_checks_common_name = False didn't work before CPython + # 3.9.3, and 3.10 (but OK on PyPy) or OpenSSL 1.1.1l+ + if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable( + OPENSSL_VERSION, + OPENSSL_VERSION_NUMBER, + sys.implementation.name, + sys.version_info, + sys.pypy_version_info if sys.implementation.name == "pypy" else None, # type: ignore[attr-defined] + ): # Defensive: for Python < 3.9.3 + HAS_NEVER_CHECK_COMMON_NAME = False + # Need to be careful here in case old TLS versions get + # removed in future 'ssl' module implementations. + for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"): + try: + _SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr( + TLSVersion, attr + ) + except AttributeError: # Defensive: + continue -# A secure default. -# Sources for more information on TLS ciphers: -# -# - https://wiki.mozilla.org/Security/Server_Side_TLS -# - https://www.ssllabs.com/projects/best-practices/index.html -# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ -# -# The general intent is: -# - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), -# - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and -# security, -# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, -# - disable NULL authentication, MD5 MACs, DSS, and other -# insecure ciphers for security reasons. -# - NOTE: TLS 1.3 cipher suites are managed through a different interface -# not exposed by CPython (yet!) and are enabled by default if they're available. -DEFAULT_CIPHERS = ":".join( - [ - "ECDHE+AESGCM", - "ECDHE+CHACHA20", - "DHE+AESGCM", - "DHE+CHACHA20", - "ECDH+AESGCM", - "DH+AESGCM", - "ECDH+AES", - "DH+AES", - "RSA+AESGCM", - "RSA+AES", - "!aNULL", - "!eNULL", - "!MD5", - "!DSS", - ] -) - -try: - from ssl import SSLContext # Modern SSL? + from .ssltransport import SSLTransport # type: ignore[assignment] except ImportError: + OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment, misc] + OP_NO_TICKET = 0x4000 # type: ignore[assignment, misc] + OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment, misc] + OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment, misc] + PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment, misc] + PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment, misc] + VERIFY_X509_PARTIAL_CHAIN = 0x80000 + VERIFY_X509_STRICT = 0x20 # type: ignore[assignment, misc] - class SSLContext(object): # Platform-specific: Python 2 - def __init__(self, protocol_version): - self.protocol = protocol_version - # Use default values from a real SSLContext - self.check_hostname = False - self.verify_mode = ssl.CERT_NONE - self.ca_certs = None - self.options = 0 - self.certfile = None - self.keyfile = None - self.ciphers = None - - def load_cert_chain(self, certfile, keyfile): - self.certfile = certfile - self.keyfile = keyfile - - def load_verify_locations(self, cafile=None, capath=None, cadata=None): - self.ca_certs = cafile - if capath is not None: - raise SSLError("CA directories not supported in older Pythons") +_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None] - if cadata is not None: - raise SSLError("CA data not supported in older Pythons") - - def set_ciphers(self, cipher_suite): - self.ciphers = cipher_suite - - def wrap_socket(self, socket, server_hostname=None, server_side=False): - warnings.warn( - "A true SSLContext object is not available. This prevents " - "urllib3 from configuring SSL appropriately and may cause " - "certain SSL connections to fail. You can upgrade to a newer " - "version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings", - InsecurePlatformWarning, - ) - kwargs = { - "keyfile": self.keyfile, - "certfile": self.certfile, - "ca_certs": self.ca_certs, - "cert_reqs": self.verify_mode, - "ssl_version": self.protocol, - "server_side": server_side, - } - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - -def assert_fingerprint(cert, fingerprint): +def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None: """ Checks if given fingerprint matches the supplied certificate. @@ -192,16 +159,17 @@ def assert_fingerprint(cert, fingerprint): Fingerprint as string of hexdigits, can be interspersed by colons. """ + if cert is None: + raise SSLError("No certificate for the peer.") + fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) if digest_length not in HASHFUNC_MAP: - raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) + raise SSLError(f"Fingerprint of invalid length: {fingerprint}") hashfunc = HASHFUNC_MAP.get(digest_length) if hashfunc is None: raise SSLError( - "Hash function implementation unavailable for fingerprint length: {0}".format( - digest_length - ) + f"Hash function implementation unavailable for fingerprint length: {digest_length}" ) # We need encode() here for py32; works on py2 and p33. @@ -209,15 +177,13 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() - if not _const_compare_digest(cert_digest, fingerprint_bytes): + if not hmac.compare_digest(cert_digest, fingerprint_bytes): raise SSLError( - 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( - fingerprint, hexlify(cert_digest) - ) + f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"' ) -def resolve_cert_reqs(candidate): +def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode: """ Resolves the argument to a numeric constant, which can be passed to the wrap_socket function/method from the ssl module. @@ -235,12 +201,12 @@ def resolve_cert_reqs(candidate): res = getattr(ssl, candidate, None) if res is None: res = getattr(ssl, "CERT_" + candidate) - return res + return res # type: ignore[no-any-return] - return candidate + return candidate # type: ignore[return-value] -def resolve_ssl_version(candidate): +def resolve_ssl_version(candidate: None | int | str) -> int: """ like resolve_cert_reqs """ @@ -251,35 +217,34 @@ def resolve_ssl_version(candidate): res = getattr(ssl, candidate, None) if res is None: res = getattr(ssl, "PROTOCOL_" + candidate) - return res + return typing.cast(int, res) return candidate def create_urllib3_context( - ssl_version=None, cert_reqs=None, options=None, ciphers=None -): - """All arguments have the same meaning as ``ssl_wrap_socket``. - - By default, this function does a lot of the same work that - ``ssl.create_default_context`` does on Python 3.4+. It: - - - Disables SSLv2, SSLv3, and compression - - Sets a restricted set of server ciphers - - If you wish to enable SSLv3, you can do:: - - from pip._vendor.urllib3.util import ssl_ - context = ssl_.create_urllib3_context() - context.options &= ~ssl_.OP_NO_SSLv3 - - You can do the same to enable compression (substituting ``COMPRESSION`` - for ``SSLv3`` in the last line above). + ssl_version: int | None = None, + cert_reqs: int | None = None, + options: int | None = None, + ciphers: str | None = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + verify_flags: int | None = None, +) -> ssl.SSLContext: + """Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3. :param ssl_version: The desired protocol version to use. This will default to PROTOCOL_SSLv23 which will negotiate the highest protocol that both the server and your installation of OpenSSL support. + + This parameter is deprecated instead use 'ssl_minimum_version'. + :param ssl_minimum_version: + The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value. + :param ssl_maximum_version: + The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value. + Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the + default value. :param cert_reqs: Whether to require the certificate verification. This defaults to ``ssl.CERT_REQUIRED``. @@ -287,18 +252,63 @@ def create_urllib3_context( Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``. :param ciphers: - Which cipher suites to allow the server to select. + Which cipher suites to allow the server to select. Defaults to either system configured + ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers. + :param verify_flags: + The flags for certificate verification operations. These default to + ``ssl.VERIFY_X509_PARTIAL_CHAIN`` and ``ssl.VERIFY_X509_STRICT`` for Python 3.13+. :returns: Constructed SSLContext object with specified options :rtype: SSLContext """ - # PROTOCOL_TLS is deprecated in Python 3.10 - if not ssl_version or ssl_version == PROTOCOL_TLS: - ssl_version = PROTOCOL_TLS_CLIENT + if SSLContext is None: + raise TypeError("Can't create an SSLContext object without an ssl module") + + # This means 'ssl_version' was specified as an exact value. + if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT): + # Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version' + # to avoid conflicts. + if ssl_minimum_version is not None or ssl_maximum_version is not None: + raise ValueError( + "Can't specify both 'ssl_version' and either " + "'ssl_minimum_version' or 'ssl_maximum_version'" + ) + + # 'ssl_version' is deprecated and will be removed in the future. + else: + # Use 'ssl_minimum_version' and 'ssl_maximum_version' instead. + ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get( + ssl_version, TLSVersion.MINIMUM_SUPPORTED + ) + ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get( + ssl_version, TLSVersion.MAXIMUM_SUPPORTED + ) - context = SSLContext(ssl_version) + # This warning message is pushing users to use 'ssl_minimum_version' + # instead of both min/max. Best practice is to only set the minimum version and + # keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED' + warnings.warn( + "'ssl_version' option is deprecated and will be " + "removed in urllib3 v2.6.0. Instead use 'ssl_minimum_version'", + category=DeprecationWarning, + stacklevel=2, + ) + + # PROTOCOL_TLS is deprecated in Python 3.10 so we always use PROTOCOL_TLS_CLIENT + context = SSLContext(PROTOCOL_TLS_CLIENT) + + if ssl_minimum_version is not None: + context.minimum_version = ssl_minimum_version + else: # Python <3.10 defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here + context.minimum_version = TLSVersion.TLSv1_2 + + if ssl_maximum_version is not None: + context.maximum_version = ssl_maximum_version - context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Unless we're given ciphers defer to either system ciphers in + # the case of OpenSSL 1.1.1+ or use our own secure default ciphers. + if ciphers: + context.set_ciphers(ciphers) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -320,65 +330,106 @@ def create_urllib3_context( context.options |= options + if verify_flags is None: + verify_flags = 0 + # In Python 3.13+ ssl.create_default_context() sets VERIFY_X509_PARTIAL_CHAIN + # and VERIFY_X509_STRICT so we do the same + if sys.version_info >= (3, 13): + verify_flags |= VERIFY_X509_PARTIAL_CHAIN + verify_flags |= VERIFY_X509_STRICT + + context.verify_flags |= verify_flags + # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is # necessary for conditional client cert authentication with TLS 1.3. - # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older - # versions of Python. We only enable on Python 3.7.4+ or if certificate - # verification is enabled to work around Python issue #37428 - # See: https://bugs.python.org/issue37428 - if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( - context, "post_handshake_auth", None - ) is not None: + # The attribute is None for OpenSSL <= 1.1.0 or does not exist when using + # an SSLContext created by pyOpenSSL. + if getattr(context, "post_handshake_auth", None) is not None: context.post_handshake_auth = True - def disable_check_hostname(): - if ( - getattr(context, "check_hostname", None) is not None - ): # Platform-specific: Python 3.2 - # We do our own verification, including fingerprints and alternative - # hostnames. So disable it here - context.check_hostname = False - # The order of the below lines setting verify_mode and check_hostname # matter due to safe-guards SSLContext has to prevent an SSLContext with - # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more - # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used - # or not so we don't know the initial state of the freshly created SSLContext. - if cert_reqs == ssl.CERT_REQUIRED: + # check_hostname=True, verify_mode=NONE/OPTIONAL. + # We always set 'check_hostname=False' for pyOpenSSL so we rely on our own + # 'ssl.match_hostname()' implementation. + if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL: context.verify_mode = cert_reqs - disable_check_hostname() + context.check_hostname = True else: - disable_check_hostname() + context.check_hostname = False context.verify_mode = cert_reqs - # Enable logging of TLS session keys via defacto standard environment variable - # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. - if hasattr(context, "keylog_filename"): - sslkeylogfile = os.environ.get("SSLKEYLOGFILE") - if sslkeylogfile: - context.keylog_filename = sslkeylogfile + try: + context.hostname_checks_common_name = False + except AttributeError: # Defensive: for CPython < 3.9.3; for PyPy < 7.3.8 + pass + + if "SSLKEYLOGFILE" in os.environ: + sslkeylogfile = os.path.expandvars(os.environ.get("SSLKEYLOGFILE")) + else: + sslkeylogfile = None + if sslkeylogfile: + context.keylog_filename = sslkeylogfile return context +def ssl_wrap_socket( + sock: socket.socket, + keyfile: str | None = ..., + certfile: str | None = ..., + cert_reqs: int | None = ..., + ca_certs: str | None = ..., + server_hostname: str | None = ..., + ssl_version: int | None = ..., + ciphers: str | None = ..., + ssl_context: ssl.SSLContext | None = ..., + ca_cert_dir: str | None = ..., + key_password: str | None = ..., + ca_cert_data: None | str | bytes = ..., + tls_in_tls: typing.Literal[False] = ..., +) -> ssl.SSLSocket: ... + + +def ssl_wrap_socket( + sock: socket.socket, + keyfile: str | None = ..., + certfile: str | None = ..., + cert_reqs: int | None = ..., + ca_certs: str | None = ..., + server_hostname: str | None = ..., + ssl_version: int | None = ..., + ciphers: str | None = ..., + ssl_context: ssl.SSLContext | None = ..., + ca_cert_dir: str | None = ..., + key_password: str | None = ..., + ca_cert_data: None | str | bytes = ..., + tls_in_tls: bool = ..., +) -> ssl.SSLSocket | SSLTransportType: ... + + def ssl_wrap_socket( - sock, - keyfile=None, - certfile=None, - cert_reqs=None, - ca_certs=None, - server_hostname=None, - ssl_version=None, - ciphers=None, - ssl_context=None, - ca_cert_dir=None, - key_password=None, - ca_cert_data=None, - tls_in_tls=False, -): + sock: socket.socket, + keyfile: str | None = None, + certfile: str | None = None, + cert_reqs: int | None = None, + ca_certs: str | None = None, + server_hostname: str | None = None, + ssl_version: int | None = None, + ciphers: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_cert_dir: str | None = None, + key_password: str | None = None, + ca_cert_data: None | str | bytes = None, + tls_in_tls: bool = False, +) -> ssl.SSLSocket | SSLTransportType: """ - All arguments except for server_hostname, ssl_context, and ca_cert_dir have - the same meaning as they do when using :func:`ssl.wrap_socket`. + All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and + ca_cert_dir have the same meaning as they do when using + :func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`, + :meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`. :param server_hostname: When SNI is supported, the expected hostname of the certificate @@ -401,19 +452,18 @@ def ssl_wrap_socket( """ context = ssl_context if context is None: - # Note: This branch of code and all the variables in it are no longer - # used by urllib3 itself. We should consider deprecating and removing - # this code. + # Note: This branch of code and all the variables in it are only used in tests. + # We should consider deprecating and removing this code. context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) if ca_certs or ca_cert_dir or ca_cert_data: try: context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data) - except (IOError, OSError) as e: - raise SSLError(e) + except OSError as e: + raise SSLError(e) from e elif ssl_context is None and hasattr(context, "load_default_certs"): - # try to load OS default certs; works well on Windows (require Python3.4+) + # try to load OS default certs; works well on Windows. context.load_default_certs() # Attempt to detect if we get the goofy behavior of the @@ -428,57 +478,28 @@ def ssl_wrap_socket( else: context.load_cert_chain(certfile, keyfile, key_password) - try: - if hasattr(context, "set_alpn_protocols"): - context.set_alpn_protocols(ALPN_PROTOCOLS) - except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols - pass - - # If we detect server_hostname is an IP address then the SNI - # extension should not be used according to RFC3546 Section 3.1 - use_sni_hostname = server_hostname and not is_ipaddress(server_hostname) - # SecureTransport uses server_hostname in certificate verification. - send_sni = (use_sni_hostname and HAS_SNI) or ( - IS_SECURETRANSPORT and server_hostname - ) - # Do not warn the user if server_hostname is an invalid SNI hostname. - if not HAS_SNI and use_sni_hostname: - warnings.warn( - "An HTTPS request has been made, but the SNI (Server Name " - "Indication) extension to TLS is not available on this platform. " - "This may cause the server to present an incorrect TLS " - "certificate, which can cause validation failures. You can upgrade to " - "a newer version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings", - SNIMissingWarning, - ) + context.set_alpn_protocols(ALPN_PROTOCOLS) - if send_sni: - ssl_sock = _ssl_wrap_socket_impl( - sock, context, tls_in_tls, server_hostname=server_hostname - ) - else: - ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls) + ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) return ssl_sock -def is_ipaddress(hostname): +def is_ipaddress(hostname: str | bytes) -> bool: """Detects whether the hostname given is an IPv4 or IPv6 address. Also detects IPv6 addresses with Zone IDs. :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if not six.PY2 and isinstance(hostname, bytes): + if isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode("ascii") - return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname)) + return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname)) -def _is_key_file_encrypted(key_file): +def _is_key_file_encrypted(key_file: str) -> bool: """Detects if a key file is encrypted or not.""" - with open(key_file, "r") as f: + with open(key_file) as f: for line in f: # Look for Proc-Type: 4,ENCRYPTED if "ENCRYPTED" in line: @@ -487,7 +508,12 @@ def _is_key_file_encrypted(key_file): return False -def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): +def _ssl_wrap_socket_impl( + sock: socket.socket, + ssl_context: ssl.SSLContext, + tls_in_tls: bool, + server_hostname: str | None = None, +) -> ssl.SSLSocket | SSLTransportType: if tls_in_tls: if not SSLTransport: # Import error, ssl is not available. @@ -498,7 +524,4 @@ def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context) return SSLTransport(sock, ssl_context, server_hostname) - if server_hostname: - return ssl_context.wrap_socket(sock, server_hostname=server_hostname) - else: - return ssl_context.wrap_socket(sock) + return ssl_context.wrap_socket(sock, server_hostname=server_hostname) diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_match_hostname.py b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_match_hostname.py index 1dd950c4896..25d91000419 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_match_hostname.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_match_hostname.py @@ -1,19 +1,18 @@ -"""The match_hostname() function from Python 3.3.3, essential when using SSL.""" +"""The match_hostname() function from Python 3.5, essential when using SSL.""" # Note: This file is under the PSF license as the code comes from the python # stdlib. http://docs.python.org/3/license.html +# It is modified to remove commonName support. +from __future__ import annotations + +import ipaddress import re -import sys +import typing +from ipaddress import IPv4Address, IPv6Address -# ipaddress has been backported to 2.6+ in pypi. If it is installed on the -# system, use it to handle IPAddress ServerAltnames (this was added in -# python-3.5) otherwise only do DNS matching. This allows -# util.ssl_match_hostname to continue to be used in Python 2.7. -try: - import ipaddress -except ImportError: - ipaddress = None +if typing.TYPE_CHECKING: + from .ssl_ import _TYPE_PEER_CERT_RET_DICT __version__ = "3.5.0.1" @@ -22,7 +21,9 @@ class CertificateError(ValueError): pass -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match( + dn: typing.Any, hostname: str, max_wildcards: int = 1 +) -> typing.Match[str] | None | bool: """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -49,7 +50,7 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # speed up common case w/o wildcards if not wildcards: - return dn.lower() == hostname.lower() + return bool(dn.lower() == hostname.lower()) # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which @@ -76,26 +77,26 @@ def _dnsname_match(dn, hostname, max_wildcards=1): return pat.match(hostname) -def _to_unicode(obj): - if isinstance(obj, str) and sys.version_info < (3,): - # ignored flake8 # F821 to support python 2.7 function - obj = unicode(obj, encoding="ascii", errors="strict") # noqa: F821 - return obj - - -def _ipaddress_match(ipname, host_ip): +def _ipaddress_match(ipname: str, host_ip: IPv4Address | IPv6Address) -> bool: """Exact matching of IP addresses. - RFC 6125 explicitly doesn't define an algorithm for this - (section 1.7.2 - "Out of Scope"). + RFC 9110 section 4.3.5: "A reference identity of IP-ID contains the decoded + bytes of the IP address. An IP version 4 address is 4 octets, and an IP + version 6 address is 16 octets. [...] A reference identity of type IP-ID + matches if the address is identical to an iPAddress value of the + subjectAltName extension of the certificate." """ # OpenSSL may add a trailing newline to a subjectAltName's IP address # Divergence from upstream: ipaddress can't handle byte str - ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) - return ip == host_ip + ip = ipaddress.ip_address(ipname.rstrip()) + return bool(ip.packed == host_ip.packed) -def match_hostname(cert, hostname): +def match_hostname( + cert: _TYPE_PEER_CERT_RET_DICT | None, + hostname: str, + hostname_checks_common_name: bool = False, +) -> None: """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 rules are followed, but IP addresses are not accepted for *hostname*. @@ -111,21 +112,22 @@ def match_hostname(cert, hostname): ) try: # Divergence from upstream: ipaddress can't handle byte str - host_ip = ipaddress.ip_address(_to_unicode(hostname)) - except (UnicodeError, ValueError): - # ValueError: Not an IP address (common case) - # UnicodeError: Divergence from upstream: Have to deal with ipaddress not taking - # byte strings. addresses should be all ascii, so we consider it not - # an ipaddress in this case + # + # The ipaddress module shipped with Python < 3.9 does not support + # scoped IPv6 addresses so we unconditionally strip the Zone IDs for + # now. Once we drop support for Python 3.9 we can remove this branch. + if "%" in hostname: + host_ip = ipaddress.ip_address(hostname[: hostname.rfind("%")]) + else: + host_ip = ipaddress.ip_address(hostname) + + except ValueError: + # Not an IP address (common case) host_ip = None - except AttributeError: - # Divergence from upstream: Make ipaddress library optional - if ipaddress is None: - host_ip = None - else: # Defensive - raise dnsnames = [] - san = cert.get("subjectAltName", ()) + san: tuple[tuple[str, str], ...] = cert.get("subjectAltName", ()) + key: str + value: str for key, value in san: if key == "DNS": if host_ip is None and _dnsname_match(value, hostname): @@ -135,25 +137,23 @@ def match_hostname(cert, hostname): if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName + + # We only check 'commonName' if it's enabled and we're not verifying + # an IP address. IP addresses aren't valid within 'commonName'. + if hostname_checks_common_name and host_ip is None and not dnsnames: for sub in cert.get("subject", ()): for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. if key == "commonName": if _dnsname_match(value, hostname): return - dnsnames.append(value) + dnsnames.append(value) # Defensive: for Python < 3.9.3 + if len(dnsnames) > 1: raise CertificateError( "hostname %r " "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) ) elif len(dnsnames) == 1: - raise CertificateError("hostname %r doesn't match %r" % (hostname, dnsnames[0])) + raise CertificateError(f"hostname {hostname!r} doesn't match {dnsnames[0]!r}") else: - raise CertificateError( - "no appropriate commonName or subjectAltName fields were found" - ) + raise CertificateError("no appropriate subjectAltName fields were found") diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/ssltransport.py b/contrib/python/pip/pip/_vendor/urllib3/util/ssltransport.py index 4a7105d1791..6d59bc3bce2 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/ssltransport.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/ssltransport.py @@ -1,9 +1,20 @@ +from __future__ import annotations + import io import socket import ssl +import typing from ..exceptions import ProxySchemeUnsupported -from ..packages import six + +if typing.TYPE_CHECKING: + from typing_extensions import Self + + from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT + + +_WriteBuffer = typing.Union[bytearray, memoryview] +_ReturnValue = typing.TypeVar("_ReturnValue") SSL_BLOCKSIZE = 16384 @@ -20,7 +31,7 @@ class SSLTransport: """ @staticmethod - def _validate_ssl_context_for_tls_in_tls(ssl_context): + def _validate_ssl_context_for_tls_in_tls(ssl_context: ssl.SSLContext) -> None: """ Raises a ProxySchemeUnsupported if the provided ssl_context can't be used for TLS in TLS. @@ -30,20 +41,18 @@ class SSLTransport: """ if not hasattr(ssl_context, "wrap_bio"): - if six.PY2: - raise ProxySchemeUnsupported( - "TLS in TLS requires SSLContext.wrap_bio() which isn't " - "supported on Python 2" - ) - else: - raise ProxySchemeUnsupported( - "TLS in TLS requires SSLContext.wrap_bio() which isn't " - "available on non-native SSLContext" - ) + raise ProxySchemeUnsupported( + "TLS in TLS requires SSLContext.wrap_bio() which isn't " + "available on non-native SSLContext" + ) def __init__( - self, socket, ssl_context, server_hostname=None, suppress_ragged_eofs=True - ): + self, + socket: socket.socket, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + suppress_ragged_eofs: bool = True, + ) -> None: """ Create an SSLTransport around socket using the provided ssl_context. """ @@ -60,33 +69,36 @@ class SSLTransport: # Perform initial handshake. self._ssl_io_loop(self.sslobj.do_handshake) - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, *_): + def __exit__(self, *_: typing.Any) -> None: self.close() - def fileno(self): + def fileno(self) -> int: return self.socket.fileno() - def read(self, len=1024, buffer=None): + def read(self, len: int = 1024, buffer: typing.Any | None = None) -> int | bytes: return self._wrap_ssl_read(len, buffer) - def recv(self, len=1024, flags=0): + def recv(self, buflen: int = 1024, flags: int = 0) -> int | bytes: if flags != 0: raise ValueError("non-zero flags not allowed in calls to recv") - return self._wrap_ssl_read(len) + return self._wrap_ssl_read(buflen) - def recv_into(self, buffer, nbytes=None, flags=0): + def recv_into( + self, + buffer: _WriteBuffer, + nbytes: int | None = None, + flags: int = 0, + ) -> None | int | bytes: if flags != 0: raise ValueError("non-zero flags not allowed in calls to recv_into") - if buffer and (nbytes is None): + if nbytes is None: nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 return self.read(nbytes, buffer) - def sendall(self, data, flags=0): + def sendall(self, data: bytes, flags: int = 0) -> None: if flags != 0: raise ValueError("non-zero flags not allowed in calls to sendall") count = 0 @@ -96,15 +108,20 @@ class SSLTransport: v = self.send(byte_view[count:]) count += v - def send(self, data, flags=0): + def send(self, data: bytes, flags: int = 0) -> int: if flags != 0: raise ValueError("non-zero flags not allowed in calls to send") - response = self._ssl_io_loop(self.sslobj.write, data) - return response + return self._ssl_io_loop(self.sslobj.write, data) def makefile( - self, mode="r", buffering=None, encoding=None, errors=None, newline=None - ): + self, + mode: str, + buffering: int | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> typing.BinaryIO | typing.TextIO | socket.SocketIO: """ Python's httpclient uses makefile and buffered io when reading HTTP messages and we need to support it. @@ -113,7 +130,7 @@ class SSLTransport: changes to point to the socket directly. """ if not set(mode) <= {"r", "w", "b"}: - raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) + raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)") writing = "w" in mode reading = "r" in mode or not writing @@ -124,8 +141,8 @@ class SSLTransport: rawmode += "r" if writing: rawmode += "w" - raw = socket.SocketIO(self, rawmode) - self.socket._io_refs += 1 + raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type] + self.socket._io_refs += 1 # type: ignore[attr-defined] if buffering is None: buffering = -1 if buffering < 0: @@ -134,8 +151,9 @@ class SSLTransport: if not binary: raise ValueError("unbuffered streams must be binary") return raw + buffer: typing.BinaryIO if reading and writing: - buffer = io.BufferedRWPair(raw, raw, buffering) + buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment] elif reading: buffer = io.BufferedReader(raw, buffering) else: @@ -144,46 +162,51 @@ class SSLTransport: if binary: return buffer text = io.TextIOWrapper(buffer, encoding, errors, newline) - text.mode = mode + text.mode = mode # type: ignore[misc] return text - def unwrap(self): + def unwrap(self) -> None: self._ssl_io_loop(self.sslobj.unwrap) - def close(self): + def close(self) -> None: self.socket.close() - def getpeercert(self, binary_form=False): - return self.sslobj.getpeercert(binary_form) + @typing.overload + def getpeercert( + self, binary_form: typing.Literal[False] = ... + ) -> _TYPE_PEER_CERT_RET_DICT | None: ... + + @typing.overload + def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ... - def version(self): + def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET: + return self.sslobj.getpeercert(binary_form) # type: ignore[return-value] + + def version(self) -> str | None: return self.sslobj.version() - def cipher(self): + def cipher(self) -> tuple[str, str, int] | None: return self.sslobj.cipher() - def selected_alpn_protocol(self): + def selected_alpn_protocol(self) -> str | None: return self.sslobj.selected_alpn_protocol() - def selected_npn_protocol(self): - return self.sslobj.selected_npn_protocol() - - def shared_ciphers(self): + def shared_ciphers(self) -> list[tuple[str, str, int]] | None: return self.sslobj.shared_ciphers() - def compression(self): + def compression(self) -> str | None: return self.sslobj.compression() - def settimeout(self, value): + def settimeout(self, value: float | None) -> None: self.socket.settimeout(value) - def gettimeout(self): + def gettimeout(self) -> float | None: return self.socket.gettimeout() - def _decref_socketios(self): - self.socket._decref_socketios() + def _decref_socketios(self) -> None: + self.socket._decref_socketios() # type: ignore[attr-defined] - def _wrap_ssl_read(self, len, buffer=None): + def _wrap_ssl_read(self, len: int, buffer: bytearray | None = None) -> int | bytes: try: return self._ssl_io_loop(self.sslobj.read, len, buffer) except ssl.SSLError as e: @@ -192,7 +215,29 @@ class SSLTransport: else: raise - def _ssl_io_loop(self, func, *args): + # func is sslobj.do_handshake or sslobj.unwrap + @typing.overload + def _ssl_io_loop(self, func: typing.Callable[[], None]) -> None: ... + + # func is sslobj.write, arg1 is data + @typing.overload + def _ssl_io_loop(self, func: typing.Callable[[bytes], int], arg1: bytes) -> int: ... + + # func is sslobj.read, arg1 is len, arg2 is buffer + @typing.overload + def _ssl_io_loop( + self, + func: typing.Callable[[int, bytearray | None], bytes], + arg1: int, + arg2: bytearray | None, + ) -> bytes: ... + + def _ssl_io_loop( + self, + func: typing.Callable[..., _ReturnValue], + arg1: None | bytes | int = None, + arg2: bytearray | None = None, + ) -> _ReturnValue: """Performs an I/O loop between incoming/outgoing and the socket.""" should_loop = True ret = None @@ -200,7 +245,12 @@ class SSLTransport: while should_loop: errno = None try: - ret = func(*args) + if arg1 is None and arg2 is None: + ret = func() + elif arg2 is None: + ret = func(arg1) + else: + ret = func(arg1, arg2) except ssl.SSLError as e: if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE): # WANT_READ, and WANT_WRITE are expected, others are not. @@ -218,4 +268,4 @@ class SSLTransport: self.incoming.write(buf) else: self.incoming.write_eof() - return ret + return typing.cast(_ReturnValue, ret) diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/timeout.py b/contrib/python/pip/pip/_vendor/urllib3/util/timeout.py index 78e18a62724..c6cdb352aae 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/timeout.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/timeout.py @@ -1,44 +1,56 @@ -from __future__ import absolute_import +from __future__ import annotations import time - -# The default socket timeout, used by httplib to indicate that no timeout was; specified by the user -from socket import _GLOBAL_DEFAULT_TIMEOUT, getdefaulttimeout +import typing +from enum import Enum +from socket import getdefaulttimeout from ..exceptions import TimeoutStateError -# A sentinel value to indicate that no timeout was specified by the user in -# urllib3 -_Default = object() +if typing.TYPE_CHECKING: + from typing import Final + + +class _TYPE_DEFAULT(Enum): + # This value should never be passed to socket.settimeout() so for safety we use a -1. + # socket.settimout() raises a ValueError for negative values. + token = -1 + +_DEFAULT_TIMEOUT: Final[_TYPE_DEFAULT] = _TYPE_DEFAULT.token -# Use time.monotonic if available. -current_time = getattr(time, "monotonic", time.time) +_TYPE_TIMEOUT = typing.Optional[typing.Union[float, _TYPE_DEFAULT]] -class Timeout(object): +class Timeout: """Timeout configuration. Timeouts can be defined as a default for a pool: .. code-block:: python - timeout = Timeout(connect=2.0, read=7.0) - http = PoolManager(timeout=timeout) - response = http.request('GET', 'http://example.com/') + from pip._vendor import urllib3 + + timeout = urllib3.util.Timeout(connect=2.0, read=7.0) + + http = urllib3.PoolManager(timeout=timeout) + + resp = http.request("GET", "https://example.com/") + + print(resp.status) Or per-request (which overrides the default for the pool): .. code-block:: python - response = http.request('GET', 'http://example.com/', timeout=Timeout(10)) + response = http.request("GET", "https://example.com/", timeout=Timeout(10)) Timeouts can be disabled by setting all the parameters to ``None``: .. code-block:: python no_timeout = Timeout(connect=None, read=None) - response = http.request('GET', 'http://example.com/, timeout=no_timeout) + response = http.request("GET", "https://example.com/", timeout=no_timeout) :param total: @@ -89,38 +101,34 @@ class Timeout(object): the case; if a server streams one byte every fifteen seconds, a timeout of 20 seconds will not trigger, even though the request will take several minutes to complete. - - If your goal is to cut off any request after a set amount of wall clock - time, consider having a second "watcher" thread to cut off a slow - request. """ #: A sentinel object representing the default timeout value - DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT + DEFAULT_TIMEOUT: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT - def __init__(self, total=None, connect=_Default, read=_Default): + def __init__( + self, + total: _TYPE_TIMEOUT = None, + connect: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + read: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + ) -> None: self._connect = self._validate_timeout(connect, "connect") self._read = self._validate_timeout(read, "read") self.total = self._validate_timeout(total, "total") - self._start_connect = None + self._start_connect: float | None = None - def __repr__(self): - return "%s(connect=%r, read=%r, total=%r)" % ( - type(self).__name__, - self._connect, - self._read, - self.total, - ) + def __repr__(self) -> str: + return f"{type(self).__name__}(connect={self._connect!r}, read={self._read!r}, total={self.total!r})" # __str__ provided for backwards compatibility __str__ = __repr__ - @classmethod - def resolve_default_timeout(cls, timeout): - return getdefaulttimeout() if timeout is cls.DEFAULT_TIMEOUT else timeout + @staticmethod + def resolve_default_timeout(timeout: _TYPE_TIMEOUT) -> float | None: + return getdefaulttimeout() if timeout is _DEFAULT_TIMEOUT else timeout @classmethod - def _validate_timeout(cls, value, name): + def _validate_timeout(cls, value: _TYPE_TIMEOUT, name: str) -> _TYPE_TIMEOUT: """Check that a timeout attribute is valid. :param value: The timeout value to validate @@ -130,10 +138,7 @@ class Timeout(object): :raises ValueError: If it is a numeric value less than or equal to zero, or the type is not an integer, float, or None. """ - if value is _Default: - return cls.DEFAULT_TIMEOUT - - if value is None or value is cls.DEFAULT_TIMEOUT: + if value is None or value is _DEFAULT_TIMEOUT: return value if isinstance(value, bool): @@ -147,7 +152,7 @@ class Timeout(object): raise ValueError( "Timeout value %s was %s, but it must be an " "int, float or None." % (name, value) - ) + ) from None try: if value <= 0: @@ -157,16 +162,15 @@ class Timeout(object): "than or equal to 0." % (name, value) ) except TypeError: - # Python 3 raise ValueError( "Timeout value %s was %s, but it must be an " "int, float or None." % (name, value) - ) + ) from None return value @classmethod - def from_float(cls, timeout): + def from_float(cls, timeout: _TYPE_TIMEOUT) -> Timeout: """Create a new Timeout from a legacy timeout value. The timeout value used by httplib.py sets the same timeout on the @@ -175,13 +179,13 @@ class Timeout(object): passed to this function. :param timeout: The legacy timeout value. - :type timeout: integer, float, sentinel default object, or None + :type timeout: integer, float, :attr:`urllib3.util.Timeout.DEFAULT_TIMEOUT`, or None :return: Timeout object :rtype: :class:`Timeout` """ return Timeout(read=timeout, connect=timeout) - def clone(self): + def clone(self) -> Timeout: """Create a copy of the timeout object Timeout properties are stored per-pool but each request needs a fresh @@ -195,7 +199,7 @@ class Timeout(object): # detect the user default. return Timeout(connect=self._connect, read=self._read, total=self.total) - def start_connect(self): + def start_connect(self) -> float: """Start the timeout clock, used during a connect() attempt :raises urllib3.exceptions.TimeoutStateError: if you attempt @@ -203,10 +207,10 @@ class Timeout(object): """ if self._start_connect is not None: raise TimeoutStateError("Timeout timer has already been started.") - self._start_connect = current_time() + self._start_connect = time.monotonic() return self._start_connect - def get_connect_duration(self): + def get_connect_duration(self) -> float: """Gets the time elapsed since the call to :meth:`start_connect`. :return: Elapsed time in seconds. @@ -218,10 +222,10 @@ class Timeout(object): raise TimeoutStateError( "Can't get connect duration for timer that has not started." ) - return current_time() - self._start_connect + return time.monotonic() - self._start_connect @property - def connect_timeout(self): + def connect_timeout(self) -> _TYPE_TIMEOUT: """Get the value to use when setting a connection timeout. This will be a positive float or integer, the value None @@ -233,13 +237,13 @@ class Timeout(object): if self.total is None: return self._connect - if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: + if self._connect is None or self._connect is _DEFAULT_TIMEOUT: return self.total - return min(self._connect, self.total) + return min(self._connect, self.total) # type: ignore[type-var] @property - def read_timeout(self): + def read_timeout(self) -> float | None: """Get the value for the read timeout. This assumes some time has elapsed in the connection timeout and @@ -251,21 +255,21 @@ class Timeout(object): raised. :return: Value to use for the read timeout. - :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None + :rtype: int, float or None :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. """ if ( self.total is not None - and self.total is not self.DEFAULT_TIMEOUT + and self.total is not _DEFAULT_TIMEOUT and self._read is not None - and self._read is not self.DEFAULT_TIMEOUT + and self._read is not _DEFAULT_TIMEOUT ): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read return max(0, min(self.total - self.get_connect_duration(), self._read)) - elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: + elif self.total is not None and self.total is not _DEFAULT_TIMEOUT: return max(0, self.total - self.get_connect_duration()) else: - return self._read + return self.resolve_default_timeout(self._read) diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/url.py b/contrib/python/pip/pip/_vendor/urllib3/util/url.py index a960b2f3c5f..82b174ceba2 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/url.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/url.py @@ -1,22 +1,20 @@ -from __future__ import absolute_import +from __future__ import annotations import re -from collections import namedtuple +import typing from ..exceptions import LocationParseError -from ..packages import six - -url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] +from .util import to_str # We only want to normalize urls with an HTTP(S) scheme. # urllib3 infers URLs without a scheme (None) to be http. -NORMALIZABLE_SCHEMES = ("http", "https", None) +_NORMALIZABLE_SCHEMES = ("http", "https", None) # Almost all of these patterns were derived from the # 'rfc3986' module: https://github.com/python-hyper/rfc3986 -PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") -SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") -URI_RE = re.compile( +_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") +_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") +_URI_RE = re.compile( r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" r"(?://([^\\/?#]*))?" r"([^?#]*)" @@ -25,10 +23,10 @@ URI_RE = re.compile( re.UNICODE | re.DOTALL, ) -IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" -HEX_PAT = "[0-9A-Fa-f]{1,4}" -LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT) -_subs = {"hex": HEX_PAT, "ls32": LS32_PAT} +_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" +_HEX_PAT = "[0-9A-Fa-f]{1,4}" +_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT) +_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT} _variations = [ # 6( h16 ":" ) ls32 "(?:%(hex)s:){6}%(ls32)s", @@ -50,69 +48,78 @@ _variations = [ "(?:(?:%(hex)s:){0,6}%(hex)s)?::", ] -UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~" -IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" -ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" -IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]" -REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" -TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") +_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~" +_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" +_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" +_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]" +_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" +_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") -IPV4_RE = re.compile("^" + IPV4_PAT + "$") -IPV6_RE = re.compile("^" + IPV6_PAT + "$") -IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") -BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") -ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") +_IPV4_RE = re.compile("^" + _IPV4_PAT + "$") +_IPV6_RE = re.compile("^" + _IPV6_PAT + "$") +_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$") +_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$") +_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$") _HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % ( - REG_NAME_PAT, - IPV4_PAT, - IPV6_ADDRZ_PAT, + _REG_NAME_PAT, + _IPV4_PAT, + _IPV6_ADDRZ_PAT, ) _HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) -UNRESERVED_CHARS = set( +_UNRESERVED_CHARS = set( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" ) -SUB_DELIM_CHARS = set("!$&'()*+,;=") -USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} -PATH_CHARS = USERINFO_CHARS | {"@", "/"} -QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} +_SUB_DELIM_CHARS = set("!$&'()*+,;=") +_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"} +_PATH_CHARS = _USERINFO_CHARS | {"@", "/"} +_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"} -class Url(namedtuple("Url", url_attrs)): +class Url( + typing.NamedTuple( + "Url", + [ + ("scheme", typing.Optional[str]), + ("auth", typing.Optional[str]), + ("host", typing.Optional[str]), + ("port", typing.Optional[int]), + ("path", typing.Optional[str]), + ("query", typing.Optional[str]), + ("fragment", typing.Optional[str]), + ], + ) +): """ Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ - __slots__ = () - - def __new__( + def __new__( # type: ignore[no-untyped-def] cls, - scheme=None, - auth=None, - host=None, - port=None, - path=None, - query=None, - fragment=None, + scheme: str | None = None, + auth: str | None = None, + host: str | None = None, + port: int | None = None, + path: str | None = None, + query: str | None = None, + fragment: str | None = None, ): if path and not path.startswith("/"): path = "/" + path if scheme is not None: scheme = scheme.lower() - return super(Url, cls).__new__( - cls, scheme, auth, host, port, path, query, fragment - ) + return super().__new__(cls, scheme, auth, host, port, path, query, fragment) @property - def hostname(self): + def hostname(self) -> str | None: """For backwards-compatibility with urlparse. We're nice like that.""" return self.host @property - def request_uri(self): + def request_uri(self) -> str: """Absolute path including the query string.""" uri = self.path or "/" @@ -122,14 +129,37 @@ class Url(namedtuple("Url", url_attrs)): return uri @property - def netloc(self): - """Network location including host and port""" + def authority(self) -> str | None: + """ + Authority component as defined in RFC 3986 3.2. + This includes userinfo (auth), host and port. + + i.e. + userinfo@host:port + """ + userinfo = self.auth + netloc = self.netloc + if netloc is None or userinfo is None: + return netloc + else: + return f"{userinfo}@{netloc}" + + @property + def netloc(self) -> str | None: + """ + Network location including host and port. + + If you need the equivalent of urllib.parse's ``netloc``, + use the ``authority`` property instead. + """ + if self.host is None: + return None if self.port: - return "%s:%d" % (self.host, self.port) + return f"{self.host}:{self.port}" return self.host @property - def url(self): + def url(self) -> str: """ Convert self into a url @@ -138,88 +168,77 @@ class Url(namedtuple("Url", url_attrs)): :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls with a blank port will have : removed). - Example: :: + Example: - >>> U = parse_url('http://google.com/mail/') - >>> U.url - 'http://google.com/mail/' - >>> Url('http', 'username:password', 'host.com', 80, - ... '/path', 'query', 'fragment').url - 'http://username:[email protected]:80/path?query#fragment' + .. code-block:: python + + from pip._vendor import urllib3 + + U = urllib3.util.parse_url("https://google.com/mail/") + + print(U.url) + # "https://google.com/mail/" + + print( urllib3.util.Url("https", "username:password", + "host.com", 80, "/path", "query", "fragment" + ).url + ) + # "https://username:[email protected]:80/path?query#fragment" """ scheme, auth, host, port, path, query, fragment = self - url = u"" + url = "" # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + u"://" + url += scheme + "://" if auth is not None: - url += auth + u"@" + url += auth + "@" if host is not None: url += host if port is not None: - url += u":" + str(port) + url += ":" + str(port) if path is not None: url += path if query is not None: - url += u"?" + query + url += "?" + query if fragment is not None: - url += u"#" + fragment + url += "#" + fragment return url - def __str__(self): + def __str__(self) -> str: return self.url -def split_first(s, delims): - """ - .. deprecated:: 1.25 - - Given a string and an iterable of delimiters, split on the first found - delimiter. Return two split parts and the matched delimiter. - - If not found, then the first part is the full input string. - - Example:: - - >>> split_first('foo/bar?baz', '?/=') - ('foo', 'bar?baz', '/') - >>> split_first('foo/bar?baz', '123') - ('foo/bar?baz', '', None) - - Scales linearly with number of delims. Not ideal for large number of delims. - """ - min_idx = None - min_delim = None - for d in delims: - idx = s.find(d) - if idx < 0: - continue - - if min_idx is None or idx < min_idx: - min_idx = idx - min_delim = d +def _encode_invalid_chars( + component: str, allowed_chars: typing.Container[str] +) -> str: # Abstract + ... - if min_idx is None or min_idx < 0: - return s, "", None - return s[:min_idx], s[min_idx + 1 :], min_delim +def _encode_invalid_chars( + component: None, allowed_chars: typing.Container[str] +) -> None: # Abstract + ... -def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): +def _encode_invalid_chars( + component: str | None, allowed_chars: typing.Container[str] +) -> str | None: """Percent-encodes a URI component without reapplying onto an already percent-encoded component. """ if component is None: return component - component = six.ensure_text(component) + component = to_str(component) # Normalize existing percent-encoded bytes. # Try to see if the component we're encoding is already percent-encoded # so we can skip all '%' characters but still encode all others. - component, percent_encodings = PERCENT_RE.subn( + component, percent_encodings = _PERCENT_RE.subn( lambda match: match.group(0).upper(), component ) @@ -228,7 +247,7 @@ def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): encoded_component = bytearray() for i in range(0, len(uri_bytes)): - # Will return a single character bytestring on both Python 2 & 3 + # Will return a single character bytestring byte = uri_bytes[i : i + 1] byte_ord = ord(byte) if (is_percent_encoded and byte == b"%") or ( @@ -238,10 +257,10 @@ def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): continue encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) - return encoded_component.decode(encoding) + return encoded_component.decode() -def _remove_path_dot_segments(path): +def _remove_path_dot_segments(path: str) -> str: # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code segments = path.split("/") # Turn the path into a list of segments output = [] # Initialize the variable to use to store output @@ -251,7 +270,7 @@ def _remove_path_dot_segments(path): if segment == ".": continue # Anything other than '..', should be appended to the output - elif segment != "..": + if segment != "..": output.append(segment) # In this case segment == '..', if we can, we should pop the last # element @@ -271,18 +290,23 @@ def _remove_path_dot_segments(path): return "/".join(output) -def _normalize_host(host, scheme): - if host: - if isinstance(host, six.binary_type): - host = six.ensure_str(host) +def _normalize_host(host: None, scheme: str | None) -> None: ... + + +def _normalize_host(host: str, scheme: str | None) -> str: ... + - if scheme in NORMALIZABLE_SCHEMES: - is_ipv6 = IPV6_ADDRZ_RE.match(host) +def _normalize_host(host: str | None, scheme: str | None) -> str | None: + if host: + if scheme in _NORMALIZABLE_SCHEMES: + is_ipv6 = _IPV6_ADDRZ_RE.match(host) if is_ipv6: # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID # separator as necessary to return a valid RFC 4007 scoped IP. - match = ZONE_ID_RE.search(host) + match = _ZONE_ID_RE.search(host) if match: start, end = match.span(1) zone_id = host[start:end] @@ -291,46 +315,56 @@ def _normalize_host(host, scheme): zone_id = zone_id[3:] else: zone_id = zone_id[1:] - zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS) - return host[:start].lower() + zone_id + host[end:] + zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS) + return f"{host[:start].lower()}%{zone_id}{host[end:]}" else: return host.lower() - elif not IPV4_RE.match(host): - return six.ensure_str( - b".".join([_idna_encode(label) for label in host.split(".")]) + elif not _IPV4_RE.match(host): + return to_str( + b".".join([_idna_encode(label) for label in host.split(".")]), + "ascii", ) return host -def _idna_encode(name): - if name and any(ord(x) >= 128 for x in name): +def _idna_encode(name: str) -> bytes: + if not name.isascii(): try: from pip._vendor import idna except ImportError: - six.raise_from( - LocationParseError("Unable to parse URL without the 'idna' module"), - None, - ) + raise LocationParseError( + "Unable to parse URL without the 'idna' module" + ) from None + try: return idna.encode(name.lower(), strict=True, std3_rules=True) except idna.IDNAError: - six.raise_from( - LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None - ) + raise LocationParseError( + f"Name '{name}' is not a valid IDNA label" + ) from None + return name.lower().encode("ascii") -def _encode_target(target): - """Percent-encodes a request target so that there are no invalid characters""" - path, query = TARGET_RE.match(target).groups() - target = _encode_invalid_chars(path, PATH_CHARS) - query = _encode_invalid_chars(query, QUERY_CHARS) +def _encode_target(target: str) -> str: + """Percent-encodes a request target so that there are no invalid characters + + Pre-condition for this function is that 'target' must start with '/'. + If that is the case then _TARGET_RE will always produce a match. + """ + match = _TARGET_RE.match(target) + if not match: # Defensive: + raise LocationParseError(f"{target!r} is not a valid request URI") + + path, query = match.groups() + encoded_target = _encode_invalid_chars(path, _PATH_CHARS) if query is not None: - target += "?" + query - return target + query = _encode_invalid_chars(query, _QUERY_CHARS) + encoded_target += "?" + query + return encoded_target -def parse_url(url): +def parse_url(url: str) -> Url: """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. @@ -341,28 +375,44 @@ def parse_url(url): :param str url: URL to parse into a :class:`.Url` namedtuple. - Partly backwards-compatible with :mod:`urlparse`. + Partly backwards-compatible with :mod:`urllib.parse`. - Example:: + Example: - >>> parse_url('http://google.com/mail/') - Url(scheme='http', host='google.com', port=None, path='/mail/', ...) - >>> parse_url('google.com:80') - Url(scheme=None, host='google.com', port=80, path=None, ...) - >>> parse_url('/foo?bar') - Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) + .. code-block:: python + + from pip._vendor import urllib3 + + print( urllib3.util.parse_url('http://google.com/mail/')) + # Url(scheme='http', host='google.com', port=None, path='/mail/', ...) + + print( urllib3.util.parse_url('google.com:80')) + # Url(scheme=None, host='google.com', port=80, path=None, ...) + + print( urllib3.util.parse_url('/foo?bar')) + # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) """ if not url: # Empty return Url() source_url = url - if not SCHEME_RE.search(url): + if not _SCHEME_RE.search(url): url = "//" + url + scheme: str | None + authority: str | None + auth: str | None + host: str | None + port: str | None + port_int: int | None + path: str | None + query: str | None + fragment: str | None + try: - scheme, authority, path, query, fragment = URI_RE.match(url).groups() - normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES + scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr] + normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES if scheme: scheme = scheme.lower() @@ -370,31 +420,33 @@ def parse_url(url): if authority: auth, _, host_port = authority.rpartition("@") auth = auth or None - host, port = _HOST_PORT_RE.match(host_port).groups() + host, port = _HOST_PORT_RE.match(host_port).groups() # type: ignore[union-attr] if auth and normalize_uri: - auth = _encode_invalid_chars(auth, USERINFO_CHARS) + auth = _encode_invalid_chars(auth, _USERINFO_CHARS) if port == "": port = None else: auth, host, port = None, None, None if port is not None: - port = int(port) - if not (0 <= port <= 65535): + port_int = int(port) + if not (0 <= port_int <= 65535): raise LocationParseError(url) + else: + port_int = None host = _normalize_host(host, scheme) if normalize_uri and path: path = _remove_path_dot_segments(path) - path = _encode_invalid_chars(path, PATH_CHARS) + path = _encode_invalid_chars(path, _PATH_CHARS) if normalize_uri and query: - query = _encode_invalid_chars(query, QUERY_CHARS) + query = _encode_invalid_chars(query, _QUERY_CHARS) if normalize_uri and fragment: - fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) + fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS) - except (ValueError, AttributeError): - return six.raise_from(LocationParseError(source_url), None) + except (ValueError, AttributeError) as e: + raise LocationParseError(source_url) from e # For the sake of backwards compatibility we put empty # string values for path if there are any defined values @@ -406,30 +458,12 @@ def parse_url(url): else: path = None - # Ensure that each part of the URL is a `str` for - # backwards compatibility. - if isinstance(url, six.text_type): - ensure_func = six.ensure_text - else: - ensure_func = six.ensure_str - - def ensure_type(x): - return x if x is None else ensure_func(x) - return Url( - scheme=ensure_type(scheme), - auth=ensure_type(auth), - host=ensure_type(host), - port=port, - path=ensure_type(path), - query=ensure_type(query), - fragment=ensure_type(fragment), + scheme=scheme, + auth=auth, + host=host, + port=port_int, + path=path, + query=query, + fragment=fragment, ) - - -def get_host(url): - """ - Deprecated. Use :func:`parse_url` instead. - """ - p = parse_url(url) - return p.scheme or "http", p.hostname, p.port diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/util.py b/contrib/python/pip/pip/_vendor/urllib3/util/util.py new file mode 100644 index 00000000000..35c77e40258 --- /dev/null +++ b/contrib/python/pip/pip/_vendor/urllib3/util/util.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import typing +from types import TracebackType + + +def to_bytes( + x: str | bytes, encoding: str | None = None, errors: str | None = None +) -> bytes: + if isinstance(x, bytes): + return x + elif not isinstance(x, str): + raise TypeError(f"not expecting type {type(x).__name__}") + if encoding or errors: + return x.encode(encoding or "utf-8", errors=errors or "strict") + return x.encode() + + +def to_str( + x: str | bytes, encoding: str | None = None, errors: str | None = None +) -> str: + if isinstance(x, str): + return x + elif not isinstance(x, bytes): + raise TypeError(f"not expecting type {type(x).__name__}") + if encoding or errors: + return x.decode(encoding or "utf-8", errors=errors or "strict") + return x.decode() + + +def reraise( + tp: type[BaseException] | None, + value: BaseException, + tb: TracebackType | None = None, +) -> typing.NoReturn: + try: + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None # type: ignore[assignment] + tb = None diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/wait.py b/contrib/python/pip/pip/_vendor/urllib3/util/wait.py index 21b4590b3dc..aeca0c7ad5b 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/wait.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/wait.py @@ -1,18 +1,10 @@ -import errno +from __future__ import annotations + import select -import sys +import socket from functools import partial -try: - from time import monotonic -except ImportError: - from time import time as monotonic - -__all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"] - - -class NoWayToWaitForSocketError(Exception): - pass +__all__ = ["wait_for_read", "wait_for_write"] # How should we wait on sockets? @@ -37,37 +29,13 @@ class NoWayToWaitForSocketError(Exception): # So: on Windows we use select(), and everywhere else we use poll(). We also # fall back to select() in case poll() is somehow broken or missing. -if sys.version_info >= (3, 5): - # Modern Python, that retries syscalls by default - def _retry_on_intr(fn, timeout): - return fn(timeout) - -else: - # Old and broken Pythons. - def _retry_on_intr(fn, timeout): - if timeout is None: - deadline = float("inf") - else: - deadline = monotonic() + timeout - while True: - try: - return fn(timeout) - # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7 - except (OSError, select.error) as e: - # 'e.args[0]' incantation works for both OSError and select.error - if e.args[0] != errno.EINTR: - raise - else: - timeout = deadline - monotonic() - if timeout < 0: - timeout = 0 - if timeout == float("inf"): - timeout = None - continue - - -def select_wait_for_socket(sock, read=False, write=False, timeout=None): +def select_wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: if not read and not write: raise RuntimeError("must specify at least one of read=True, write=True") rcheck = [] @@ -82,11 +50,16 @@ def select_wait_for_socket(sock, read=False, write=False, timeout=None): # sockets for both conditions. (The stdlib selectors module does the same # thing.) fn = partial(select.select, rcheck, wcheck, wcheck) - rready, wready, xready = _retry_on_intr(fn, timeout) + rready, wready, xready = fn(timeout) return bool(rready or wready or xready) -def poll_wait_for_socket(sock, read=False, write=False, timeout=None): +def poll_wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: if not read and not write: raise RuntimeError("must specify at least one of read=True, write=True") mask = 0 @@ -98,32 +71,33 @@ def poll_wait_for_socket(sock, read=False, write=False, timeout=None): poll_obj.register(sock, mask) # For some reason, poll() takes timeout in milliseconds - def do_poll(t): + def do_poll(t: float | None) -> list[tuple[int, int]]: if t is not None: t *= 1000 return poll_obj.poll(t) - return bool(_retry_on_intr(do_poll, timeout)) - - -def null_wait_for_socket(*args, **kwargs): - raise NoWayToWaitForSocketError("no select-equivalent available") + return bool(do_poll(timeout)) -def _have_working_poll(): +def _have_working_poll() -> bool: # Apparently some systems have a select.poll that fails as soon as you try # to use it, either due to strange configuration or broken monkeypatching # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - _retry_on_intr(poll_obj.poll, 0) + poll_obj.poll(0) except (AttributeError, OSError): return False else: return True -def wait_for_socket(*args, **kwargs): +def wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: # We delay choosing which implementation to use until the first time we're # called. We could do it at import time, but then we might make the wrong # decision if someone goes wild with monkeypatching select.poll after @@ -133,19 +107,17 @@ def wait_for_socket(*args, **kwargs): wait_for_socket = poll_wait_for_socket elif hasattr(select, "select"): wait_for_socket = select_wait_for_socket - else: # Platform-specific: Appengine. - wait_for_socket = null_wait_for_socket - return wait_for_socket(*args, **kwargs) + return wait_for_socket(sock, read, write, timeout) -def wait_for_read(sock, timeout=None): +def wait_for_read(sock: socket.socket, timeout: float | None = None) -> bool: """Waits for reading to be available on a given socket. Returns True if the socket is readable, or False if the timeout expired. """ return wait_for_socket(sock, read=True, timeout=timeout) -def wait_for_write(sock, timeout=None): +def wait_for_write(sock: socket.socket, timeout: float | None = None) -> bool: """Waits for writing to be available on a given socket. Returns True if the socket is readable, or False if the timeout expired. """ diff --git a/contrib/python/pip/pip/_vendor/vendor.txt b/contrib/python/pip/pip/_vendor/vendor.txt index dd2f5f34853..4e85aef4793 100644 --- a/contrib/python/pip/pip/_vendor/vendor.txt +++ b/contrib/python/pip/pip/_vendor/vendor.txt @@ -2,18 +2,17 @@ CacheControl==0.14.4 distlib==0.4.0 distro==1.9.0 msgpack==1.1.2 -packaging==26.0 +packaging==26.2 platformdirs==4.5.1 pyproject-hooks==1.2.0 -requests==2.32.5 - certifi==2026.1.4 +requests==2.33.1 + certifi==2026.2.25 idna==3.11 - urllib3==1.26.20 + urllib3==2.6.3 rich==14.2.0 pygments==2.19.2 resolvelib==1.2.1 setuptools==70.3.0 -tomli==2.3.0 +tomli==2.3.1 tomli-w==1.2.0 truststore==0.10.4 -dependency-groups==1.3.1 diff --git a/contrib/python/pip/ya.make b/contrib/python/pip/ya.make index a28cd22e7d7..43b486c8f17 100644 --- a/contrib/python/pip/ya.make +++ b/contrib/python/pip/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(26.0.1) +VERSION(26.1) LICENSE(MIT) @@ -184,12 +184,6 @@ PY_SRCS( pip/_vendor/certifi/__init__.py pip/_vendor/certifi/__main__.py pip/_vendor/certifi/core.py - pip/_vendor/dependency_groups/__init__.py - pip/_vendor/dependency_groups/__main__.py - pip/_vendor/dependency_groups/_implementation.py - pip/_vendor/dependency_groups/_lint_dependency_groups.py - pip/_vendor/dependency_groups/_pip_wrapper.py - pip/_vendor/dependency_groups/_toml_compat.py pip/_vendor/distlib/__init__.py pip/_vendor/distlib/compat.py pip/_vendor/distlib/resources.py @@ -217,6 +211,9 @@ PY_SRCS( pip/_vendor/packaging/_parser.py pip/_vendor/packaging/_structures.py pip/_vendor/packaging/_tokenizer.py + pip/_vendor/packaging/dependency_groups.py + pip/_vendor/packaging/direct_url.py + pip/_vendor/packaging/errors.py pip/_vendor/packaging/licenses/__init__.py pip/_vendor/packaging/licenses/_spdx.py pip/_vendor/packaging/markers.py @@ -380,35 +377,31 @@ PY_SRCS( pip/_vendor/truststore/_ssl_constants.py pip/_vendor/truststore/_windows.py pip/_vendor/urllib3/__init__.py + pip/_vendor/urllib3/_base_connection.py pip/_vendor/urllib3/_collections.py + pip/_vendor/urllib3/_request_methods.py pip/_vendor/urllib3/_version.py pip/_vendor/urllib3/connection.py pip/_vendor/urllib3/connectionpool.py pip/_vendor/urllib3/contrib/__init__.py - pip/_vendor/urllib3/contrib/_appengine_environ.py - pip/_vendor/urllib3/contrib/_securetransport/__init__.py - pip/_vendor/urllib3/contrib/_securetransport/bindings.py - pip/_vendor/urllib3/contrib/_securetransport/low_level.py - pip/_vendor/urllib3/contrib/appengine.py - pip/_vendor/urllib3/contrib/ntlmpool.py + pip/_vendor/urllib3/contrib/emscripten/__init__.py + pip/_vendor/urllib3/contrib/emscripten/connection.py + pip/_vendor/urllib3/contrib/emscripten/fetch.py + pip/_vendor/urllib3/contrib/emscripten/request.py + pip/_vendor/urllib3/contrib/emscripten/response.py pip/_vendor/urllib3/contrib/pyopenssl.py - pip/_vendor/urllib3/contrib/securetransport.py pip/_vendor/urllib3/contrib/socks.py pip/_vendor/urllib3/exceptions.py pip/_vendor/urllib3/fields.py pip/_vendor/urllib3/filepost.py - pip/_vendor/urllib3/packages/__init__.py - pip/_vendor/urllib3/packages/backports/__init__.py - pip/_vendor/urllib3/packages/backports/makefile.py - pip/_vendor/urllib3/packages/backports/weakref_finalize.py - pip/_vendor/urllib3/packages/six.py + pip/_vendor/urllib3/http2/__init__.py + pip/_vendor/urllib3/http2/connection.py + pip/_vendor/urllib3/http2/probe.py pip/_vendor/urllib3/poolmanager.py - pip/_vendor/urllib3/request.py pip/_vendor/urllib3/response.py pip/_vendor/urllib3/util/__init__.py pip/_vendor/urllib3/util/connection.py pip/_vendor/urllib3/util/proxy.py - pip/_vendor/urllib3/util/queue.py pip/_vendor/urllib3/util/request.py pip/_vendor/urllib3/util/response.py pip/_vendor/urllib3/util/retry.py @@ -417,6 +410,7 @@ PY_SRCS( pip/_vendor/urllib3/util/ssltransport.py pip/_vendor/urllib3/util/timeout.py pip/_vendor/urllib3/util/url.py + pip/_vendor/urllib3/util/util.py pip/_vendor/urllib3/util/wait.py ) @@ -431,8 +425,6 @@ RESOURCE_FILES( pip/_vendor/certifi/LICENSE pip/_vendor/certifi/cacert.pem pip/_vendor/certifi/py.typed - pip/_vendor/dependency_groups/LICENSE.txt - pip/_vendor/dependency_groups/py.typed pip/_vendor/distlib/LICENSE.txt pip/_vendor/distro/LICENSE pip/_vendor/distro/py.typed @@ -461,6 +453,8 @@ RESOURCE_FILES( pip/_vendor/truststore/LICENSE pip/_vendor/truststore/py.typed pip/_vendor/urllib3/LICENSE.txt + pip/_vendor/urllib3/contrib/emscripten/emscripten_fetch_worker.js + pip/_vendor/urllib3/py.typed pip/_vendor/vendor.txt pip/py.typed ) |