diff options
Diffstat (limited to 'contrib/python')
114 files changed, 3286 insertions, 2500 deletions
diff --git a/contrib/python/Twisted/py3/.dist-info/METADATA b/contrib/python/Twisted/py3/.dist-info/METADATA index 22f3987e612..838ec650c53 100644 --- a/contrib/python/Twisted/py3/.dist-info/METADATA +++ b/contrib/python/Twisted/py3/.dist-info/METADATA @@ -1,14 +1,16 @@ Metadata-Version: 2.3 Name: Twisted -Version: 24.7.0 +Version: 24.10.0 Summary: An asynchronous networking framework written in Python Project-URL: Changelog, https://github.com/twisted/twisted/blob/HEAD/NEWS.rst -Project-URL: Documentation, https://docs.twistedmatrix.com/ -Project-URL: Homepage, https://twistedmatrix.com/ -Project-URL: Issues, https://twistedmatrix.com/trac/report +Project-URL: Documentation, https://docs.twisted.org/ +Project-URL: Homepage, https://twisted.org/ +Project-URL: Issues, https://github.com/twisted/twisted/issues Project-URL: Source, https://github.com/twisted/twisted Project-URL: Twitter, https://twitter.com/twistedmatrix -Author-email: Twisted Matrix Laboratories <[email protected]> +Project-URL: Funding-PSF, https://psfmember.org/civicrm/contribute/transact/?reset=1&id=44 +Project-URL: Funding-GitHub, https://github.com/sponsors/twisted +Author-email: Twisted Matrix Community <[email protected]> License: MIT License License-File: LICENSE Classifier: Programming Language :: Python :: 3 @@ -18,9 +20,10 @@ Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Requires-Python: >=3.8.0 -Requires-Dist: attrs>=21.3.0 -Requires-Dist: automat>=0.8.0 +Requires-Dist: attrs>=22.2.0 +Requires-Dist: automat>=24.8.0 Requires-Dist: constantly>=15.1 Requires-Dist: hyperlink>=17.1.1 Requires-Dist: incremental>=24.7.0 @@ -31,7 +34,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'all-non-platform' Requires-Dist: bcrypt>=3.1.3; extra == 'all-non-platform' Requires-Dist: cryptography>=3.3; extra == 'all-non-platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'all-non-platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'all-non-platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'all-non-platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'all-non-platform' Requires-Dist: hypothesis>=6.56; extra == 'all-non-platform' Requires-Dist: idna>=2.4; extra == 'all-non-platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'all-non-platform' @@ -45,7 +49,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'all_non_platform' Requires-Dist: bcrypt>=3.1.3; extra == 'all_non_platform' Requires-Dist: cryptography>=3.3; extra == 'all_non_platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'all_non_platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'all_non_platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'all_non_platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'all_non_platform' Requires-Dist: hypothesis>=6.56; extra == 'all_non_platform' Requires-Dist: idna>=2.4; extra == 'all_non_platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'all_non_platform' @@ -61,6 +66,7 @@ Requires-Dist: cryptography>=3.3; extra == 'conch' Provides-Extra: dev Requires-Dist: coverage~=7.5; extra == 'dev' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'dev' +Requires-Dist: httpx[http2]>=0.27; extra == 'dev' Requires-Dist: hypothesis>=6.56; extra == 'dev' Requires-Dist: pydoctor~=23.9.0; extra == 'dev' Requires-Dist: pyflakes~=2.2; extra == 'dev' @@ -85,7 +91,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'gtk-platform' Requires-Dist: bcrypt>=3.1.3; extra == 'gtk-platform' Requires-Dist: cryptography>=3.3; extra == 'gtk-platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'gtk-platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'gtk-platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'gtk-platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'gtk-platform' Requires-Dist: hypothesis>=6.56; extra == 'gtk-platform' Requires-Dist: idna>=2.4; extra == 'gtk-platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'gtk-platform' @@ -100,7 +107,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'gtk_platform' Requires-Dist: bcrypt>=3.1.3; extra == 'gtk_platform' Requires-Dist: cryptography>=3.3; extra == 'gtk_platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'gtk_platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'gtk_platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'gtk_platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'gtk_platform' Requires-Dist: hypothesis>=6.56; extra == 'gtk_platform' Requires-Dist: idna>=2.4; extra == 'gtk_platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'gtk_platform' @@ -111,14 +119,15 @@ Requires-Dist: pyserial>=3.0; extra == 'gtk_platform' Requires-Dist: pywin32!=226; (platform_system == 'Windows') and extra == 'gtk_platform' Requires-Dist: service-identity>=18.1.0; extra == 'gtk_platform' Provides-Extra: http2 -Requires-Dist: h2<5.0,>=3.0; extra == 'http2' +Requires-Dist: h2<5.0,>=3.2; extra == 'http2' Requires-Dist: priority<2.0,>=1.1.0; extra == 'http2' Provides-Extra: macos-platform Requires-Dist: appdirs>=1.4.0; extra == 'macos-platform' Requires-Dist: bcrypt>=3.1.3; extra == 'macos-platform' Requires-Dist: cryptography>=3.3; extra == 'macos-platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'macos-platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'macos-platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'macos-platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'macos-platform' Requires-Dist: hypothesis>=6.56; extra == 'macos-platform' Requires-Dist: idna>=2.4; extra == 'macos-platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'macos-platform' @@ -135,7 +144,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'macos_platform' Requires-Dist: bcrypt>=3.1.3; extra == 'macos_platform' Requires-Dist: cryptography>=3.3; extra == 'macos_platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'macos_platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'macos_platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'macos_platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'macos_platform' Requires-Dist: hypothesis>=6.56; extra == 'macos_platform' Requires-Dist: idna>=2.4; extra == 'macos_platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'macos_platform' @@ -153,11 +163,12 @@ Requires-Dist: bcrypt>=3.1.3; extra == 'mypy' Requires-Dist: coverage~=7.5; extra == 'mypy' Requires-Dist: cryptography>=3.3; extra == 'mypy' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'mypy' -Requires-Dist: h2<5.0,>=3.0; extra == 'mypy' +Requires-Dist: h2<5.0,>=3.2; extra == 'mypy' +Requires-Dist: httpx[http2]>=0.27; extra == 'mypy' Requires-Dist: hypothesis>=6.56; extra == 'mypy' Requires-Dist: idna>=2.4; extra == 'mypy' -Requires-Dist: mypy-zope~=1.0.3; extra == 'mypy' -Requires-Dist: mypy~=1.8; extra == 'mypy' +Requires-Dist: mypy-zope==1.0.6; extra == 'mypy' +Requires-Dist: mypy==1.10.1; extra == 'mypy' Requires-Dist: priority<2.0,>=1.1.0; extra == 'mypy' Requires-Dist: pydoctor~=23.9.0; extra == 'mypy' Requires-Dist: pyflakes~=2.2; extra == 'mypy' @@ -178,7 +189,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'osx-platform' Requires-Dist: bcrypt>=3.1.3; extra == 'osx-platform' Requires-Dist: cryptography>=3.3; extra == 'osx-platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'osx-platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'osx-platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'osx-platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'osx-platform' Requires-Dist: hypothesis>=6.56; extra == 'osx-platform' Requires-Dist: idna>=2.4; extra == 'osx-platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'osx-platform' @@ -195,7 +207,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'osx_platform' Requires-Dist: bcrypt>=3.1.3; extra == 'osx_platform' Requires-Dist: cryptography>=3.3; extra == 'osx_platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'osx_platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'osx_platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'osx_platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'osx_platform' Requires-Dist: hypothesis>=6.56; extra == 'osx_platform' Requires-Dist: idna>=2.4; extra == 'osx_platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'osx_platform' @@ -212,6 +225,7 @@ Requires-Dist: pyserial>=3.0; extra == 'serial' Requires-Dist: pywin32!=226; (platform_system == 'Windows') and extra == 'serial' Provides-Extra: test Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'test' +Requires-Dist: httpx[http2]>=0.27; extra == 'test' Requires-Dist: hypothesis>=6.56; extra == 'test' Requires-Dist: pyhamcrest>=2; extra == 'test' Provides-Extra: tls @@ -223,7 +237,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'windows-platform' Requires-Dist: bcrypt>=3.1.3; extra == 'windows-platform' Requires-Dist: cryptography>=3.3; extra == 'windows-platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'windows-platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'windows-platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'windows-platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'windows-platform' Requires-Dist: hypothesis>=6.56; extra == 'windows-platform' Requires-Dist: idna>=2.4; extra == 'windows-platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'windows-platform' @@ -239,7 +254,8 @@ Requires-Dist: appdirs>=1.4.0; extra == 'windows_platform' Requires-Dist: bcrypt>=3.1.3; extra == 'windows_platform' Requires-Dist: cryptography>=3.3; extra == 'windows_platform' Requires-Dist: cython-test-exception-raiser<2,>=1.0.2; extra == 'windows_platform' -Requires-Dist: h2<5.0,>=3.0; extra == 'windows_platform' +Requires-Dist: h2<5.0,>=3.2; extra == 'windows_platform' +Requires-Dist: httpx[http2]>=0.27; extra == 'windows_platform' Requires-Dist: hypothesis>=6.56; extra == 'windows_platform' Requires-Dist: idna>=2.4; extra == 'windows_platform' Requires-Dist: priority<2.0,>=1.1.0; extra == 'windows_platform' @@ -263,6 +279,17 @@ Twisted For information on changes in this release, see the `NEWS <https://github.com/twisted/twisted/blob/trunk/NEWS.rst>`_ file. +Sponsors +-------- + +Twisted is an MIT-licensed open source project with its ongoing development made possible entirely by the support of community and these awesome sponsors. +If you'd like to join them, please consider `sponsoring Twisted's <https://docs.twisted.org/en/latest/development/sponsorship.html>`_ development. + +|thinkst|_ + +|sftpplus|_ + + What is this? ------------- @@ -288,13 +315,13 @@ To install the latest version of Twisted using pip:: $ pip install twisted -Additional instructions for installing this software are in `the installation instructions <https://docs.twisted.org/en/latest/installations.rst>`_. +Additional instructions for installing this software are in `the installation instructions <https://docs.twisted.org/en/latest/installation.html>`_. Documentation and Support ------------------------- -Twisted's documentation is available from the `Twisted Matrix website <https://twistedmatrix.com/documents/current/>`_. +Twisted's documentation is available from the `Twisted Matrix Read The Docs website <https://docs.twisted.org/>`_. This documentation contains how-tos, code examples, and an API reference. Help is also available on the `Twisted mailing list <https://mail.python.org/mailman3/lists/twisted.python.org/>`_. @@ -329,7 +356,7 @@ Some of these tests may fail if you: Static Code Checkers -------------------- -You can ensure that code complies to Twisted `coding standards <https://twistedmatrix.com/documents/current/core/development/policy/coding-standard.html>`_:: +You can ensure that code complies to Twisted `coding standards <https://docs.twisted.org/en/latest/development/coding-standard.html>`_:: $ tox -e lint # run pre-commit to check coding stanards $ tox -e mypy # run MyPy static type checker to check for type errors @@ -375,3 +402,11 @@ Again, see the included `LICENSE <https://github.com/twisted/twisted/blob/trunk/ .. |rtd| image:: https://readthedocs.org/projects/twisted/badge/?version=latest&style=flat .. _rtd: https://docs.twistedmatrix.com + +.. |thinkst| image:: https://github.com/user-attachments/assets/a5b52432-2d18-4d91-a3c9-772fb2e02781 + :alt: Thinkst Canary +.. _thinkst: https://thinkst.com/ + +.. |sftpplus| image:: https://github.com/user-attachments/assets/5f585316-c7e8-4ef1-8fbb-923f0756ceed + :alt: SFTPPlus +.. _sftpplus: https://www.sftpplus.com/ diff --git a/contrib/python/Twisted/py3/README.rst b/contrib/python/Twisted/py3/README.rst index 1d2f85648cc..a30c266eb37 100644 --- a/contrib/python/Twisted/py3/README.rst +++ b/contrib/python/Twisted/py3/README.rst @@ -9,6 +9,17 @@ Twisted For information on changes in this release, see the `NEWS <NEWS.rst>`_ file. +Sponsors +-------- + +Twisted is an MIT-licensed open source project with its ongoing development made possible entirely by the support of community and these awesome sponsors. +If you'd like to join them, please consider `sponsoring Twisted's <https://docs.twisted.org/en/latest/development/sponsorship.html>`_ development. + +|thinkst|_ + +|sftpplus|_ + + What is this? ------------- @@ -34,13 +45,13 @@ To install the latest version of Twisted using pip:: $ pip install twisted -Additional instructions for installing this software are in `the installation instructions <https://docs.twisted.org/en/latest/installations.rst>`_. +Additional instructions for installing this software are in `the installation instructions <https://docs.twisted.org/en/latest/installation.html>`_. Documentation and Support ------------------------- -Twisted's documentation is available from the `Twisted Matrix website <https://twistedmatrix.com/documents/current/>`_. +Twisted's documentation is available from the `Twisted Matrix Read The Docs website <https://docs.twisted.org/>`_. This documentation contains how-tos, code examples, and an API reference. Help is also available on the `Twisted mailing list <https://mail.python.org/mailman3/lists/twisted.python.org/>`_. @@ -75,7 +86,7 @@ Some of these tests may fail if you: Static Code Checkers -------------------- -You can ensure that code complies to Twisted `coding standards <https://twistedmatrix.com/documents/current/core/development/policy/coding-standard.html>`_:: +You can ensure that code complies to Twisted `coding standards <https://docs.twisted.org/en/latest/development/coding-standard.html>`_:: $ tox -e lint # run pre-commit to check coding stanards $ tox -e mypy # run MyPy static type checker to check for type errors @@ -121,3 +132,11 @@ Again, see the included `LICENSE <LICENSE>`_ file for specific legal details. .. |rtd| image:: https://readthedocs.org/projects/twisted/badge/?version=latest&style=flat .. _rtd: https://docs.twistedmatrix.com + +.. |thinkst| image:: https://github.com/user-attachments/assets/a5b52432-2d18-4d91-a3c9-772fb2e02781 + :alt: Thinkst Canary +.. _thinkst: https://thinkst.com/ + +.. |sftpplus| image:: https://github.com/user-attachments/assets/5f585316-c7e8-4ef1-8fbb-923f0756ceed + :alt: SFTPPlus +.. _sftpplus: https://www.sftpplus.com/ diff --git a/contrib/python/Twisted/py3/twisted/_threads/_convenience.py b/contrib/python/Twisted/py3/twisted/_threads/_convenience.py index deff5764624..3d8c6682f01 100644 --- a/contrib/python/Twisted/py3/twisted/_threads/_convenience.py +++ b/contrib/python/Twisted/py3/twisted/_threads/_convenience.py @@ -18,13 +18,13 @@ class Quit: @type isSet: L{bool} """ - def __init__(self): + def __init__(self) -> None: """ Create a L{Quit} un-set. """ self.isSet = False - def set(self): + def set(self) -> None: """ Set the flag if it has not been set. @@ -33,7 +33,7 @@ class Quit: self.check() self.isSet = True - def check(self): + def check(self) -> None: """ Check if the flag has been set. diff --git a/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py b/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py index cab9135f874..b2e01752e52 100644 --- a/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py +++ b/contrib/python/Twisted/py3/twisted/_threads/_ithreads.py @@ -45,7 +45,7 @@ class IWorker(Interface): @raise AlreadyQuit: if C{quit} has been called. """ - def quit(): + def quit() -> None: """ Free any resources associated with this L{IWorker} and cause it to reject all future work. diff --git a/contrib/python/Twisted/py3/twisted/_threads/_memory.py b/contrib/python/Twisted/py3/twisted/_threads/_memory.py index 4c56db02ae9..5483b8e5770 100644 --- a/contrib/python/Twisted/py3/twisted/_threads/_memory.py +++ b/contrib/python/Twisted/py3/twisted/_threads/_memory.py @@ -6,16 +6,25 @@ Implementation of an in-memory worker that defers execution. """ +from __future__ import annotations + +from enum import Enum, auto +from typing import Callable, Literal from zope.interface import implementer -from . import IWorker from ._convenience import Quit +from ._ithreads import IExclusiveWorker + + +class NoMore(Enum): + Work = auto() + -NoMoreWork = object() +NoMoreWork = NoMore.Work -@implementer(IWorker) +@implementer(IExclusiveWorker) class MemoryWorker: """ An L{IWorker} that queues work for later performance. @@ -24,14 +33,17 @@ class MemoryWorker: @type _quit: L{Quit} """ - def __init__(self, pending=list): + def __init__( + self, + pending: Callable[[], list[Callable[[], object] | Literal[NoMore.Work]]] = list, + ) -> None: """ Create a L{MemoryWorker}. """ self._quit = Quit() self._pending = pending() - def do(self, work): + def do(self, work: Callable[[], object]) -> None: """ Queue some work for to perform later; see L{createMemoryWorker}. @@ -40,7 +52,7 @@ class MemoryWorker: self._quit.check() self._pending.append(work) - def quit(self): + def quit(self) -> None: """ Quit this worker. """ @@ -48,22 +60,23 @@ class MemoryWorker: self._pending.append(NoMoreWork) -def createMemoryWorker(): +def createMemoryWorker() -> tuple[MemoryWorker, Callable[[], bool]]: """ Create an L{IWorker} that does nothing but defer work, to be performed later. @return: a worker that will enqueue work to perform later, and a callable that will perform one element of that work. - @rtype: 2-L{tuple} of (L{IWorker}, L{callable}) """ - def perform(): + def perform() -> bool: if not worker._pending: return False - if worker._pending[0] is NoMoreWork: + peek = worker._pending[0] + if peek is NoMoreWork: return False - worker._pending.pop(0)() + worker._pending.pop(0) + peek() return True worker = MemoryWorker() diff --git a/contrib/python/Twisted/py3/twisted/_threads/_team.py b/contrib/python/Twisted/py3/twisted/_threads/_team.py index d15ae04242d..95e40cffa95 100644 --- a/contrib/python/Twisted/py3/twisted/_threads/_team.py +++ b/contrib/python/Twisted/py3/twisted/_threads/_team.py @@ -158,7 +158,7 @@ class Team: if self._shouldQuitCoordinator and self._busyCount == 0: self._coordinator.quit() - def do(self, task: Callable[[], None]) -> None: + def do(self, task: Callable[[], object]) -> None: """ Perform some work in a worker created by C{createWorker}. diff --git a/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py b/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py index e7ffc097580..a4617a1974c 100644 --- a/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py +++ b/contrib/python/Twisted/py3/twisted/_threads/_threadworker.py @@ -5,16 +5,41 @@ """ Implementation of an L{IWorker} based on native threads and queues. """ +from __future__ import annotations +from enum import Enum, auto +from typing import TYPE_CHECKING, Callable, Iterator, Literal, Protocol, TypeVar -from typing import Callable +if TYPE_CHECKING: + import threading from zope.interface import implementer from ._convenience import Quit from ._ithreads import IExclusiveWorker -_stop = object() + +class Stop(Enum): + Thread = auto() + + +StopThread = Stop.Thread + +T = TypeVar("T") +U = TypeVar("U") + + +class SimpleQueue(Protocol[T]): + def put(self, item: T) -> None: + ... + + def get(self) -> T: + ... + + +# when the sentinel value is a literal in a union, this is how iter works +smartiter: Callable[[Callable[[], T | U], U], Iterator[T]] +smartiter = iter # type:ignore[assignment] @implementer(IExclusiveWorker) @@ -27,25 +52,26 @@ class ThreadWorker: thread. """ - def __init__(self, startThread, queue): + def __init__( + self, + startThread: Callable[[Callable[[], object]], object], + queue: SimpleQueue[Callable[[], object] | Literal[Stop.Thread]], + ): """ Create a L{ThreadWorker} with a function to start a thread and a queue to use to communicate with that thread. @param startThread: a callable that takes a callable to run in another thread. - @type startThread: callable taking a 0-argument callable and returning - nothing. @param queue: A L{Queue} to use to give tasks to the thread created by C{startThread}. - @type queue: L{Queue} """ self._q = queue self._hasQuit = Quit() - def work(): - for task in iter(queue.get, _stop): + def work() -> None: + for task in smartiter(queue.get, StopThread): task() startThread(work) @@ -59,14 +85,22 @@ class ThreadWorker: self._hasQuit.check() self._q.put(task) - def quit(self): + def quit(self) -> None: """ Reject all future work and stop the thread started by C{__init__}. """ # Reject all future work. Set this _before_ enqueueing _stop, so # that no work is ever enqueued _after_ _stop. self._hasQuit.set() - self._q.put(_stop) + self._q.put(StopThread) + + +class SimpleLock(Protocol): + def acquire(self) -> bool: + ... + + def release(self) -> None: + ... @implementer(IExclusiveWorker) @@ -75,7 +109,7 @@ class LockWorker: An L{IWorker} implemented based on a mutual-exclusion lock. """ - def __init__(self, lock, local): + def __init__(self, lock: SimpleLock, local: threading.local): """ @param lock: A mutual-exclusion lock, with C{acquire} and C{release} methods. @@ -85,7 +119,7 @@ class LockWorker: @type local: L{threading.local} """ self._quit = Quit() - self._lock = lock + self._lock: SimpleLock | None = lock self._local = local def do(self, work: Callable[[], None]) -> None: @@ -101,6 +135,7 @@ class LockWorker: self._quit.check() working = getattr(local, "working", None) if working is None: + assert lock is not None, "LockWorker used after quit()" working = local.working = [] working.append(work) lock.acquire() @@ -113,7 +148,7 @@ class LockWorker: else: working.append(work) - def quit(self): + def quit(self) -> None: """ Quit this L{LockWorker}. """ diff --git a/contrib/python/Twisted/py3/twisted/_version.py b/contrib/python/Twisted/py3/twisted/_version.py index f1f493452d5..b43dec8e331 100644 --- a/contrib/python/Twisted/py3/twisted/_version.py +++ b/contrib/python/Twisted/py3/twisted/_version.py @@ -7,5 +7,5 @@ Provides Twisted version information. from incremental import Version -__version__ = Version("Twisted", 24, 7, 0) +__version__ = Version("Twisted", 24, 10, 0) __all__ = ["__version__"] diff --git a/contrib/python/Twisted/py3/twisted/application/_client_service.py b/contrib/python/Twisted/py3/twisted/application/_client_service.py new file mode 100644 index 00000000000..32b6c4c5234 --- /dev/null +++ b/contrib/python/Twisted/py3/twisted/application/_client_service.py @@ -0,0 +1,596 @@ +# -*- test-case-name: twisted.application.test.test_internet,twisted.test.test_application,twisted.test.test_cooperator -*- + +""" +Implementation of L{twisted.application.internet.ClientService}, particularly +its U{automat <https://automat.readthedocs.org/>} state machine. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from random import random as _goodEnoughRandom +from typing import Callable, Optional, Protocol as TypingProtocol, TypeVar, Union + +from zope.interface import implementer + +from automat import TypeMachineBuilder, pep614 + +from twisted.application.service import Service +from twisted.internet.defer import ( + CancelledError, + Deferred, + fail, + maybeDeferred, + succeed, +) +from twisted.internet.interfaces import ( + IAddress, + IDelayedCall, + IProtocol, + IProtocolFactory, + IReactorTime, + IStreamClientEndpoint, + ITransport, +) +from twisted.logger import Logger +from twisted.python.failure import Failure + +T = TypeVar("T") + + +def _maybeGlobalReactor(maybeReactor: Optional[T]) -> T: + """ + @return: the argument, or the global reactor if the argument is L{None}. + """ + if maybeReactor is None: + from twisted.internet import reactor + + return reactor # type:ignore[return-value] + else: + return maybeReactor + + +class _Client(TypingProtocol): + def start(self) -> None: + """ + Start this L{ClientService}, initiating the connection retry loop. + """ + + def stop(self) -> Deferred[None]: + """ + Stop trying to connect and disconnect any current connection. + + @return: a L{Deferred} that fires when all outstanding connections are + closed and all in-progress connection attempts halted. + """ + + def _connectionMade(self, protocol: _ReconnectingProtocolProxy) -> None: + """ + A connection has been made. + + @param protocol: The protocol of the connection. + """ + + def _connectionFailed(self, failure: Failure) -> None: + """ + Deliver connection failures to any L{ClientService.whenConnected} + L{Deferred}s that have met their failAfterFailures threshold. + + @param failure: the Failure to fire the L{Deferred}s with. + """ + + def _reconnect(self, failure: Optional[Failure] = None) -> None: + """ + The wait between connection attempts is done. + """ + + def _clientDisconnected(self, failure: Optional[Failure] = None) -> None: + """ + The current connection has been disconnected. + """ + + def whenConnected( + self, /, failAfterFailures: Optional[int] = None + ) -> Deferred[IProtocol]: + """ + Retrieve the currently-connected L{Protocol}, or the next one to + connect. + + @param failAfterFailures: number of connection failures after which the + Deferred will deliver a Failure (None means the Deferred will only + fail if/when the service is stopped). Set this to 1 to make the + very first connection failure signal an error. Use 2 to allow one + failure but signal an error if the subsequent retry then fails. + + @return: a Deferred that fires with a protocol produced by the factory + passed to C{__init__}. It may: + + - fire with L{IProtocol} + + - fail with L{CancelledError} when the service is stopped + + - fail with e.g. + L{DNSLookupError<twisted.internet.error.DNSLookupError>} or + L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>} + when the number of consecutive failed connection attempts + equals the value of "failAfterFailures" + """ + + +@implementer(IProtocol) +class _ReconnectingProtocolProxy: + """ + A proxy for a Protocol to provide connectionLost notification to a client + connection service, in support of reconnecting when connections are lost. + """ + + def __init__( + self, protocol: IProtocol, lostNotification: Callable[[Failure], None] + ) -> None: + """ + Create a L{_ReconnectingProtocolProxy}. + + @param protocol: the application-provided L{interfaces.IProtocol} + provider. + @type protocol: provider of L{interfaces.IProtocol} which may + additionally provide L{interfaces.IHalfCloseableProtocol} and + L{interfaces.IFileDescriptorReceiver}. + + @param lostNotification: a 1-argument callable to invoke with the + C{reason} when the connection is lost. + """ + self._protocol = protocol + self._lostNotification = lostNotification + + def makeConnection(self, transport: ITransport) -> None: + self._transport = transport + self._protocol.makeConnection(transport) + + def connectionLost(self, reason: Failure) -> None: + """ + The connection was lost. Relay this information. + + @param reason: The reason the connection was lost. + + @return: the underlying protocol's result + """ + try: + return self._protocol.connectionLost(reason) + finally: + self._lostNotification(reason) + + def __getattr__(self, item: str) -> object: + return getattr(self._protocol, item) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} wrapping {self._protocol!r}>" + + +@implementer(IProtocolFactory) +class _DisconnectFactory: + """ + A L{_DisconnectFactory} is a proxy for L{IProtocolFactory} that catches + C{connectionLost} notifications and relays them. + """ + + def __init__( + self, + protocolFactory: IProtocolFactory, + protocolDisconnected: Callable[[Failure], None], + ) -> None: + self._protocolFactory = protocolFactory + self._protocolDisconnected = protocolDisconnected + + def buildProtocol(self, addr: IAddress) -> Optional[IProtocol]: + """ + Create a L{_ReconnectingProtocolProxy} with the disconnect-notification + callback we were called with. + + @param addr: The address the connection is coming from. + + @return: a L{_ReconnectingProtocolProxy} for a protocol produced by + C{self._protocolFactory} + """ + built = self._protocolFactory.buildProtocol(addr) + if built is None: + return None + return _ReconnectingProtocolProxy(built, self._protocolDisconnected) + + def __getattr__(self, item: str) -> object: + return getattr(self._protocolFactory, item) + + def __repr__(self) -> str: + return "<{} wrapping {!r}>".format( + self.__class__.__name__, self._protocolFactory + ) + + +def _deinterface(o: object) -> None: + """ + Remove the special runtime attributes set by L{implementer} so that a class + can proxy through those attributes with C{__getattr__} and thereby forward + optionally-provided interfaces by the delegated class. + """ + for zopeSpecial in ["__providedBy__", "__provides__", "__implemented__"]: + delattr(o, zopeSpecial) + + +_deinterface(_DisconnectFactory) +_deinterface(_ReconnectingProtocolProxy) + + +@dataclass +class _Core: + """ + Shared core for ClientService state machine. + """ + + # required parameters + endpoint: IStreamClientEndpoint + factory: IProtocolFactory + timeoutForAttempt: Callable[[int], float] + clock: IReactorTime + prepareConnection: Optional[Callable[[IProtocol], object]] + + # internal state + stopWaiters: list[Deferred[None]] = field(default_factory=list) + awaitingConnected: list[tuple[Deferred[IProtocol], Optional[int]]] = field( + default_factory=list + ) + failedAttempts: int = 0 + log: Logger = Logger() + + def waitForStop(self) -> Deferred[None]: + self.stopWaiters.append(Deferred()) + return self.stopWaiters[-1] + + def unawait(self, value: Union[IProtocol, Failure]) -> None: + self.awaitingConnected, waiting = [], self.awaitingConnected + for w, remaining in waiting: + w.callback(value) + + def cancelConnectWaiters(self) -> None: + self.unawait(Failure(CancelledError())) + + def finishStopping(self) -> None: + self.stopWaiters, waiting = [], self.stopWaiters + for w in waiting: + w.callback(None) + + +def makeMachine() -> Callable[[_Core], _Client]: + machine = TypeMachineBuilder(_Client, _Core) + + def waitForRetry( + c: _Client, s: _Core, failure: Optional[Failure] = None + ) -> IDelayedCall: + s.failedAttempts += 1 + delay = s.timeoutForAttempt(s.failedAttempts) + s.log.info( + "Scheduling retry {attempt} to connect {endpoint} in {delay} seconds.", + attempt=s.failedAttempts, + endpoint=s.endpoint, + delay=delay, + ) + return s.clock.callLater(delay, c._reconnect) + + def rememberConnection( + c: _Client, s: _Core, protocol: _ReconnectingProtocolProxy + ) -> _ReconnectingProtocolProxy: + s.failedAttempts = 0 + s.unawait(protocol._protocol) + return protocol + + def attemptConnection( + c: _Client, s: _Core, failure: Optional[Failure] = None + ) -> Deferred[_ReconnectingProtocolProxy]: + factoryProxy = _DisconnectFactory(s.factory, c._clientDisconnected) + connecting: Deferred[IProtocol] = s.endpoint.connect(factoryProxy) + + def prepare( + protocol: _ReconnectingProtocolProxy, + ) -> Deferred[_ReconnectingProtocolProxy]: + if s.prepareConnection is not None: + return maybeDeferred(s.prepareConnection, protocol).addCallback( + lambda _: protocol + ) + return succeed(protocol) + + # endpoint.connect() is actually generic on the type of the protocol, + # but this is not expressible via zope.interface, so we have to cast + # https://github.com/Shoobx/mypy-zope/issues/95 + connectingProxy: Deferred[_ReconnectingProtocolProxy] + connectingProxy = connecting # type:ignore[assignment] + ( + connectingProxy.addCallback(prepare) + .addCallback(c._connectionMade) + .addErrback(c._connectionFailed) + ) + return connectingProxy + + # States: + Init = machine.state("Init") + Connecting = machine.state("Connecting", attemptConnection) + Stopped = machine.state("Stopped") + Waiting = machine.state("Waiting", waitForRetry) + Connected = machine.state("Connected", rememberConnection) + Disconnecting = machine.state("Disconnecting") + Restarting = machine.state("Restarting") + Stopped = machine.state("Stopped") + + # Behavior-less state transitions: + Init.upon(_Client.start).to(Connecting).returns(None) + Connecting.upon(_Client.start).loop().returns(None) + Connecting.upon(_Client._connectionMade).to(Connected).returns(None) + Waiting.upon(_Client.start).loop().returns(None) + Waiting.upon(_Client._reconnect).to(Connecting).returns(None) + Connected.upon(_Client._connectionFailed).to(Waiting).returns(None) + Connected.upon(_Client.start).loop().returns(None) + Connected.upon(_Client._clientDisconnected).to(Waiting).returns(None) + Disconnecting.upon(_Client.start).to(Restarting).returns(None) + Restarting.upon(_Client.start).to(Restarting).returns(None) + Stopped.upon(_Client.start).to(Connecting).returns(None) + + # Behavior-full state transitions: + @pep614(Init.upon(_Client.stop).to(Stopped)) + @pep614(Stopped.upon(_Client.stop).to(Stopped)) + def immediateStop(c: _Client, s: _Core) -> Deferred[None]: + return succeed(None) + + @pep614(Connecting.upon(_Client.stop).to(Disconnecting)) + def connectingStop( + c: _Client, s: _Core, attempt: Deferred[_ReconnectingProtocolProxy] + ) -> Deferred[None]: + waited = s.waitForStop() + attempt.cancel() + return waited + + @pep614(Connecting.upon(_Client._connectionFailed, nodata=True).to(Waiting)) + def failedWhenConnecting(c: _Client, s: _Core, failure: Failure) -> None: + ready = [] + notReady: list[tuple[Deferred[IProtocol], Optional[int]]] = [] + for w, remaining in s.awaitingConnected: + if remaining is None: + notReady.append((w, remaining)) + elif remaining <= 1: + ready.append(w) + else: + notReady.append((w, remaining - 1)) + s.awaitingConnected = notReady + for w in ready: + w.callback(failure) + + @pep614(Waiting.upon(_Client.stop).to(Stopped)) + def stop(c: _Client, s: _Core, futureRetry: IDelayedCall) -> Deferred[None]: + waited = s.waitForStop() + s.cancelConnectWaiters() + futureRetry.cancel() + s.finishStopping() + return waited + + @pep614(Connected.upon(_Client.stop).to(Disconnecting)) + def stopWhileConnected( + c: _Client, s: _Core, protocol: _ReconnectingProtocolProxy + ) -> Deferred[None]: + waited = s.waitForStop() + protocol._transport.loseConnection() + return waited + + @pep614(Connected.upon(_Client.whenConnected).loop()) + def whenConnectedWhenConnected( + c: _Client, + s: _Core, + protocol: _ReconnectingProtocolProxy, + failAfterFailures: Optional[int] = None, + ) -> Deferred[IProtocol]: + return succeed(protocol._protocol) + + @pep614(Disconnecting.upon(_Client.stop).loop()) + @pep614(Restarting.upon(_Client.stop).to(Disconnecting)) + def discoStop(c: _Client, s: _Core) -> Deferred[None]: + return s.waitForStop() + + @pep614(Disconnecting.upon(_Client._connectionFailed).to(Stopped)) + @pep614(Disconnecting.upon(_Client._clientDisconnected).to(Stopped)) + def disconnectingFinished( + c: _Client, s: _Core, failure: Optional[Failure] = None + ) -> None: + s.cancelConnectWaiters() + s.finishStopping() + + @pep614(Connecting.upon(_Client.whenConnected, nodata=True).loop()) + @pep614(Waiting.upon(_Client.whenConnected, nodata=True).loop()) + @pep614(Init.upon(_Client.whenConnected).to(Init)) + @pep614(Restarting.upon(_Client.whenConnected).to(Restarting)) + @pep614(Disconnecting.upon(_Client.whenConnected).to(Disconnecting)) + def awaitingConnection( + c: _Client, s: _Core, failAfterFailures: Optional[int] = None + ) -> Deferred[IProtocol]: + result: Deferred[IProtocol] = Deferred() + s.awaitingConnected.append((result, failAfterFailures)) + return result + + @pep614(Restarting.upon(_Client._clientDisconnected).to(Connecting)) + def restartDone(c: _Client, s: _Core, failure: Optional[Failure] = None) -> None: + s.finishStopping() + + @pep614(Stopped.upon(_Client.whenConnected).to(Stopped)) + def notGoingToConnect( + c: _Client, s: _Core, failAfterFailures: Optional[int] = None + ) -> Deferred[IProtocol]: + return fail(CancelledError()) + + return machine.build() + + +def backoffPolicy( + initialDelay: float = 1.0, + maxDelay: float = 60.0, + factor: float = 1.5, + jitter: Callable[[], float] = _goodEnoughRandom, +) -> Callable[[int], float]: + """ + A timeout policy for L{ClientService} which computes an exponential backoff + interval with configurable parameters. + + @since: 16.1.0 + + @param initialDelay: Delay for the first reconnection attempt (default + 1.0s). + @type initialDelay: L{float} + + @param maxDelay: Maximum number of seconds between connection attempts + (default 60 seconds, or one minute). Note that this value is before + jitter is applied, so the actual maximum possible delay is this value + plus the maximum possible result of C{jitter()}. + @type maxDelay: L{float} + + @param factor: A multiplicative factor by which the delay grows on each + failed reattempt. Default: 1.5. + @type factor: L{float} + + @param jitter: A 0-argument callable that introduces noise into the delay. + By default, C{random.random}, i.e. a pseudorandom floating-point value + between zero and one. + @type jitter: 0-argument callable returning L{float} + + @return: a 1-argument callable that, given an attempt count, returns a + floating point number; the number of seconds to delay. + @rtype: see L{ClientService.__init__}'s C{retryPolicy} argument. + """ + + def policy(attempt: int) -> float: + try: + delay = min(initialDelay * (factor ** min(100, attempt)), maxDelay) + except OverflowError: + delay = maxDelay + return delay + jitter() + + return policy + + +_defaultPolicy = backoffPolicy() +ClientMachine = makeMachine() + + +class ClientService(Service): + """ + A L{ClientService} maintains a single outgoing connection to a client + endpoint, reconnecting after a configurable timeout when a connection + fails, either before or after connecting. + + @since: 16.1.0 + """ + + _log = Logger() + + def __init__( + self, + endpoint: IStreamClientEndpoint, + factory: IProtocolFactory, + retryPolicy: Optional[Callable[[int], float]] = None, + clock: Optional[IReactorTime] = None, + prepareConnection: Optional[Callable[[IProtocol], object]] = None, + ): + """ + @param endpoint: A L{stream client endpoint + <interfaces.IStreamClientEndpoint>} provider which will be used to + connect when the service starts. + + @param factory: A L{protocol factory <interfaces.IProtocolFactory>} + which will be used to create clients for the endpoint. + + @param retryPolicy: A policy configuring how long L{ClientService} will + wait between attempts to connect to C{endpoint}; a callable taking + (the number of failed connection attempts made in a row (L{int})) + and returning the number of seconds to wait before making another + attempt. + + @param clock: The clock used to schedule reconnection. It's mainly + useful to be parametrized in tests. If the factory is serialized, + this attribute will not be serialized, and the default value (the + reactor) will be restored when deserialized. + + @param prepareConnection: A single argument L{callable} that may return + a L{Deferred}. It will be called once with the L{protocol + <interfaces.IProtocol>} each time a new connection is made. It may + call methods on the protocol to prepare it for use (e.g. + authenticate) or validate it (check its health). + + The C{prepareConnection} callable may raise an exception or return + a L{Deferred} which fails to reject the connection. A rejected + connection is not used to fire an L{Deferred} returned by + L{whenConnected}. Instead, L{ClientService} handles the failure + and continues as if the connection attempt were a failure + (incrementing the counter passed to C{retryPolicy}). + + L{Deferred}s returned by L{whenConnected} will not fire until any + L{Deferred} returned by the C{prepareConnection} callable fire. + Otherwise its successful return value is consumed, but ignored. + + Present Since Twisted 18.7.0 + """ + clock = _maybeGlobalReactor(clock) + retryPolicy = _defaultPolicy if retryPolicy is None else retryPolicy + + self._machine: _Client = ClientMachine( + _Core( + endpoint, + factory, + retryPolicy, + clock, + prepareConnection=prepareConnection, + log=self._log, + ) + ) + + def whenConnected( + self, failAfterFailures: Optional[int] = None + ) -> Deferred[IProtocol]: + """ + Retrieve the currently-connected L{Protocol}, or the next one to + connect. + + @param failAfterFailures: number of connection failures after which + the Deferred will deliver a Failure (None means the Deferred will + only fail if/when the service is stopped). Set this to 1 to make + the very first connection failure signal an error. Use 2 to + allow one failure but signal an error if the subsequent retry + then fails. + @type failAfterFailures: L{int} or None + + @return: a Deferred that fires with a protocol produced by the + factory passed to C{__init__} + @rtype: L{Deferred} that may: + + - fire with L{IProtocol} + + - fail with L{CancelledError} when the service is stopped + + - fail with e.g. + L{DNSLookupError<twisted.internet.error.DNSLookupError>} or + L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>} + when the number of consecutive failed connection attempts + equals the value of "failAfterFailures" + """ + return self._machine.whenConnected(failAfterFailures) + + def startService(self) -> None: + """ + Start this L{ClientService}, initiating the connection retry loop. + """ + if self.running: + self._log.warn("Duplicate ClientService.startService {log_source}") + return + super().startService() + self._machine.start() + + def stopService(self) -> Deferred[None]: + """ + Stop attempting to reconnect and close any existing connections. + + @return: a L{Deferred} that fires when all outstanding connections are + closed and all in-progress connection attempts halted. + """ + super().stopService() + return self._machine.stop() diff --git a/contrib/python/Twisted/py3/twisted/application/internet.py b/contrib/python/Twisted/py3/twisted/application/internet.py index 9e702f7e844..8bcc9722a0a 100644 --- a/contrib/python/Twisted/py3/twisted/application/internet.py +++ b/contrib/python/Twisted/py3/twisted/application/internet.py @@ -38,35 +38,13 @@ reactor.listen/connect* methods for more information. """ -from random import random as _goodEnoughRandom from typing import List -from automat import MethodicalMachine - from twisted.application import service from twisted.internet import task -from twisted.internet.defer import ( - CancelledError, - Deferred, - fail, - maybeDeferred, - succeed, -) -from twisted.logger import Logger +from twisted.internet.defer import CancelledError from twisted.python import log -from twisted.python.failure import Failure - - -def _maybeGlobalReactor(maybeReactor): - """ - @return: the argument, or the global reactor if the argument is L{None}. - """ - if maybeReactor is None: - from twisted.internet import reactor - - return reactor - else: - return maybeReactor +from ._client_service import ClientService, _maybeGlobalReactor, backoffPolicy class _VolatileDataService(service.Service): @@ -429,764 +407,6 @@ class StreamServerEndpointService(service.Service): return d -class _ReconnectingProtocolProxy: - """ - A proxy for a Protocol to provide connectionLost notification to a client - connection service, in support of reconnecting when connections are lost. - """ - - def __init__(self, protocol, lostNotification): - """ - Create a L{_ReconnectingProtocolProxy}. - - @param protocol: the application-provided L{interfaces.IProtocol} - provider. - @type protocol: provider of L{interfaces.IProtocol} which may - additionally provide L{interfaces.IHalfCloseableProtocol} and - L{interfaces.IFileDescriptorReceiver}. - - @param lostNotification: a 1-argument callable to invoke with the - C{reason} when the connection is lost. - """ - self._protocol = protocol - self._lostNotification = lostNotification - - def connectionLost(self, reason): - """ - The connection was lost. Relay this information. - - @param reason: The reason the connection was lost. - - @return: the underlying protocol's result - """ - try: - return self._protocol.connectionLost(reason) - finally: - self._lostNotification(reason) - - def __getattr__(self, item): - return getattr(self._protocol, item) - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} wrapping {self._protocol!r}>" - - -class _DisconnectFactory: - """ - A L{_DisconnectFactory} is a proxy for L{IProtocolFactory} that catches - C{connectionLost} notifications and relays them. - """ - - def __init__(self, protocolFactory, protocolDisconnected): - self._protocolFactory = protocolFactory - self._protocolDisconnected = protocolDisconnected - - def buildProtocol(self, addr): - """ - Create a L{_ReconnectingProtocolProxy} with the disconnect-notification - callback we were called with. - - @param addr: The address the connection is coming from. - - @return: a L{_ReconnectingProtocolProxy} for a protocol produced by - C{self._protocolFactory} - """ - return _ReconnectingProtocolProxy( - self._protocolFactory.buildProtocol(addr), self._protocolDisconnected - ) - - def __getattr__(self, item): - return getattr(self._protocolFactory, item) - - def __repr__(self) -> str: - return "<{} wrapping {!r}>".format( - self.__class__.__name__, self._protocolFactory - ) - - -def backoffPolicy( - initialDelay=1.0, maxDelay=60.0, factor=1.5, jitter=_goodEnoughRandom -): - """ - A timeout policy for L{ClientService} which computes an exponential backoff - interval with configurable parameters. - - @since: 16.1.0 - - @param initialDelay: Delay for the first reconnection attempt (default - 1.0s). - @type initialDelay: L{float} - - @param maxDelay: Maximum number of seconds between connection attempts - (default 60 seconds, or one minute). Note that this value is before - jitter is applied, so the actual maximum possible delay is this value - plus the maximum possible result of C{jitter()}. - @type maxDelay: L{float} - - @param factor: A multiplicative factor by which the delay grows on each - failed reattempt. Default: 1.5. - @type factor: L{float} - - @param jitter: A 0-argument callable that introduces noise into the delay. - By default, C{random.random}, i.e. a pseudorandom floating-point value - between zero and one. - @type jitter: 0-argument callable returning L{float} - - @return: a 1-argument callable that, given an attempt count, returns a - floating point number; the number of seconds to delay. - @rtype: see L{ClientService.__init__}'s C{retryPolicy} argument. - """ - - def policy(attempt): - try: - delay = min(initialDelay * (factor ** min(100, attempt)), maxDelay) - except OverflowError: - delay = maxDelay - return delay + jitter() - - return policy - - -_defaultPolicy = backoffPolicy() - - -def _firstResult(gen): - """ - Return the first element of a generator and exhaust it. - - C{MethodicalMachine.upon}'s C{collector} argument takes a generator of - output results. If the generator is exhausted, the later outputs aren't - actually run. - - @param gen: Generator to extract values from - - @return: The first element of the generator. - """ - return list(gen)[0] - - -class _ClientMachine: - """ - State machine for maintaining a single outgoing connection to an endpoint. - - @ivar _awaitingConnected: notifications to make when connection - succeeds, fails, or is cancelled - @type _awaitingConnected: list of (Deferred, count) tuples - - @see: L{ClientService} - """ - - _machine = MethodicalMachine() - - def __init__(self, endpoint, factory, retryPolicy, clock, prepareConnection, log): - """ - @see: L{ClientService.__init__} - - @param log: The logger for the L{ClientService} instance this state - machine is associated to. - @type log: L{Logger} - """ - self._endpoint = endpoint - self._failedAttempts = 0 - self._stopped = False - self._factory = factory - self._timeoutForAttempt = retryPolicy - self._clock = clock - self._prepareConnection = prepareConnection - self._connectionInProgress = succeed(None) - - self._awaitingConnected = [] - - self._stopWaiters = [] - self._log = log - - @_machine.state(initial=True) - def _init(self): - """ - The service has not been started. - """ - - @_machine.state() - def _connecting(self): - """ - The service has started connecting. - """ - - @_machine.state() - def _waiting(self): - """ - The service is waiting for the reconnection period - before reconnecting. - """ - - @_machine.state() - def _connected(self): - """ - The service is connected. - """ - - @_machine.state() - def _disconnecting(self): - """ - The service is disconnecting after being asked to shutdown. - """ - - @_machine.state() - def _restarting(self): - """ - The service is disconnecting and has been asked to restart. - """ - - @_machine.state() - def _stopped(self): - """ - The service has been stopped and is disconnected. - """ - - @_machine.input() - def start(self): - """ - Start this L{ClientService}, initiating the connection retry loop. - """ - - @_machine.output() - def _connect(self): - """ - Start a connection attempt. - """ - factoryProxy = _DisconnectFactory( - self._factory, lambda _: self._clientDisconnected() - ) - - self._connectionInProgress = ( - self._endpoint.connect(factoryProxy) - .addCallback(self._runPrepareConnection) - .addCallback(self._connectionMade) - .addErrback(self._connectionFailed) - ) - - def _runPrepareConnection(self, protocol): - """ - Run any C{prepareConnection} callback with the connected protocol, - ignoring its return value but propagating any failure. - - @param protocol: The protocol of the connection. - @type protocol: L{IProtocol} - - @return: Either: - - - A L{Deferred} that succeeds with the protocol when the - C{prepareConnection} callback has executed successfully. - - - A L{Deferred} that fails when the C{prepareConnection} callback - throws or returns a failed L{Deferred}. - - - The protocol, when no C{prepareConnection} callback is defined. - """ - if self._prepareConnection: - return maybeDeferred(self._prepareConnection, protocol).addCallback( - lambda _: protocol - ) - return protocol - - @_machine.output() - def _resetFailedAttempts(self): - """ - Reset the number of failed attempts. - """ - self._failedAttempts = 0 - - @_machine.input() - def stop(self): - """ - Stop trying to connect and disconnect any current connection. - - @return: a L{Deferred} that fires when all outstanding connections are - closed and all in-progress connection attempts halted. - """ - - @_machine.output() - def _waitForStop(self): - """ - Return a deferred that will fire when the service has finished - disconnecting. - - @return: L{Deferred} that fires when the service has finished - disconnecting. - """ - self._stopWaiters.append(Deferred()) - return self._stopWaiters[-1] - - @_machine.output() - def _stopConnecting(self): - """ - Stop pending connection attempt. - """ - self._connectionInProgress.cancel() - - @_machine.output() - def _stopRetrying(self): - """ - Stop pending attempt to reconnect. - """ - self._retryCall.cancel() - del self._retryCall - - @_machine.output() - def _disconnect(self): - """ - Disconnect the current connection. - """ - self._currentConnection.transport.loseConnection() - - @_machine.input() - def _connectionMade(self, protocol): - """ - A connection has been made. - - @param protocol: The protocol of the connection. - @type protocol: L{IProtocol} - """ - - @_machine.output() - def _notifyWaiters(self, protocol): - """ - Notify all pending requests for a connection that a connection has been - made. - - @param protocol: The protocol of the connection. - @type protocol: L{IProtocol} - """ - # This should be in _resetFailedAttempts but the signature doesn't - # match. - self._failedAttempts = 0 - - self._currentConnection = protocol._protocol - self._unawait(self._currentConnection) - - @_machine.input() - def _connectionFailed(self, f): - """ - The current connection attempt failed. - """ - - @_machine.output() - def _wait(self): - """ - Schedule a retry attempt. - """ - self._doWait() - - @_machine.output() - def _ignoreAndWait(self, f): - """ - Schedule a retry attempt, and ignore the Failure passed in. - """ - return self._doWait() - - def _doWait(self): - self._failedAttempts += 1 - delay = self._timeoutForAttempt(self._failedAttempts) - self._log.info( - "Scheduling retry {attempt} to connect {endpoint} " "in {delay} seconds.", - attempt=self._failedAttempts, - endpoint=self._endpoint, - delay=delay, - ) - self._retryCall = self._clock.callLater(delay, self._reconnect) - - @_machine.input() - def _reconnect(self): - """ - The wait between connection attempts is done. - """ - - @_machine.input() - def _clientDisconnected(self): - """ - The current connection has been disconnected. - """ - - @_machine.output() - def _forgetConnection(self): - """ - Forget the current connection. - """ - del self._currentConnection - - @_machine.output() - def _cancelConnectWaiters(self): - """ - Notify all pending requests for a connection that no more connections - are expected. - """ - self._unawait(Failure(CancelledError())) - - @_machine.output() - def _ignoreAndCancelConnectWaiters(self, f): - """ - Notify all pending requests for a connection that no more connections - are expected, after ignoring the Failure passed in. - """ - self._unawait(Failure(CancelledError())) - - @_machine.output() - def _finishStopping(self): - """ - Notify all deferreds waiting on the service stopping. - """ - self._doFinishStopping() - - @_machine.output() - def _ignoreAndFinishStopping(self, f): - """ - Notify all deferreds waiting on the service stopping, and ignore the - Failure passed in. - """ - self._doFinishStopping() - - def _doFinishStopping(self): - self._stopWaiters, waiting = [], self._stopWaiters - for w in waiting: - w.callback(None) - - @_machine.input() - def whenConnected(self, failAfterFailures=None): - """ - Retrieve the currently-connected L{Protocol}, or the next one to - connect. - - @param failAfterFailures: number of connection failures after which - the Deferred will deliver a Failure (None means the Deferred will - only fail if/when the service is stopped). Set this to 1 to make - the very first connection failure signal an error. Use 2 to - allow one failure but signal an error if the subsequent retry - then fails. - @type failAfterFailures: L{int} or None - - @return: a Deferred that fires with a protocol produced by the - factory passed to C{__init__} - @rtype: L{Deferred} that may: - - - fire with L{IProtocol} - - - fail with L{CancelledError} when the service is stopped - - - fail with e.g. - L{DNSLookupError<twisted.internet.error.DNSLookupError>} or - L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>} - when the number of consecutive failed connection attempts - equals the value of "failAfterFailures" - """ - - @_machine.output() - def _currentConnection(self, failAfterFailures=None): - """ - Return the currently connected protocol. - - @return: L{Deferred} that is fired with currently connected protocol. - """ - return succeed(self._currentConnection) - - @_machine.output() - def _noConnection(self, failAfterFailures=None): - """ - Notify the caller that no connection is expected. - - @return: L{Deferred} that is fired with L{CancelledError}. - """ - return fail(CancelledError()) - - @_machine.output() - def _awaitingConnection(self, failAfterFailures=None): - """ - Return a deferred that will fire with the next connected protocol. - - @return: L{Deferred} that will fire with the next connected protocol. - """ - result = Deferred() - self._awaitingConnected.append((result, failAfterFailures)) - return result - - @_machine.output() - def _deferredSucceededWithNone(self): - """ - Return a deferred that has already fired with L{None}. - - @return: A L{Deferred} that has already fired with L{None}. - """ - return succeed(None) - - def _unawait(self, value): - """ - Fire all outstanding L{ClientService.whenConnected} L{Deferred}s. - - @param value: the value to fire the L{Deferred}s with. - """ - self._awaitingConnected, waiting = [], self._awaitingConnected - for w, remaining in waiting: - w.callback(value) - - @_machine.output() - def _deliverConnectionFailure(self, f): - """ - Deliver connection failures to any L{ClientService.whenConnected} - L{Deferred}s that have met their failAfterFailures threshold. - - @param f: the Failure to fire the L{Deferred}s with. - """ - ready = [] - notReady = [] - for w, remaining in self._awaitingConnected: - if remaining is None: - notReady.append((w, remaining)) - elif remaining <= 1: - ready.append(w) - else: - notReady.append((w, remaining - 1)) - self._awaitingConnected = notReady - for w in ready: - w.callback(f) - - # State Transitions - - _init.upon(start, enter=_connecting, outputs=[_connect]) - _init.upon( - stop, - enter=_stopped, - outputs=[_deferredSucceededWithNone], - collector=_firstResult, - ) - - _connecting.upon(start, enter=_connecting, outputs=[]) - # Note that this synchonously triggers _connectionFailed in the - # _disconnecting state. - _connecting.upon( - stop, - enter=_disconnecting, - outputs=[_waitForStop, _stopConnecting], - collector=_firstResult, - ) - _connecting.upon(_connectionMade, enter=_connected, outputs=[_notifyWaiters]) - _connecting.upon( - _connectionFailed, - enter=_waiting, - outputs=[_ignoreAndWait, _deliverConnectionFailure], - ) - - _waiting.upon(start, enter=_waiting, outputs=[]) - _waiting.upon( - stop, - enter=_stopped, - outputs=[_waitForStop, _cancelConnectWaiters, _stopRetrying, _finishStopping], - collector=_firstResult, - ) - _waiting.upon(_reconnect, enter=_connecting, outputs=[_connect]) - - _connected.upon(start, enter=_connected, outputs=[]) - _connected.upon( - stop, - enter=_disconnecting, - outputs=[_waitForStop, _disconnect], - collector=_firstResult, - ) - _connected.upon( - _clientDisconnected, enter=_waiting, outputs=[_forgetConnection, _wait] - ) - - _disconnecting.upon(start, enter=_restarting, outputs=[_resetFailedAttempts]) - _disconnecting.upon( - stop, enter=_disconnecting, outputs=[_waitForStop], collector=_firstResult - ) - _disconnecting.upon( - _clientDisconnected, - enter=_stopped, - outputs=[_cancelConnectWaiters, _finishStopping, _forgetConnection], - ) - # Note that this is triggered synchonously with the transition from - # _connecting - _disconnecting.upon( - _connectionFailed, - enter=_stopped, - outputs=[_ignoreAndCancelConnectWaiters, _ignoreAndFinishStopping], - ) - - _restarting.upon(start, enter=_restarting, outputs=[]) - _restarting.upon( - stop, enter=_disconnecting, outputs=[_waitForStop], collector=_firstResult - ) - _restarting.upon( - _clientDisconnected, enter=_connecting, outputs=[_finishStopping, _connect] - ) - - _stopped.upon(start, enter=_connecting, outputs=[_connect]) - _stopped.upon( - stop, - enter=_stopped, - outputs=[_deferredSucceededWithNone], - collector=_firstResult, - ) - - _init.upon( - whenConnected, - enter=_init, - outputs=[_awaitingConnection], - collector=_firstResult, - ) - _connecting.upon( - whenConnected, - enter=_connecting, - outputs=[_awaitingConnection], - collector=_firstResult, - ) - _waiting.upon( - whenConnected, - enter=_waiting, - outputs=[_awaitingConnection], - collector=_firstResult, - ) - _connected.upon( - whenConnected, - enter=_connected, - outputs=[_currentConnection], - collector=_firstResult, - ) - _disconnecting.upon( - whenConnected, - enter=_disconnecting, - outputs=[_awaitingConnection], - collector=_firstResult, - ) - _restarting.upon( - whenConnected, - enter=_restarting, - outputs=[_awaitingConnection], - collector=_firstResult, - ) - _stopped.upon( - whenConnected, enter=_stopped, outputs=[_noConnection], collector=_firstResult - ) - - -class ClientService(service.Service): - """ - A L{ClientService} maintains a single outgoing connection to a client - endpoint, reconnecting after a configurable timeout when a connection - fails, either before or after connecting. - - @since: 16.1.0 - """ - - _log = Logger() - - def __init__( - self, endpoint, factory, retryPolicy=None, clock=None, prepareConnection=None - ): - """ - @param endpoint: A L{stream client endpoint - <interfaces.IStreamClientEndpoint>} provider which will be used to - connect when the service starts. - - @param factory: A L{protocol factory <interfaces.IProtocolFactory>} - which will be used to create clients for the endpoint. - - @param retryPolicy: A policy configuring how long L{ClientService} will - wait between attempts to connect to C{endpoint}. - @type retryPolicy: callable taking (the number of failed connection - attempts made in a row (L{int})) and returning the number of - seconds to wait before making another attempt. - - @param clock: The clock used to schedule reconnection. It's mainly - useful to be parametrized in tests. If the factory is serialized, - this attribute will not be serialized, and the default value (the - reactor) will be restored when deserialized. - @type clock: L{IReactorTime} - - @param prepareConnection: A single argument L{callable} that may return - a L{Deferred}. It will be called once with the L{protocol - <interfaces.IProtocol>} each time a new connection is made. It may - call methods on the protocol to prepare it for use (e.g. - authenticate) or validate it (check its health). - - The C{prepareConnection} callable may raise an exception or return - a L{Deferred} which fails to reject the connection. A rejected - connection is not used to fire an L{Deferred} returned by - L{whenConnected}. Instead, L{ClientService} handles the failure - and continues as if the connection attempt were a failure - (incrementing the counter passed to C{retryPolicy}). - - L{Deferred}s returned by L{whenConnected} will not fire until - any L{Deferred} returned by the C{prepareConnection} callable - fire. Otherwise its successful return value is consumed, but - ignored. - - Present Since Twisted 18.7.0 - - @type prepareConnection: L{callable} - - """ - clock = _maybeGlobalReactor(clock) - retryPolicy = _defaultPolicy if retryPolicy is None else retryPolicy - - self._machine = _ClientMachine( - endpoint, - factory, - retryPolicy, - clock, - prepareConnection=prepareConnection, - log=self._log, - ) - - def whenConnected(self, failAfterFailures=None): - """ - Retrieve the currently-connected L{Protocol}, or the next one to - connect. - - @param failAfterFailures: number of connection failures after which - the Deferred will deliver a Failure (None means the Deferred will - only fail if/when the service is stopped). Set this to 1 to make - the very first connection failure signal an error. Use 2 to - allow one failure but signal an error if the subsequent retry - then fails. - @type failAfterFailures: L{int} or None - - @return: a Deferred that fires with a protocol produced by the - factory passed to C{__init__} - @rtype: L{Deferred} that may: - - - fire with L{IProtocol} - - - fail with L{CancelledError} when the service is stopped - - - fail with e.g. - L{DNSLookupError<twisted.internet.error.DNSLookupError>} or - L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>} - when the number of consecutive failed connection attempts - equals the value of "failAfterFailures" - """ - return self._machine.whenConnected(failAfterFailures) - - def startService(self): - """ - Start this L{ClientService}, initiating the connection retry loop. - """ - if self.running: - self._log.warn("Duplicate ClientService.startService {log_source}") - return - super().startService() - self._machine.start() - - def stopService(self): - """ - Stop attempting to reconnect and close any existing connections. - - @return: a L{Deferred} that fires when all outstanding connections are - closed and all in-progress connection attempts halted. - """ - super().stopService() - return self._machine.stop() - - __all__ = [ "TimerService", "CooperatorService", @@ -1202,4 +422,6 @@ __all__ = [ "SSLClient", "UNIXDatagramServer", "UNIXDatagramClient", + "ClientService", + "backoffPolicy", ] diff --git a/contrib/python/Twisted/py3/twisted/conch/manhole.py b/contrib/python/Twisted/py3/twisted/conch/manhole.py index f552af5bbdc..670ac0480ec 100644 --- a/contrib/python/Twisted/py3/twisted/conch/manhole.py +++ b/contrib/python/Twisted/py3/twisted/conch/manhole.py @@ -124,7 +124,14 @@ class ManholeInterpreter(code.InteractiveInterpreter): """ Format exception tracebacks and write them to the output handler. """ - lines = format_exception(excType, excValue, excTraceback.tb_next) + code_obj = excTraceback.tb_frame.f_code + if code_obj.co_filename == code.__file__ and code_obj.co_name == "runcode": + traceback = excTraceback.tb_next + else: + # Workaround for https://github.com/python/cpython/issues/122478, + # present e.g. in Python 3.12.6: + traceback = excTraceback + lines = format_exception(excType, excValue, traceback) self.write("".join(lines)) def displayhook(self, obj): diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py b/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py index e959f022a0b..7d2f1072f47 100644 --- a/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py +++ b/contrib/python/Twisted/py3/twisted/conch/ssh/keys.py @@ -256,24 +256,38 @@ class Key: if keyType == b"ssh-rsa": e, n, rest = common.getMP(rest, 2) return cls(rsa.RSAPublicNumbers(e, n).public_key(default_backend())) - elif keyType == b"ssh-dss": + + if keyType == b"ssh-dss": p, q, g, y, rest = common.getMP(rest, 4) return cls( dsa.DSAPublicNumbers( y=y, parameter_numbers=dsa.DSAParameterNumbers(p=p, q=q, g=g) ).public_key(default_backend()) ) - elif keyType in _curveTable: + + if keyType in _curveTable: return cls( ec.EllipticCurvePublicKey.from_encoded_point( _curveTable[keyType], common.getNS(rest, 2)[1] ) ) - elif keyType == b"ssh-ed25519": + + if keyType == b"[email protected]": + keyObject = cls._fromECEncodedPoint( + encodedPoint=common.getNS(rest, 2)[1], + curve=b"ecdsa-sha2-nistp256", + ) + keyObject._sk = True + return keyObject + + if keyType in [b"ssh-ed25519", b"[email protected]"]: a, rest = common.getNS(rest) - return cls._fromEd25519Components(a) - else: - raise BadKeyError(f"unknown blob type: {keyType}") + keyObject = cls._fromEd25519Components(a) + if keyType.startswith(b"sk-ssh-"): + keyObject._sk = True + return keyObject + + raise BadKeyError(f"unknown blob type: {keyType}") @classmethod def _fromString_PRIVATE_BLOB(cls, blob): @@ -676,16 +690,37 @@ class Key: """ if data.startswith(b"ssh-") or data.startswith(b"ecdsa-sha2-"): return "public_openssh" - elif data.startswith(b"-----BEGIN"): + + # Twisted doesn't support certificate based keys yet. + # https://github.com/openssh/openssh-portable/blob/05f2b141cfcc60c7cdedf9450d2b9d390c19eaad/PROTOCOL.u2f#L96C1-L97C31 + if data.startswith(b"sk-ecdsa-sha2-nistp256-cert-v01") or data.startswith( + b"sk-ssh-ed25519-cert-v01" + ): + raise BadKeyError("certificate based keys are not supported") + + if data.startswith(b"sk-ecdsa-sha2-nistp256") or data.startswith( + b"sk-ssh-ed25519" + ): + # OpenSSH FIDO2 security keys have similar public format. + # They have the extra "application" string, + # which for now is ignored. + return "public_openssh" + + if data.startswith(b"-----BEGIN"): return "private_openssh" - elif data.startswith(b"{"): + + if data.startswith(b"{"): return "public_lsh" - elif data.startswith(b"("): + + if data.startswith(b"("): return "private_lsh" - elif ( + + if ( data.startswith(b"\x00\x00\x00\x07ssh-") or data.startswith(b"\x00\x00\x00\x13ecdsa-") or data.startswith(b"\x00\x00\x00\x0bssh-ed25519") + or data.startswith(b'\x00\x00\x00"[email protected]') + or data.startswith(b"\x00\x00\x00\[email protected]") ): ignored, rest = common.getNS(data) count = 0 @@ -869,6 +904,7 @@ class Key: @type keyObject: C{cryptography.hazmat.primitives.asymmetric} key. """ self._keyObject = keyObject + self._sk = False def __eq__(self, other: object) -> bool: """ @@ -1029,16 +1065,25 @@ class Key: @return: The key type format. @rtype: L{bytes} """ + if self._sk: + if self.type() == "EC": + return b"[email protected]" + # FIXME: https://github.com/twisted/twisted/issues/12304 + # We only support 2 key types, + # so if the key was loaded with success and it's + # not ECDSA, it must be an ED25519 key. + return b"[email protected]" + if self.type() == "EC": return ( b"ecdsa-sha2-" + _secToNist[self._keyObject.curve.name.encode("ascii")] ) - else: - return { - "RSA": b"ssh-rsa", - "DSA": b"ssh-dss", - "Ed25519": b"ssh-ed25519", - }[self.type()] + + return { + "RSA": b"ssh-rsa", + "DSA": b"ssh-dss", + "Ed25519": b"ssh-ed25519", + }[self.type()] def supportedSignatureAlgorithms(self): """ @@ -1070,14 +1115,16 @@ class Key: return hashes.SHA512() else: return None - else: - return { - ("RSA", b"ssh-rsa"): hashes.SHA1(), - ("RSA", b"rsa-sha2-256"): hashes.SHA256(), - ("RSA", b"rsa-sha2-512"): hashes.SHA512(), - ("DSA", b"ssh-dss"): hashes.SHA1(), - ("Ed25519", b"ssh-ed25519"): hashes.SHA512(), - }.get((self.type(), signatureType)) + + if self.type() == "Ed25519": + return hashes.SHA512() + + return { + ("RSA", b"ssh-rsa"): hashes.SHA1(), + ("RSA", b"rsa-sha2-256"): hashes.SHA256(), + ("RSA", b"rsa-sha2-512"): hashes.SHA512(), + ("DSA", b"ssh-dss"): hashes.SHA1(), + }.get((self.type(), signatureType)) def size(self): """ diff --git a/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py b/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py index d46f093dff9..545c010f76e 100644 --- a/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py +++ b/contrib/python/Twisted/py3/twisted/conch/ssh/transport.py @@ -103,17 +103,13 @@ class SSHCiphers: cipherMap = { b"3des-cbc": (algorithms.TripleDES, 24, modes.CBC), - b"blowfish-cbc": (algorithms.Blowfish, 16, modes.CBC), b"aes256-cbc": (algorithms.AES, 32, modes.CBC), b"aes192-cbc": (algorithms.AES, 24, modes.CBC), b"aes128-cbc": (algorithms.AES, 16, modes.CBC), - b"cast128-cbc": (algorithms.CAST5, 16, modes.CBC), b"aes128-ctr": (algorithms.AES, 16, modes.CTR), b"aes192-ctr": (algorithms.AES, 24, modes.CTR), b"aes256-ctr": (algorithms.AES, 32, modes.CTR), b"3des-ctr": (algorithms.TripleDES, 24, modes.CTR), - b"blowfish-ctr": (algorithms.Blowfish, 16, modes.CTR), - b"cast128-ctr": (algorithms.CAST5, 16, modes.CTR), b"none": (None, 0, modes.CBC), } macMap = { @@ -295,10 +291,6 @@ def _getSupportedCiphers(): b"aes192-cbc", b"aes128-ctr", b"aes128-cbc", - b"cast128-ctr", - b"cast128-cbc", - b"blowfish-ctr", - b"blowfish-cbc", b"3des-ctr", b"3des-cbc", ] diff --git a/contrib/python/Twisted/py3/twisted/internet/address.py b/contrib/python/Twisted/py3/twisted/internet/address.py index 10fa85241e5..d0ab1f69290 100644 --- a/contrib/python/Twisted/py3/twisted/internet/address.py +++ b/contrib/python/Twisted/py3/twisted/internet/address.py @@ -21,7 +21,7 @@ from twisted.python.runtime import platform @implementer(IAddress) [email protected](hash=True, auto_attribs=True) [email protected](unsafe_hash=True, auto_attribs=True) class IPv4Address: """ An L{IPv4Address} represents the address of an IPv4 socket endpoint. @@ -45,7 +45,7 @@ class IPv4Address: @implementer(IAddress) [email protected](hash=True, auto_attribs=True) [email protected](unsafe_hash=True, auto_attribs=True) class IPv6Address: """ An L{IPv6Address} represents the address of an IPv6 socket endpoint. @@ -85,7 +85,7 @@ class _ProcessAddress: """ [email protected](hash=True, auto_attribs=True) [email protected](unsafe_hash=True, auto_attribs=True) @implementer(IAddress) class HostnameAddress: """ @@ -102,7 +102,7 @@ class HostnameAddress: port: int [email protected](hash=False, repr=False, eq=False, auto_attribs=True) [email protected](unsafe_hash=False, repr=False, eq=False, auto_attribs=True) @implementer(IAddress) class UNIXAddress: """ diff --git a/contrib/python/Twisted/py3/twisted/internet/base.py b/contrib/python/Twisted/py3/twisted/internet/base.py index c807f418731..2c38b80b0ca 100644 --- a/contrib/python/Twisted/py3/twisted/internet/base.py +++ b/contrib/python/Twisted/py3/twisted/internet/base.py @@ -213,29 +213,23 @@ class DelayedCall: """ return not (self.cancelled or self.called) - def __le__(self, other: object) -> bool: + def __le__(self, other: "DelayedCall") -> bool: """ Implement C{<=} operator between two L{DelayedCall} instances. Comparison is based on the C{time} attribute (unadjusted by the delayed time). """ - if isinstance(other, DelayedCall): - return self.time <= other.time - else: - return NotImplemented + return self.time <= other.time - def __lt__(self, other: object) -> bool: + def __lt__(self, other: "DelayedCall") -> bool: """ Implement C{<} operator between two L{DelayedCall} instances. Comparison is based on the C{time} attribute (unadjusted by the delayed time). """ - if isinstance(other, DelayedCall): - return self.time < other.time - else: - return NotImplemented + return self.time < other.time def __repr__(self) -> str: """ @@ -577,6 +571,8 @@ class PluggableResolverMixin: _SystemEventID = NewType("_SystemEventID", Tuple[str, _ThreePhaseEventTriggerHandle]) _ThreadCall = Tuple[Callable[..., Any], Tuple[object, ...], Dict[str, object]] +_DEFAULT_DELAYED_CALL_LOGGING_HANDLER = _log.failureHandler("while handling timed call") + @implementer(IReactorCore, IReactorTime, _ISupportsExitSignalCapturing) class ReactorBase(PluggableResolverMixin): @@ -964,7 +960,6 @@ class ReactorBase(PluggableResolverMixin): """ See twisted.internet.interfaces.IReactorTime.callLater. """ - assert builtins.callable(callable), f"{callable} is not callable" assert delay >= 0, f"{delay} is not greater than or equal to 0 seconds" delayedCall = DelayedCall( self.seconds() + delay, @@ -1012,6 +1007,12 @@ class ReactorBase(PluggableResolverMixin): ] def _insertNewDelayedCalls(self) -> None: + # This function is called twice per reactor iteration, once in + # timeout() and once in runUntilCurrent(), and in most cases there + # won't be any new timeouts. So have a fast path for the empty case. + if not self._newTimedCalls: + return + for call in self._newTimedCalls: if call.cancelled: self._cancellations -= 1 @@ -1083,18 +1084,23 @@ class ReactorBase(PluggableResolverMixin): heappush(self._pendingTimedCalls, call) continue - with _log.failuresHandled( - "while handling timed call {previous()}", - previous=lambda creator=call.creator: ( - "" - if creator is None - else "\n" - + (" C: from a DelayedCall created here:\n") - + " C:" - + "".join(creator).rstrip().replace("\n", "\n C:") - + "\n" - ), - ): + logHandler = ( + _log.failuresHandled( + "while handling timed call {previous()}", + previous=lambda creator=call.creator: ( + "\n" + + (" C: from a DelayedCall created here:\n") + + " C:" + + "".join(creator).rstrip().replace("\n", "\n C:") + + "\n" + ), + ) + if call.creator + # A much faster logging handler for the common case where extra + # debug info is not being output: + else _DEFAULT_DELAYED_CALL_LOGGING_HANDLER + ) + with logHandler: call.called = 1 call.func(*call.args, **call.kw) diff --git a/contrib/python/Twisted/py3/twisted/internet/defer.py b/contrib/python/Twisted/py3/twisted/internet/defer.py index 1c58baea7cd..951ca87d6b5 100644 --- a/contrib/python/Twisted/py3/twisted/internet/defer.py +++ b/contrib/python/Twisted/py3/twisted/internet/defer.py @@ -117,7 +117,11 @@ def succeed(result: _T) -> "Deferred[_T]": method. """ d: Deferred[_T] = Deferred() - d.callback(result) + # This violate abstraction boundaries, so code that is not internal to + # Twisted shouldn't do it, but it's a significant performance optimization: + d.result = result + d.called = True + d._chainedTo = None return d @@ -213,8 +217,8 @@ def maybeDeferred( except BaseException: return fail(Failure(captureVars=Deferred.debug)) - if isinstance(result, Deferred): - return result + if type(result) in _DEFERRED_SUBCLASSES: + return result # type: ignore[return-value] elif isinstance(result, Failure): return fail(result) elif type(result) is CoroutineType: @@ -522,6 +526,8 @@ class Deferred(Awaitable[_SelfResultT]): if errbackKeywords is None: errbackKeywords = {} # type: ignore[unreachable] + # Note that this logic is duplicated in addCallbac/addErrback/addBoth + # for performance reasons. self.callbacks.append( ( (callback, callbackArgs, callbackKeywords), @@ -614,10 +620,14 @@ class Deferred(Awaitable[_SelfResultT]): See L{addCallbacks}. """ - # Implementation Note: Any annotations for brevity; the overloads above - # handle specifying the actual signature, and there's nothing worth - # type-checking in this implementation. - return self.addCallbacks(callback, callbackArgs=args, callbackKeywords=kwargs) + # This could be implemented as a call to addCallbacks, but doing it + # directly is faster. + self.callbacks.append(((callback, args, kwargs), (_failthru, (), {}))) + + if self.called: + self._runCallbacks() + + return self @overload def addErrback( @@ -652,10 +662,14 @@ class Deferred(Awaitable[_SelfResultT]): See L{addCallbacks}. """ - # See implementation note in addCallbacks about Any arguments - return self.addCallbacks( - passthru, errback, errbackArgs=args, errbackKeywords=kwargs - ) + # This could be implemented as a call to addCallbacks, but doing it + # directly is faster. + self.callbacks.append(((passthru, (), {}), (errback, args, kwargs))) + + if self.called: + self._runCallbacks() + + return self @overload def addBoth( @@ -737,15 +751,15 @@ class Deferred(Awaitable[_SelfResultT]): See L{addCallbacks}. """ - # See implementation note in addCallbacks about Any arguments - return self.addCallbacks( - callback, - callback, - callbackArgs=args, - errbackArgs=args, - callbackKeywords=kwargs, - errbackKeywords=kwargs, - ) + # This could be implemented as a call to addCallbacks, but doing it + # directly is faster. + call = (callback, args, kwargs) + self.callbacks.append((call, call)) + + if self.called: + self._runCallbacks() + + return self # END way too many overloads @@ -915,13 +929,13 @@ class Deferred(Awaitable[_SelfResultT]): """ Stop processing on a L{Deferred} until L{unpause}() is called. """ - self.paused = self.paused + 1 + self.paused += 1 def unpause(self) -> None: """ Process all callbacks made since L{pause}() was called. """ - self.paused = self.paused - 1 + self.paused -= 1 if self.paused: return if self.called: @@ -983,10 +997,8 @@ class Deferred(Awaitable[_SelfResultT]): """ Build a tuple of callback and errback with L{_Sentinel._CONTINUE}. """ - return ( - (_Sentinel._CONTINUE, (self,), _NONE_KWARGS), - (_Sentinel._CONTINUE, (self,), _NONE_KWARGS), - ) + triple = (_CONTINUE, (self,), _NONE_KWARGS) + return (triple, triple) # type: ignore[return-value] def _runCallbacks(self) -> None: """ @@ -1049,7 +1061,9 @@ class Deferred(Awaitable[_SelfResultT]): if callback is _CONTINUE: # Give the waiting Deferred our current result and then # forget about that result ourselves. - chainee = cast(Deferred[object], args[0]) + + # We don't use cast() for performance reasons: + chainee: Deferred[object] = args[0] # type: ignore[assignment] chainee.result = current.result current.result = None # Making sure to update _debugInfo @@ -1127,7 +1141,6 @@ class Deferred(Awaitable[_SelfResultT]): if isinstance(current.result, Failure): # Stash the Failure in the _debugInfo for unhandled error # reporting. - current.result.cleanFailure() if current._debugInfo is None: current._debugInfo = DebugInfo() current._debugInfo.failResult = current.result @@ -1158,38 +1171,28 @@ class Deferred(Awaitable[_SelfResultT]): __repr__ = __str__ - def __iter__(self) -> "Deferred[_SelfResultT]": - return self - - @_extraneous - def send(self, value: object = None) -> "Deferred[_SelfResultT]": - if self.paused: - # If we're paused, we have no result to give - return self + def __iter__(self) -> Generator[Deferred[_SelfResultT], None, _SelfResultT]: + while True: + if self.paused: + # If we're paused, we have no result to give + yield self + continue - result = getattr(self, "result", _NO_RESULT) - if result is _NO_RESULT: - return self - if isinstance(result, Failure): - # Clear the failure on debugInfo so it doesn't raise "unhandled - # exception" - assert self._debugInfo is not None - self._debugInfo.failResult = None - result.value.__failure__ = result - raise result.value - else: - raise StopIteration(result) + result = getattr(self, "result", _NO_RESULT) + if result is _NO_RESULT: + yield self + continue - # For PEP-492 support (async/await) - # type note: base class "Awaitable" defined the type as: - # Callable[[], Generator[Any, None, _SelfResultT]] - # See: https://github.com/python/typeshed/issues/5125 - # When the typeshed patch is included in a mypy release, - # this method can be replaced by `__await__ = __iter__`. - def __await__(self) -> Generator[Any, None, _SelfResultT]: - return self.__iter__() # type: ignore[return-value] + if isinstance(result, Failure): + # Clear the failure on debugInfo so it doesn't raise "unhandled + # exception" + assert self._debugInfo is not None + self._debugInfo.failResult = None + result.raiseException() + else: + return result # type: ignore[return-value] - __next__ = send + __await__ = __iter__ def asFuture(self, loop: AbstractEventLoop) -> "Future[_SelfResultT]": """ @@ -1351,11 +1354,11 @@ def ensureDeferred( @param coro: The coroutine object to schedule, or a L{Deferred}. """ - if isinstance(coro, Deferred): - return coro + if type(coro) in _DEFERRED_SUBCLASSES: + return coro # type: ignore[return-value] else: try: - return Deferred.fromCoroutine(coro) + return Deferred.fromCoroutine(coro) # type: ignore[arg-type] except NotACoroutineError: # It's not a coroutine. Raise an exception, but say that it's also # not a Deferred so the error makes sense. @@ -2112,17 +2115,23 @@ def _inlineCallbacks( status.deferred.callback(callbackValue) return - if iscoroutine(result) or inspect.isgenerator(result): + isDeferred = type(result) in _DEFERRED_SUBCLASSES + # iscoroutine() is pretty expensive in this context, so avoid calling + # it unnecessarily: + if not isDeferred and (iscoroutine(result) or inspect.isgenerator(result)): result = _cancellableInlineCallbacks(result) + isDeferred = True + + if isDeferred: + # We don't cast() to Deferred because that does more work in the hot path - if isinstance(result, Deferred): # a deferred was yielded, get the result. - result.addBoth(_gotResultInlineCallbacks, waiting, gen, status, context) + result.addBoth(_gotResultInlineCallbacks, waiting, gen, status, context) # type: ignore[attr-defined] if waiting[0]: # Haven't called back yet, set flag so that we get reinvoked # and return from the loop waiting[0] = False - status.waitingOn = result + status.waitingOn = result # type: ignore[assignment] return result = waiting[1] diff --git a/contrib/python/Twisted/py3/twisted/internet/endpoints.py b/contrib/python/Twisted/py3/twisted/internet/endpoints.py index 7ab1d817319..a98fd2ba43b 100644 --- a/contrib/python/Twisted/py3/twisted/internet/endpoints.py +++ b/contrib/python/Twisted/py3/twisted/internet/endpoints.py @@ -808,9 +808,18 @@ class HostnameEndpoint: seconds to wait before assuming the connection has failed. @type timeout: L{float} or L{int} - @param bindAddress: the local address of the network interface to make - the connections from. - @type bindAddress: L{bytes} + @param bindAddress: The client socket normally uses whatever + local interface (eth0, en0, lo, etc) is best suited for the + target address, and a randomly-assigned port. This argument + allows that local address/port to be overridden. Providing + just an address (as a str) will bind the client socket to + whichever interface is assigned that address. Providing a + tuple of (str, int) will bind it to both an interface and a + specific local port. To bind the port, but leave the + interface unbound, use a tuple of ("", port), or ("0.0.0.0", + port) for IPv4, or ("::0", port) for IPv6. To leave both + interface and port unbound, just use None. + @type bindAddress: L{str}, L{tuple}, or None @param attemptDelay: The number of seconds to delay between connection attempts. @@ -827,6 +836,11 @@ class HostnameEndpoint: self._hostStr = self._hostBytes if bytes is str else self._hostText self._port = port self._timeout = timeout + if bindAddress is not None: + if isinstance(bindAddress, (bytes, str)): + bindAddress = (bindAddress, 0) + if isinstance(bindAddress[0], bytes): + bindAddress = (bindAddress[0].decode(), bindAddress[1]) self._bindAddress = bindAddress if attemptDelay is None: attemptDelay = self._DEFAULT_ATTEMPT_DELAY @@ -2299,7 +2313,9 @@ def _parseClientTLS( ), clientFromString(reactor, endpoint) if endpoint is not None - else HostnameEndpoint(reactor, _idnaBytes(host), port, timeout, bindAddress), + else HostnameEndpoint( + reactor, _idnaBytes(host), port, timeout, (bindAddress, 0) + ), ) diff --git a/contrib/python/Twisted/py3/twisted/internet/tcp.py b/contrib/python/Twisted/py3/twisted/internet/tcp.py index 8f85025556b..018d1912d27 100644 --- a/contrib/python/Twisted/py3/twisted/internet/tcp.py +++ b/contrib/python/Twisted/py3/twisted/internet/tcp.py @@ -727,6 +727,7 @@ class _BaseTCPClient: whenDone = None if whenDone and bindAddress is not None: try: + assert type(bindAddress) == tuple if abstract.isIPv6Address(bindAddress[0]): bindinfo = _resolveIPv6(*bindAddress) else: diff --git a/contrib/python/Twisted/py3/twisted/internet/testing.py b/contrib/python/Twisted/py3/twisted/internet/testing.py index 2c372495844..6563184edf9 100644 --- a/contrib/python/Twisted/py3/twisted/internet/testing.py +++ b/contrib/python/Twisted/py3/twisted/internet/testing.py @@ -9,7 +9,18 @@ from __future__ import annotations from io import BytesIO from socket import AF_INET, AF_INET6 -from typing import Callable, Iterator, Sequence, overload +from time import time +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Iterator, + Sequence, + TypeVar, + Union, + overload, +) from zope.interface import implementedBy, implementer from zope.interface.verify import verifyClass @@ -19,7 +30,7 @@ from typing_extensions import ParamSpec, Self from twisted.internet import address, error, protocol, task from twisted.internet.abstract import _dataMustBeBytes, isIPv6Address from twisted.internet.address import IPv4Address, IPv6Address, UNIXAddress -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, ensureDeferred, succeed from twisted.internet.error import UnsupportedAddressFamily from twisted.internet.interfaces import ( IConnector, @@ -967,3 +978,95 @@ class EventLoggingObserver(Sequence[LogEvent]): publisher.addObserver(obs) testInstance.addCleanup(lambda: publisher.removeObserver(obs)) return obs + + +_T = TypeVar("_T") + + +def _benchmarkWithReactor( + test_target: Callable[ + [], + Union[ + Coroutine[Deferred[Any], Any, _T], + Generator[Deferred[Any], Any, _T], + Deferred[_T], + ], + ] +) -> Callable[[Any], None]: # pragma: no cover + """ + Decorator for running a benchmark tests that loops the reactor. + + This is designed to decorate test method executed using pytest and + pytest-benchmark. + """ + + def deferredWrapper(): + return ensureDeferred(test_target()) + + def benchmark_test(benchmark: Any) -> None: + # Spinning up and spinning down the reactor adds quite a lot of + # overhead to the benchmarked function. So, make sure that the overhead + # isn't making the benchmark meaningless before we bother with any real + # benchmarking. + start = time() + _runReactor(lambda: succeed(None)) + justReactorElapsed = time() - start + + start = time() + _runReactor(deferredWrapper) + benchmarkElapsed = time() - start + + if benchmarkElapsed / justReactorElapsed < 5: + raise RuntimeError( # pragma: no cover + "The function you are benchmarking is fast enough that its " + "run time is being swamped by the startup/shutdown of the " + "reactor. Consider adding a for loop to the benchmark " + "function so it does the work a number of times." + ) + + benchmark(_runReactor, deferredWrapper) + + return benchmark_test + + +def _runReactor(callback: Callable[[], Deferred[_T]]) -> None: # pragma: no cover + """ + (re)Start a reactor that might have been previously started. + """ + # Delay to import to prevent side-effect in normal tests that are + # expecting to import twisted.internet.testing while no reactor is + # installed. + from twisted.internet import reactor + + errors: list[failure.Failure] = [] + + deferred = callback() + deferred.addErrback(errors.append) + deferred.addBoth(lambda _: reactor.callLater(0, _stopReactor, reactor)) # type: ignore[attr-defined] + reactor.run(installSignalHandlers=False) # type: ignore[attr-defined] + + if errors: # pragma: no cover + # Make sure the test fails in a visible way: + errors[0].raiseException() + + +def _stopReactor(reactor): # pragma: no cover + """ + Stop the reactor and allow it to be re-started later. + """ + reactor.stop() + # Allow for on shutdown hooks to execute. + reactor.iterate() + # Since we're going to be poking the reactor's guts, let's make sure what + # we're doing is vaguely reasonable: + assert hasattr(reactor, "_startedBefore") + assert hasattr(reactor, "_started") + assert hasattr(reactor, "_justStopped") + assert hasattr(reactor, "running") + reactor._startedBefore = False + reactor._started = False + reactor._justStopped = False + reactor.running = False + # Start running has consumed the startup events, so we need + # to restore them. + reactor.addSystemEventTrigger("during", "startup", reactor._reallyStartRunning) diff --git a/contrib/python/Twisted/py3/twisted/logger/_format.py b/contrib/python/Twisted/py3/twisted/logger/_format.py index 59b44c7f723..57d8c3b1e60 100644 --- a/contrib/python/Twisted/py3/twisted/logger/_format.py +++ b/contrib/python/Twisted/py3/twisted/logger/_format.py @@ -199,7 +199,7 @@ class PotentialCallWrapper(object): self._wrapped = wrapped def __getattr__(self, name: str) -> object: - return keycall(name, self._wrapped.__getattribute__) + return keycall(name, lambda name_: getattr(self._wrapped, name_)) def __getitem__(self, name: str) -> object: # The sub-object may not be indexable, but if it isn't, that's the @@ -208,13 +208,13 @@ class PotentialCallWrapper(object): return PotentialCallWrapper(value) def __format__(self, format_spec: str) -> str: - return self._wrapped.__format__(format_spec) + return format(self._wrapped, format_spec) def __repr__(self) -> str: - return self._wrapped.__repr__() + return repr(self._wrapped) def __str__(self) -> str: - return self._wrapped.__str__() + return str(self._wrapped) class CallMapping(Mapping[str, Any]): diff --git a/contrib/python/Twisted/py3/twisted/python/failure.py b/contrib/python/Twisted/py3/twisted/python/failure.py index d253ffad743..987287e0c76 100644 --- a/contrib/python/Twisted/py3/twisted/python/failure.py +++ b/contrib/python/Twisted/py3/twisted/python/failure.py @@ -24,9 +24,10 @@ from inspect import getmro from io import StringIO from typing import Callable, NoReturn, TypeVar -import opcode +from incremental import Version from twisted.python import reflect +from twisted.python.deprecate import deprecatedProperty _T_Callable = TypeVar("_T_Callable", bound=Callable[..., object]) @@ -84,8 +85,7 @@ def format_frames(frames, write, detail="default"): w(f" {name} : {repr(val)}\n") -# slyphon: i have a need to check for this value in trial -# so I made it a module-level constant +# Unused, here for backwards compatibility. EXCEPTION_CAUGHT_HERE = "--- <exception caught here> ---" @@ -96,28 +96,20 @@ class NoCurrentExceptionError(Exception): """ -def _Traceback(stackFrames, tbFrames): +def _Traceback(tbFrames): """ Construct a fake traceback object using a list of frames. It should have the same API as stdlib to allow interaction with other tools. - @param stackFrames: [(methodname, filename, lineno, locals, globals), ...] @param tbFrames: [(methodname, filename, lineno, locals, globals), ...] """ assert len(tbFrames) > 0, "Must pass some frames" # We deliberately avoid using recursion here, as the frames list may be # long. - # 'stackFrames' is a list of frames above (ie, older than) the point the - # exception was caught, with oldest at the start. Start by building these - # into a linked list of _Frame objects (with the f_back links pointing back - # towards the oldest frame). stack = None - for sf in stackFrames: - stack = _Frame(sf, stack) - # 'tbFrames' is a list of frames from the point the exception was caught, # down to where it was thrown, with the oldest at the start. Add these to # the linked list of _Frames, but also wrap each one with a _Traceback @@ -237,30 +229,29 @@ class Failure(BaseException): This is necessary because Python's built-in error mechanisms are inconvenient for asynchronous communication. - The C{stack} and C{frame} attributes contain frames. Each frame is a tuple + The C{frame} attribute contain the traceback frames. Each frame is a tuple of (funcName, fileName, lineNumber, localsItems, globalsItems), where localsItems and globalsItems are the contents of C{locals().items()}/C{globals().items()} for that frame, or an empty tuple if those details were not captured. + Local/global variables in C{frame} will only be captured if + C{captureVars=True} when constructing the L{Failure}. + @ivar value: The exception instance responsible for this failure. + @ivar type: The exception's class. - @ivar stack: list of frames, innermost last, excluding C{Failure.__init__}. + + @ivar stack: Deprecated, always an empty list. Equivalent information can + be extracted from C{import traceback; + traceback.extract_stack(your_failure.tb)} + @ivar frames: list of frames, innermost first. """ pickled = 0 - stack = None _parents = None - # The opcode of "yield" in Python bytecode. We need this in - # _findFailure in order to identify whether an exception was - # thrown by a throwExceptionIntoGenerator. - # on PY3, b'a'[0] == 97 while in py2 b'a'[0] == b'a' opcodes - # are stored in bytes so we need to properly account for this - # difference. - _yieldOpcode = opcode.opmap["YIELD_VALUE"] - def __init__(self, exc_value=None, exc_type=None, exc_tb=None, captureVars=False): """ Initialize me with an explanation of the error. @@ -291,16 +282,10 @@ class Failure(BaseException): self.type = self.value = tb = None self.captureVars = captureVars - stackOffset = 0 - - if exc_value is None: - exc_value = self._findFailure() - if exc_value is None: self.type, self.value, tb = sys.exc_info() if self.type is None: raise NoCurrentExceptionError() - stackOffset = 1 elif exc_type is None: if isinstance(exc_value, Exception): self.type = exc_value.__class__ @@ -316,84 +301,25 @@ class Failure(BaseException): self._extrapolate(self.value) return - if hasattr(self.value, "__failure__"): - # For exceptions propagated through coroutine-awaiting (see - # Deferred.send, AKA Deferred.__next__), which can't be raised as - # Failure because that would mess up the ability to except: them: - self._extrapolate(self.value.__failure__) - - # Clean up the inherently circular reference established by storing - # the failure there. This should make the common case of a Twisted - # / Deferred-returning coroutine somewhat less hard on the garbage - # collector. - del self.value.__failure__ - return - if tb is None: if exc_tb: tb = exc_tb elif getattr(self.value, "__traceback__", None): # Python 3 tb = self.value.__traceback__ - - frames = self.frames = [] - stack = self.stack = [] - - # Added 2003-06-23 by Chris Armstrong. Yes, I actually have a - # use case where I need this traceback object, and I've made - # sure that it'll be cleaned up. self.tb = tb - if tb: - f = tb.tb_frame - elif not isinstance(self.value, Failure): - # We don't do frame introspection since it's expensive, - # and if we were passed a plain exception with no - # traceback, it's not useful anyway - f = stackOffset = None + @property + def frames(self): + if hasattr(self, "_frames"): + return self._frames - while stackOffset and f: - # This excludes this Failure.__init__ frame from the - # stack, leaving it to start with our caller instead. - f = f.f_back - stackOffset -= 1 - - # Keeps the *full* stack. Formerly in spread.pb.print_excFullStack: - # - # The need for this function arises from the fact that several - # PB classes have the peculiar habit of discarding exceptions - # with bareword "except:"s. This premature exception - # catching means tracebacks generated here don't tend to show - # what called upon the PB object. - while f: - if captureVars: - localz = f.f_locals.copy() - if f.f_locals is f.f_globals: - globalz = {} - else: - globalz = f.f_globals.copy() - for d in globalz, localz: - if "__builtins__" in d: - del d["__builtins__"] - localz = localz.items() - globalz = globalz.items() - else: - localz = globalz = () - stack.insert( - 0, - ( - f.f_code.co_name, - f.f_code.co_filename, - f.f_lineno, - localz, - globalz, - ), - ) - f = f.f_back + frames = self._frames = [] + tb = self.tb while tb is not None: f = tb.tb_frame - if captureVars: + if self.captureVars: localz = f.f_locals.copy() if f.f_locals is f.f_globals: globalz = {} @@ -416,6 +342,19 @@ class Failure(BaseException): ) ) tb = tb.tb_next + return frames + + @frames.setter + def frames(self, frames): + self._frames = frames + + @deprecatedProperty(Version("Twisted", 24, 10, 0)) + def stack(self): + return [] + + @stack.setter # type: ignore[no-redef] + def stack(self, stack): + del stack @property def parents(self): @@ -442,26 +381,9 @@ class Failure(BaseException): one. @type otherFailure: L{Failure} """ - # Copy all infos from that failure (including self.frames). + # Copy all infos from that failure (including self._frames). self.__dict__ = copy.copy(otherFailure.__dict__) - # If we are re-throwing a Failure, we merge the stack-trace stored in - # the failure with the current exception's stack. This integrated with - # throwExceptionIntoGenerator and allows to provide full stack trace, - # even if we go through several layers of inlineCallbacks. - _, _, tb = sys.exc_info() - frames = [] - while tb is not None: - f = tb.tb_frame - if f.f_code not in _inlineCallbacksExtraneous: - frames.append( - (f.f_code.co_name, f.f_code.co_filename, tb.tb_lineno, (), ()) - ) - tb = tb.tb_next - # Merging current stack with stack stored in the Failure. - frames.extend(self.frames) - self.frames = frames - @staticmethod def _withoutTraceback(value: BaseException) -> Failure: """ @@ -475,8 +397,6 @@ class Failure(BaseException): count += 1 result.captureVars = False result.count = count - result.frames = [] - result.stack = [] # type: ignore result.value = value result.type = value.__class__ result.tb = None @@ -544,69 +464,8 @@ class Failure(BaseException): @raise StopIteration: If there are no more values in the generator. @raise anything else: Anything that the generator raises. """ - # Note that the actual magic to find the traceback information - # is done in _findFailure. return g.throw(self.value.with_traceback(self.tb)) - @classmethod - def _findFailure(cls): - """ - Find the failure that represents the exception currently in context. - """ - tb = sys.exc_info()[-1] - if not tb: - return - - secondLastTb = None - lastTb = tb - while lastTb.tb_next: - secondLastTb = lastTb - lastTb = lastTb.tb_next - - lastFrame = lastTb.tb_frame - - # NOTE: f_locals.get('self') is used rather than - # f_locals['self'] because psyco frames do not contain - # anything in their locals() dicts. psyco makes debugging - # difficult anyhow, so losing the Failure objects (and thus - # the tracebacks) here when it is used is not that big a deal. - - # Handle raiseException-originated exceptions - if lastFrame.f_code is cls.raiseException.__code__: - return lastFrame.f_locals.get("self") - - # Handle throwExceptionIntoGenerator-originated exceptions - # this is tricky, and differs if the exception was caught - # inside the generator, or above it: - - # It is only really originating from - # throwExceptionIntoGenerator if the bottom of the traceback - # is a yield. - # Pyrex and Cython extensions create traceback frames - # with no co_code, but they can't yield so we know it's okay to - # just return here. - if (not lastFrame.f_code.co_code) or lastFrame.f_code.co_code[ - lastTb.tb_lasti - ] != cls._yieldOpcode: - return - - # If the exception was caught above the generator.throw - # (outside the generator), it will appear in the tb (as the - # second last item): - if secondLastTb: - frame = secondLastTb.tb_frame - if frame.f_code is cls.throwExceptionIntoGenerator.__code__: - return frame.f_locals.get("self") - - # If the exception was caught below the generator.throw - # (inside the generator), it will appear in the frames' linked - # list, above the top-level traceback item (which must be the - # generator frame itself, thus its caller is - # throwExceptionIntoGenerator). - frame = tb.tb_frame.f_back - if frame and frame.f_code is cls.throwExceptionIntoGenerator.__code__: - return frame.f_locals.get("self") - def __repr__(self) -> str: return "<{} {}: {}>".format( reflect.qual(self.__class__), @@ -618,7 +477,10 @@ class Failure(BaseException): return "[Failure instance: %s]" % self.getBriefTraceback() def __setstate__(self, state): + if "stack" in state: + state.pop("stack") state["_parents"] = state.pop("parents") + state["_frames"] = state.pop("frames") self.__dict__.update(state) def __getstate__(self): @@ -636,6 +498,10 @@ class Failure(BaseException): # Backwards compatibility with old code, e.g. for Perspective Broker: c["parents"] = c.pop("_parents") + c["stack"] = [] + + if "_frames" in c: + c.pop("_frames") if self.captureVars: c["frames"] = [ @@ -648,25 +514,12 @@ class Failure(BaseException): ] for v in self.frames ] + else: + c["frames"] = self.frames # Added 2003-06-23. See comment above in __init__ c["tb"] = None - if self.stack is not None: - # XXX: This is a band-aid. I can't figure out where these - # (failure.stack is None) instances are coming from. - if self.captureVars: - c["stack"] = [ - [ - v[0], - v[1], - v[2], - _safeReprVars(v[3]), - _safeReprVars(v[4]), - ] - for v in self.stack - ] - c["pickled"] = 1 return c @@ -682,7 +535,9 @@ class Failure(BaseException): On Python 3, this will also set the C{__traceback__} attribute of the exception instance to L{None}. """ - self.__dict__ = self.__getstate__() + state = self.__getstate__() + state["_frames"] = state.pop("frames") + self.__dict__ = state if getattr(self.value, "__traceback__", None): # Python 3 self.value.__traceback__ = None @@ -700,7 +555,7 @@ class Failure(BaseException): if self.tb is not None: return self.tb elif len(self.frames) > 0: - return _Traceback(self.stack, self.frames) + return _Traceback(self.frames) else: return None @@ -731,9 +586,7 @@ class Failure(BaseException): @param file: If specified, a file-like object to which to write the traceback. - @param elideFrameworkCode: A flag indicating whether to attempt to - remove uninteresting frames from within Twisted itself from the - output. + @param elideFrameworkCode: Deprecated, ignored. @param detail: A string indicating how much information to include in the traceback. Must be one of C{'brief'}, C{'default'}, or @@ -743,6 +596,7 @@ class Failure(BaseException): from twisted.python import log file = log.logerr + w = file.write if detail == "verbose" and not self.captureVars: @@ -773,9 +627,6 @@ class Failure(BaseException): # Frames, formatted in appropriate style if self.frames: - if not elideFrameworkCode: - format_frames(self.stack[-traceupLength:], w, formatDetail) - w(f"{EXCEPTION_CAUGHT_HERE}\n") format_frames(self.frames, w, formatDetail) elif not detail == "brief": # Yeah, it's not really a traceback, despite looking like one... diff --git a/contrib/python/Twisted/py3/twisted/scripts/trial.py b/contrib/python/Twisted/py3/twisted/scripts/trial.py index 531fe46ce17..f3259ed21b7 100644 --- a/contrib/python/Twisted/py3/twisted/scripts/trial.py +++ b/contrib/python/Twisted/py3/twisted/scripts/trial.py @@ -36,6 +36,39 @@ TBFORMAT_MAP = { } +def _autoJobs() -> int: + """ + Heuristically guess the number of job workers to run. + + When ``os.process_cpu_count()`` is available (Python 3.13+), + return the number of logical CPUs usable by the current + process. This respects the ``PYTHON_CPU_COUNT`` environment + variable and/or ``python -X cpu_count`` flag. + + Otherwise, if ``os.sched_getaffinity()`` is available (on some + Unixes) this returns the number of CPUs this process is + restricted to, under the assumption that this affinity will + be inherited. + + Otherwise, consult ``os.cpu_count()`` to get the number of + logical CPUs. + + Failing all else, return 1. + + @returns: A strictly positive integer. + """ + number: Optional[int] + if getattr(os, "process_cpu_count", None) is not None: + number = os.process_cpu_count() # type: ignore[attr-defined] + elif getattr(os, "sched_getaffinity", None) is not None: + number = len(os.sched_getaffinity(0)) + else: + number = os.cpu_count() + if number is None or number < 1: + return 1 + return number + + def _parseLocalVariables(line): """ Accepts a single line in Emacs local variable declaration format and @@ -477,18 +510,22 @@ class Options(_BasicOptions, usage.Options, app.ReactorSelectionMixin): def opt_jobs(self, number): """ - Number of local workers to run, a strictly positive integer. + Number of local workers to run, a strictly positive integer or 'auto' + to spawn one worker for each available CPU. """ - try: - number = int(number) - except ValueError: - raise usage.UsageError( - "Expecting integer argument to jobs, got '%s'" % number - ) - if number <= 0: - raise usage.UsageError( - "Argument to jobs must be a strictly positive integer" - ) + if number == "auto": + number = _autoJobs() + else: + try: + number = int(number) + except ValueError: + raise usage.UsageError( + "Expecting integer argument to jobs, got '%s'" % number + ) + if number <= 0: + raise usage.UsageError( + "Argument to jobs must be a strictly positive integer or 'auto'" + ) self["jobs"] = number def _getWorkerArguments(self): diff --git a/contrib/python/Twisted/py3/twisted/spread/pb.py b/contrib/python/Twisted/py3/twisted/spread/pb.py index 1a58dc6c59f..031292e318c 100644 --- a/contrib/python/Twisted/py3/twisted/spread/pb.py +++ b/contrib/python/Twisted/py3/twisted/spread/pb.py @@ -520,7 +520,7 @@ setUnjellyableForClass(CopyableFailure, CopiedFailure) def failure2Copyable(fail, unsafeTracebacks=0): - f = _newInstance(CopyableFailure, fail.__dict__) + f = _newInstance(CopyableFailure, fail.__getstate__()) f.unsafeTracebacks = unsafeTracebacks return f diff --git a/contrib/python/Twisted/py3/twisted/web/_abnf.py b/contrib/python/Twisted/py3/twisted/web/_abnf.py new file mode 100644 index 00000000000..8029943beab --- /dev/null +++ b/contrib/python/Twisted/py3/twisted/web/_abnf.py @@ -0,0 +1,68 @@ +# -*- test-case-name: twisted.web.test.test_abnf -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tools for pedantically processing the HTTP protocol. +""" + + +def _istoken(b: bytes) -> bool: + """ + Is the string a token per RFC 9110 section 5.6.2? + """ + for c in b: + if c not in ( + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" # ALPHA + b"0123456789" # DIGIT + b"!#$%&'*+-.^_`|~" + ): + return False + return b != b"" + + +def _decint(data: bytes) -> int: + """ + Parse a decimal integer of the form C{1*DIGIT}, i.e. consisting only of + decimal digits. The integer may be embedded in whitespace (space and + horizontal tab). This differs from the built-in L{int()} function by + disallowing a leading C{+} character and various forms of whitespace + (note that we sanitize linear whitespace in header values in + L{twisted.web.http_headers.Headers}). + + @param data: Value to parse. + + @returns: A non-negative integer. + + @raises ValueError: When I{value} contains non-decimal characters. + """ + data = data.strip(b" \t") + if not data.isdigit(): + raise ValueError(f"Value contains non-decimal digits: {data!r}") + return int(data) + + +def _ishexdigits(b: bytes) -> bool: + """ + Is the string case-insensitively hexidecimal? + + It must be composed of one or more characters in the ranges a-f, A-F + and 0-9. + """ + for c in b: + if c not in b"0123456789abcdefABCDEF": + return False + return b != b"" + + +def _hexint(b: bytes) -> int: + """ + Decode a hexadecimal integer. + + Unlike L{int(b, 16)}, this raises L{ValueError} when the integer has + a prefix like C{b'0x'}, C{b'+'}, or C{b'-'}, which is desirable when + parsing network protocols. + """ + if not _ishexdigits(b): + raise ValueError(b) + return int(b, 16) diff --git a/contrib/python/Twisted/py3/twisted/web/_http2.py b/contrib/python/Twisted/py3/twisted/web/_http2.py index f048c7335ec..301e9ea196b 100644 --- a/contrib/python/Twisted/py3/twisted/web/_http2.py +++ b/contrib/python/Twisted/py3/twisted/web/_http2.py @@ -14,7 +14,6 @@ This API is currently considered private because it's in early draft form. When it has stabilised, it'll be made public. """ - import io from collections import deque from typing import List @@ -36,6 +35,7 @@ from twisted.internet.interfaces import ( IProtocol, IPushProducer, ISSLTransport, + ITCPTransport, ITransport, ) from twisted.internet.protocol import Protocol @@ -144,6 +144,8 @@ class H2Connection(Protocol, TimeoutMixin): by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade to HTTP/2. """ + if ITCPTransport.providedBy(self.transport): + self.transport.setTcpNoDelay(True) self.setTimeout(self.timeOut) self.conn.initiate_connection() self.transport.write(self.conn.data_to_send()) @@ -972,7 +974,7 @@ class H2Stream: self._request.gotLength(None) self._request.parseCookies() - expectContinue = self._request.requestHeaders.getRawHeaders(b"expect") + expectContinue = self._request.requestHeaders.getRawHeaders(b"Expect") if expectContinue and expectContinue[0].lower() == b"100-continue": self._send100Continue() diff --git a/contrib/python/Twisted/py3/twisted/web/_newclient.py b/contrib/python/Twisted/py3/twisted/web/_newclient.py index a151bdae05c..32e12521288 100644 --- a/contrib/python/Twisted/py3/twisted/web/_newclient.py +++ b/contrib/python/Twisted/py3/twisted/web/_newclient.py @@ -35,13 +35,14 @@ from zope.interface import implementer from twisted.internet.defer import CancelledError, Deferred, fail, succeed from twisted.internet.error import ConnectionDone -from twisted.internet.interfaces import IConsumer, IPushProducer +from twisted.internet.interfaces import IConsumer, IPushProducer, ITCPTransport from twisted.internet.protocol import Protocol from twisted.logger import Logger from twisted.protocols.basic import LineReceiver from twisted.python.compat import networkString from twisted.python.components import proxyForInterface from twisted.python.failure import Failure +from twisted.web._abnf import _decint, _istoken from twisted.web.http import ( NO_CONTENT, NOT_MODIFIED, @@ -478,7 +479,7 @@ class HTTPClientParser(HTTPParser): self.response._bodyDataFinished() else: transferEncodingHeaders = self.connHeaders.getRawHeaders( - b"transfer-encoding" + b"Transfer-Encoding" ) if transferEncodingHeaders: # This could be a KeyError. However, that would mean we do not @@ -556,35 +557,6 @@ class HTTPClientParser(HTTPParser): del self._responseDeferred -_VALID_METHOD = re.compile( - rb"\A[%s]+\Z" - % ( - bytes().join( - ( - b"!", - b"#", - b"$", - b"%", - b"&", - b"'", - b"*", - b"+", - b"-", - b".", - b"^", - b"_", - b"`", - b"|", - b"~", - b"\x30-\x39", - b"\x41-\x5a", - b"\x61-\x7a", - ), - ), - ), -) - - def _ensureValidMethod(method): """ An HTTP method is an HTTP token, which consists of any visible @@ -603,7 +575,7 @@ def _ensureValidMethod(method): U{https://tools.ietf.org/html/rfc7230#section-3.2.6}, U{https://tools.ietf.org/html/rfc5234#appendix-B.1} """ - if _VALID_METHOD.match(method): + if _istoken(method): return method raise ValueError(f"Invalid method {method!r}") @@ -634,27 +606,6 @@ def _ensureValidURI(uri): raise ValueError(f"Invalid URI {uri!r}") -def _decint(data: bytes) -> int: - """ - Parse a decimal integer of the form C{1*DIGIT}, i.e. consisting only of - decimal digits. The integer may be embedded in whitespace (space and - horizontal tab). This differs from the built-in L{int()} function by - disallowing a leading C{+} character and various forms of whitespace - (note that we sanitize linear whitespace in header values in - L{twisted.web.http_headers.Headers}). - - @param data: Value to parse. - - @returns: A non-negative integer. - - @raises ValueError: When I{value} contains non-decimal characters. - """ - data = data.strip(b" \t") - if not data.isdigit(): - raise ValueError(f"Value contains non-decimal digits: {data!r}") - return int(data) - - def _contentLength(connHeaders: Headers) -> Optional[int]: """ Parse the I{Content-Length} connection header. @@ -681,7 +632,7 @@ def _contentLength(connHeaders: Headers) -> Optional[int]: @see: U{https://datatracker.ietf.org/doc/html/rfc9110#section-8.6} """ - headers = connHeaders.getRawHeaders(b"content-length") + headers = connHeaders.getRawHeaders(b"Content-Length") if headers is None: return None @@ -781,7 +732,7 @@ class Request: return getattr(self._parsedURI, "toBytes", lambda: None)() def _writeHeaders(self, transport, TEorCL): - hosts = self.headers.getRawHeaders(b"host", ()) + hosts = self.headers.getRawHeaders(b"Host", ()) if len(hosts) != 1: raise BadHeaders("Exactly one Host header required") @@ -1547,6 +1498,10 @@ class HTTP11ClientProtocol(Protocol): self._quiescentCallback = quiescentCallback self._abortDeferreds = [] + def connectionMade(self) -> None: + if ITCPTransport.providedBy(self.transport): + self.transport.setTcpNoDelay(True) + @property def state(self): return self._state @@ -1661,7 +1616,7 @@ class HTTP11ClientProtocol(Protocol): return reason = ConnectionDone("synthetic!") - connHeaders = self._parser.connHeaders.getRawHeaders(b"connection", ()) + connHeaders = self._parser.connHeaders.getRawHeaders(b"Connection", ()) if ( (b"close" in connHeaders) or self._state != "QUIESCENT" diff --git a/contrib/python/Twisted/py3/twisted/web/_stan.py b/contrib/python/Twisted/py3/twisted/web/_stan.py index 88e82d2dfe2..b165bdb6fda 100644 --- a/contrib/python/Twisted/py3/twisted/web/_stan.py +++ b/contrib/python/Twisted/py3/twisted/web/_stan.py @@ -32,7 +32,7 @@ if TYPE_CHECKING: from twisted.web.template import Flattenable [email protected](hash=False, eq=False, auto_attribs=True) [email protected](unsafe_hash=False, eq=False, auto_attribs=True) class slot: """ Marker for markup insertion in a template. @@ -82,7 +82,7 @@ class slot: """ [email protected](hash=False, eq=False, repr=False, auto_attribs=True) [email protected](unsafe_hash=False, eq=False, repr=False, auto_attribs=True) class Tag: """ A L{Tag} represents an XML tags with a tag name, attributes, and children. @@ -314,7 +314,7 @@ voidElements = ( ) [email protected](hash=False, eq=False, repr=False, auto_attribs=True) [email protected](unsafe_hash=False, eq=False, repr=False, auto_attribs=True) class CDATA: """ A C{<![CDATA[]]>} block from a template. Given a separate representation in @@ -329,7 +329,7 @@ class CDATA: return f"CDATA({self.data!r})" [email protected](hash=False, eq=False, repr=False, auto_attribs=True) [email protected](unsafe_hash=False, eq=False, repr=False, auto_attribs=True) class Comment: """ A C{<!-- -->} comment from a template. Given a separate representation in @@ -344,7 +344,7 @@ class Comment: return f"Comment({self.data!r})" [email protected](hash=False, eq=False, repr=False, auto_attribs=True) [email protected](unsafe_hash=False, eq=False, repr=False, auto_attribs=True) class CharRef: """ A numeric character reference. Given a separate representation in the DOM diff --git a/contrib/python/Twisted/py3/twisted/web/client.py b/contrib/python/Twisted/py3/twisted/web/client.py index b06f1bef286..cfd945d5d40 100644 --- a/contrib/python/Twisted/py3/twisted/web/client.py +++ b/contrib/python/Twisted/py3/twisted/web/client.py @@ -37,7 +37,7 @@ from twisted.python.deprecate import ( from twisted.python.failure import Failure from twisted.web import error, http from twisted.web._newclient import _ensureValidMethod, _ensureValidURI -from twisted.web.http_headers import Headers +from twisted.web.http_headers import Headers, _nameEncoder from twisted.web.iweb import ( UNKNOWN_LENGTH, IAgent, @@ -924,10 +924,10 @@ class _AgentBase: # Create minimal headers, if necessary: if headers is None: headers = Headers() - if not headers.hasHeader(b"host"): + if not headers.hasHeader(b"Host"): headers = headers.copy() headers.addRawHeader( - b"host", + b"Host", self._computeHostValue( parsedURI.scheme, parsedURI.host, parsedURI.port ), @@ -1367,12 +1367,12 @@ class CookieAgent: lastRequest = _FakeStdlibRequest(uri) # Setting a cookie header explicitly will disable automatic request # cookies. - if not actualHeaders.hasHeader(b"cookie"): + if not actualHeaders.hasHeader(b"Cookie"): self.cookieJar.add_cookie_header(lastRequest) cookieHeader = lastRequest.get_header("Cookie", None) if cookieHeader is not None: actualHeaders = actualHeaders.copy() - actualHeaders.addRawHeader(b"cookie", networkString(cookieHeader)) + actualHeaders.addRawHeader(b"Cookie", networkString(cookieHeader)) return self._agent.request( method, uri, actualHeaders, bodyProducer @@ -1502,7 +1502,7 @@ class ContentDecoderAgent: headers = Headers() else: headers = headers.copy() - headers.addRawHeader(b"accept-encoding", self._supported) + headers.addRawHeader(b"Accept-Encoding", self._supported) deferred = self._agent.request(method, uri, headers, bodyProducer) return deferred.addCallback(self._handleResponse) @@ -1510,7 +1510,7 @@ class ContentDecoderAgent: """ Check if the response is encoded, and wrap it to handle decompression. """ - contentEncodingHeaders = response.headers.getRawHeaders(b"content-encoding", []) + contentEncodingHeaders = response.headers.getRawHeaders(b"Content-Encoding", []) contentEncodingHeaders = b",".join(contentEncodingHeaders).split(b",") while contentEncodingHeaders: name = contentEncodingHeaders.pop().strip() @@ -1523,14 +1523,14 @@ class ContentDecoderAgent: break if contentEncodingHeaders: response.headers.setRawHeaders( - b"content-encoding", [b",".join(contentEncodingHeaders)] + b"Content-Encoding", [b",".join(contentEncodingHeaders)] ) else: - response.headers.removeHeader(b"content-encoding") + response.headers.removeHeader(b"Content-Encoding") return response -_canonicalHeaderName = Headers()._encodeName +_canonicalHeaderName = _nameEncoder.encode _defaultSensitiveHeaders = frozenset( [ b"Authorization", diff --git a/contrib/python/Twisted/py3/twisted/web/http.py b/contrib/python/Twisted/py3/twisted/web/http.py index e80f6cb365f..8aa31dfe306 100644 --- a/contrib/python/Twisted/py3/twisted/web/http.py +++ b/contrib/python/Twisted/py3/twisted/web/http.py @@ -107,11 +107,11 @@ import math import os import re import tempfile -import time import warnings from email import message_from_bytes from email.message import EmailMessage, Message from io import BufferedIOBase, BytesIO, TextIOWrapper +from time import gmtime, time from typing import ( AnyStr, Callable, @@ -134,7 +134,13 @@ from incremental import Version from twisted.internet import address, interfaces, protocol from twisted.internet._producer_helpers import _PullToPush from twisted.internet.defer import Deferred -from twisted.internet.interfaces import IAddress, IDelayedCall, IProtocol, IReactorTime +from twisted.internet.interfaces import ( + IAddress, + IDelayedCall, + IProtocol, + IReactorTime, + ITCPTransport, +) from twisted.internet.protocol import Protocol from twisted.logger import Logger from twisted.protocols import basic, policies @@ -143,6 +149,7 @@ from twisted.python.compat import nativeString, networkString from twisted.python.components import proxyForInterface from twisted.python.deprecate import deprecated, deprecatedModuleAttribute from twisted.python.failure import Failure +from twisted.web._abnf import _hexint, _istoken from twisted.web._responses import ( ACCEPTED, BAD_GATEWAY, @@ -190,7 +197,12 @@ from twisted.web._responses import ( UNSUPPORTED_MEDIA_TYPE, USE_PROXY, ) -from twisted.web.http_headers import Headers, _sanitizeLinearWhitespace +from twisted.web.http_headers import ( + Headers, + InvalidHeaderName, + _nameEncoder, + _sanitizeLinearWhitespace, +) from twisted.web.iweb import IAccessLogFormatter, INonQueuedRequestFactory, IRequest try: @@ -217,8 +229,7 @@ responses = RESPONSES # datetime parsing and formatting weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] -monthname = [ - None, +_months = [ "Jan", "Feb", "Mar", @@ -232,6 +243,9 @@ monthname = [ "Nov", "Dec", ] +monthname = [None] + _months +_weekdaynameBytes = [s.encode("ascii") for s in weekdayname] +_monthnameBytes = [None] + [s.encode("ascii") for s in _months] weekdayname_lower = [name.lower() for name in weekdayname] monthname_lower = [name and name.lower() for name in monthname] @@ -391,14 +405,18 @@ def datetimeToString(msSinceEpoch=None): @rtype: C{bytes} """ - if msSinceEpoch == None: - msSinceEpoch = time.time() - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch) - s = networkString( - "%s, %02d %3s %4d %02d:%02d:%02d GMT" - % (weekdayname[wd], day, monthname[month], year, hh, mm, ss) + year, month, day, hh, mm, ss, wd, _, _ = ( + gmtime() if msSinceEpoch is None else gmtime(msSinceEpoch) + ) + return b"%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdaynameBytes[wd], + day, + _monthnameBytes[month], + year, + hh, + mm, + ss, ) - return s def datetimeToLogString(msSinceEpoch=None): @@ -408,8 +426,9 @@ def datetimeToLogString(msSinceEpoch=None): @rtype: C{str} """ if msSinceEpoch == None: - msSinceEpoch = time.time() - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch) + # This code path is apparently never used in practice inside Twisted. + msSinceEpoch = time() # pragma: no cover + year, month, day, hh, mm, ss, wd, y, z = gmtime(msSinceEpoch) s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( day, monthname[month], @@ -507,46 +526,6 @@ def toChunk(data): return (networkString(f"{len(data):x}"), b"\r\n", data, b"\r\n") -def _istoken(b: bytes) -> bool: - """ - Is the string a token per RFC 9110 section 5.6.2? - """ - for c in b: - if c not in ( - b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" # ALPHA - b"0123456789" # DIGIT - b"!#$%^'*+-.^_`|~" - ): - return False - return b != b"" - - -def _ishexdigits(b: bytes) -> bool: - """ - Is the string case-insensitively hexidecimal? - - It must be composed of one or more characters in the ranges a-f, A-F - and 0-9. - """ - for c in b: - if c not in b"0123456789abcdefABCDEF": - return False - return b != b"" - - -def _hexint(b: bytes) -> int: - """ - Decode a hexadecimal integer. - - Unlike L{int(b, 16)}, this raises L{ValueError} when the integer has - a prefix like C{b'0x'}, C{b'+'}, or C{b'-'}, which is desirable when - parsing network protocols. - """ - if not _ishexdigits(b): - raise ValueError(b) - return int(b, 16) - - def fromChunk(data: bytes) -> Tuple[bytes, bytes]: """ Convert chunk to string. @@ -1031,7 +1010,7 @@ class Request: This method is not intended for users. """ - cookieheaders = self.requestHeaders.getRawHeaders(b"cookie") + cookieheaders = self.requestHeaders.getRawHeaders(b"Cookie") if cookieheaders is None: return @@ -1086,7 +1065,7 @@ class Request: # Argument processing args = self.args - ctype = self.requestHeaders.getRawHeaders(b"content-type") + ctype = self.requestHeaders.getRawHeaders(b"Content-Type") if ctype is not None: ctype = ctype[0] @@ -1270,7 +1249,7 @@ class Request: """ if self.finished: raise RuntimeError( - "Request.write called on a request after " "Request.finish was called." + "Request.write called on a request after Request.finish was called." ) if self._disconnected: @@ -1290,7 +1269,7 @@ class Request: # persistent connections. if ( (version == b"HTTP/1.1") - and (self.responseHeaders.getRawHeaders(b"content-length") is None) + and (self.responseHeaders.getRawHeaders(b"Content-Length") is None) and self.method != b"HEAD" and self.code not in NO_BODY_CODES ): @@ -1298,14 +1277,14 @@ class Request: self.chunked = 1 if self.lastModified is not None: - if self.responseHeaders.hasHeader(b"last-modified"): + if self.responseHeaders.hasHeader(b"Last-Modified"): self._log.info( "Warning: last-modified specified both in" " header list and lastModified attribute." ) else: self.responseHeaders.setRawHeaders( - b"last-modified", [datetimeToString(self.lastModified)] + b"Last-Modified", [datetimeToString(self.lastModified)] ) if self.etag is not None: @@ -1483,7 +1462,7 @@ class Request: @type url: L{bytes} or L{str} """ self.setResponseCode(FOUND) - self.setHeader(b"location", url) + self.setHeader(b"Location", url) def setLastModified(self, when): """ @@ -1510,7 +1489,7 @@ class Request: if (not self.lastModified) or (self.lastModified < when): self.lastModified = when - modifiedSince = self.getHeader(b"if-modified-since") + modifiedSince = self.getHeader(b"If-Modified-Since") if modifiedSince: firstPart = modifiedSince.split(b";", 1)[0] try: @@ -1544,7 +1523,7 @@ class Request: if etag: self.etag = etag - tags = self.getHeader(b"if-none-match") + tags = self.getHeader(b"If-None-Match") if tags: tags = tags.split() if (etag in tags) or (b"*" in tags): @@ -1578,7 +1557,7 @@ class Request: @rtype: C{bytes} """ - host = self.getHeader(b"host") + host = self.getHeader(b"Host") if host is not None: match = _hostHeaderExpression.match(host) if match is not None: @@ -1626,7 +1605,7 @@ class Request: hostHeader = host else: hostHeader = b"%b:%d" % (host, port) - self.requestHeaders.setRawHeaders(b"host", [hostHeader]) + self.requestHeaders.setRawHeaders(b"Host", [hostHeader]) self.host = address.IPv4Address("TCP", host, port) @deprecated(Version("Twisted", 18, 4, 0), replacement="getClientAddress") @@ -1828,6 +1807,8 @@ class _IdentityTransferDecoder: chunk. """ + __slots__ = ["contentLength", "dataCallback", "finishCallback"] + def __init__(self, contentLength, dataCallback, finishCallback): self.contentLength = contentLength self.dataCallback = dataCallback @@ -2324,7 +2305,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): totalHeadersSize = 16384 abortTimeout = 15 - length = 0 + length: Optional[int] = 0 persistent = 1 __header = b"" __first_line = 1 @@ -2351,6 +2332,8 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): self._transferDecoder = None def connectionMade(self): + if ITCPTransport.providedBy(self.transport): + self.transport.setTcpNoDelay(True) self.setTimeout(self.timeOut) self._networkProducer = interfaces.IPushProducer( self.transport, _NoPushProducer() @@ -2431,6 +2414,14 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): self._dataBuffer.append(data) self.allContentReceived() + def _failChooseTransferDecoder(self) -> bool: + """ + Utility to indicate failure to choose a decoder. + """ + self._respondToBadRequestAndDisconnect() + self.length = None + return False + def _maybeChooseTransferDecoder(self, header, data): """ If the provided header is C{content-length} or @@ -2438,24 +2429,15 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): Returns L{True} if the request can proceed and L{False} if not. """ - - def fail(): - self._respondToBadRequestAndDisconnect() - self.length = None - return False - # Can this header determine the length? - if header == b"content-length": + if header == b"Content-Length": if not data.isdigit(): - return fail() - try: - length = int(data) - except ValueError: - return fail() + return self._failChooseTransferDecoder() + length = int(data) newTransferDecoder = _IdentityTransferDecoder( length, self.requests[-1].handleContentChunk, self._finishRequestBody ) - elif header == b"transfer-encoding": + elif header == b"Transfer-Encoding": # XXX Rather poorly tested code block, apparently only exercised by # test_chunkedEncoding if data.lower() == b"chunked": @@ -2466,13 +2448,13 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): elif data.lower() == b"identity": return True else: - return fail() + return self._failChooseTransferDecoder() else: # It's not a length related header, so exit return True if self._transferDecoder is not None: - return fail() + return self._failChooseTransferDecoder() else: self.length = length self._transferDecoder = newTransferDecoder @@ -2480,7 +2462,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): def headerReceived(self, line): """ - Do pre-processing (for content-length) and store this header away. + Do pre-processing (for Content-Length) and store this header away. Enforce the per-request header limit. @type line: C{bytes} @@ -2496,13 +2478,17 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): self._respondToBadRequestAndDisconnect() return False - # Header names must be tokens, per RFC 9110 section 5.1. - if not _istoken(header): + # Canonicalize the header name. + try: + header = _nameEncoder.encode(header) + except InvalidHeaderName: self._respondToBadRequestAndDisconnect() return False - header = header.lower() data = data.strip(b" \t") + if b"\x00" in data: + self._respondToBadRequestAndDisconnect() + return False if not self._maybeChooseTransferDecoder(header, data): return False @@ -2592,7 +2578,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): req.gotLength(self.length) # Handle 'Expect: 100-continue' with automated 100 response code, # a simplistic implementation of RFC 2686 8.2.3: - expectContinue = req.requestHeaders.getRawHeaders(b"expect") + expectContinue = req.requestHeaders.getRawHeaders(b"Expect") if ( expectContinue and expectContinue[0].lower() == b"100-continue" @@ -2617,7 +2603,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): must be closed in order to indicate the completion of the response to C{request}. """ - connection = request.requestHeaders.getRawHeaders(b"connection") + connection = request.requestHeaders.getRawHeaders(b"Connection") if connection: tokens = [t.lower() for t in connection[0].split(b" ")] else: @@ -2636,7 +2622,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): if version == b"HTTP/1.1": if b"close" in tokens: - request.responseHeaders.setRawHeaders(b"connection", [b"close"]) + request.responseHeaders.setRawHeaders(b"Connection", [b"close"]) return False else: return True @@ -3034,7 +3020,7 @@ class _XForwardedForRequest(proxyForInterface(IRequest, "_request")): # type: i expected by L{combinedLogFormatter}. """ host = ( - self._request.requestHeaders.getRawHeaders(b"x-forwarded-for", [b"-"])[0] + self._request.requestHeaders.getRawHeaders(b"X-Forwarded-For", [b"-"])[0] .split(b",")[0] .strip() ) diff --git a/contrib/python/Twisted/py3/twisted/web/http_headers.py b/contrib/python/Twisted/py3/twisted/web/http_headers.py index 8b1d41adb64..88b653439b5 100644 --- a/contrib/python/Twisted/py3/twisted/web/http_headers.py +++ b/contrib/python/Twisted/py3/twisted/web/http_headers.py @@ -22,18 +22,26 @@ from typing import ( ) from twisted.python.compat import cmp, comparable +from twisted.web._abnf import _istoken + + +class InvalidHeaderName(ValueError): + """ + HTTP header names must be tokens, per RFC 9110 section 5.1. + """ + _T = TypeVar("_T") def _sanitizeLinearWhitespace(headerComponent: bytes) -> bytes: r""" - Replace linear whitespace (C{\n}, C{\r\n}, C{\r}) in a header key - or value with a single space. + Replace linear whitespace (C{\n}, C{\r\n}, C{\r}) in a header + value with a single space. - @param headerComponent: The header key or value to sanitize. + @param headerComponent: The header value to sanitize. - @return: The sanitized header key or value. + @return: The sanitized header value. """ return b" ".join(headerComponent.splitlines()) @@ -53,31 +61,10 @@ class Headers: ensure no decoding or encoding is done, and L{Headers} will treat the keys and values as opaque byte strings. - @cvar _caseMappings: A L{dict} that maps lowercase header names - to their canonicalized representation, for headers with unconventional - capitalization. - - @cvar _canonicalHeaderCache: A L{dict} that maps header names to their - canonicalized representation. - @ivar _rawHeaders: A L{dict} mapping header names as L{bytes} to L{list}s of header values as L{bytes}. """ - _caseMappings: ClassVar[Dict[bytes, bytes]] = { - b"content-md5": b"Content-MD5", - b"dnt": b"DNT", - b"etag": b"ETag", - b"p3p": b"P3P", - b"te": b"TE", - b"www-authenticate": b"WWW-Authenticate", - b"x-xss-protection": b"X-XSS-Protection", - } - - _canonicalHeaderCache: ClassVar[Dict[Union[bytes, str], bytes]] = {} - - _MAX_CACHED_HEADERS: ClassVar[int] = 10_000 - __slots__ = ["_rawHeaders"] def __init__( @@ -109,39 +96,6 @@ class Headers: ) return NotImplemented - def _encodeName(self, name: Union[str, bytes]) -> bytes: - """ - Encode the name of a header (eg 'Content-Type') to an ISO-8859-1 - encoded bytestring if required. It will be canonicalized and - whitespace-sanitized. - - @param name: A HTTP header name - - @return: C{name}, encoded if required, lowercased - """ - if canonicalName := self._canonicalHeaderCache.get(name, None): - return canonicalName - - bytes_name = name.encode("iso-8859-1") if isinstance(name, str) else name - - if bytes_name.lower() in self._caseMappings: - # Some headers have special capitalization: - result = self._caseMappings[bytes_name.lower()] - else: - result = _sanitizeLinearWhitespace( - b"-".join([word.capitalize() for word in bytes_name.split(b"-")]) - ) - - # In general, we should only see a very small number of header - # variations in the real world, so caching them is fine. However, an - # attacker could generate infinite header variations to fill up RAM, so - # we cap how many we cache. The performance degradation from lack of - # caching won't be that bad, and legit traffic won't hit it. - if len(self._canonicalHeaderCache) < self._MAX_CACHED_HEADERS: - self._canonicalHeaderCache[name] = result - - return result - def copy(self): """ Return a copy of itself with the same headers set. @@ -158,7 +112,7 @@ class Headers: @return: C{True} if the header exists, otherwise C{False}. """ - return self._encodeName(name) in self._rawHeaders + return _nameEncoder.encode(name) in self._rawHeaders def removeHeader(self, name: AnyStr) -> None: """ @@ -168,7 +122,7 @@ class Headers: @return: L{None} """ - self._rawHeaders.pop(self._encodeName(name), None) + self._rawHeaders.pop(_nameEncoder.encode(name), None) def setRawHeaders( self, name: Union[str, bytes], values: Sequence[Union[str, bytes]] @@ -186,7 +140,7 @@ class Headers: @return: L{None} """ - _name = self._encodeName(name) + _name = _nameEncoder.encode(name) encodedValues: List[bytes] = [] for v in values: if isinstance(v, str): @@ -205,7 +159,7 @@ class Headers: @param value: The value to set for the named header. """ - self._rawHeaders.setdefault(self._encodeName(name), []).append( + self._rawHeaders.setdefault(_nameEncoder.encode(name), []).append( _sanitizeLinearWhitespace( value.encode("utf8") if isinstance(value, str) else value ) @@ -234,7 +188,7 @@ class Headers: @return: If the named header is present, a sequence of its values. Otherwise, C{default}. """ - encodedName = self._encodeName(name) + encodedName = _nameEncoder.encode(name) values = self._rawHeaders.get(encodedName, []) if not values: return default @@ -252,4 +206,79 @@ class Headers: return iter(self._rawHeaders.items()) +class _NameEncoder: + """ + C{_NameEncoder} converts HTTP header names to L{bytes} and canonicalizies + their capitalization. + + @cvar _caseMappings: A L{dict} that maps conventionally-capitalized + header names to their canonicalized representation, for headers with + unconventional capitalization. + + @cvar _canonicalHeaderCache: A L{dict} that maps header names to their + canonicalized representation. + """ + + __slots__ = ("_canonicalHeaderCache",) + _canonicalHeaderCache: Dict[Union[bytes, str], bytes] + + _caseMappings: ClassVar[Dict[bytes, bytes]] = { + b"Content-Md5": b"Content-MD5", + b"Dnt": b"DNT", + b"Etag": b"ETag", + b"P3p": b"P3P", + b"Te": b"TE", + b"Www-Authenticate": b"WWW-Authenticate", + b"X-Xss-Protection": b"X-XSS-Protection", + } + + _MAX_CACHED_HEADERS: ClassVar[int] = 10_000 + + def __init__(self): + self._canonicalHeaderCache = {} + + def encode(self, name: Union[str, bytes]) -> bytes: + """ + Encode the name of a header (eg 'Content-Type') to an ISO-8859-1 + bytestring if required. It will be canonicalized to Http-Header-Case. + + @raises InvalidHeaderName: + If the header name contains invalid characters like whitespace + or NUL. + + @param name: An HTTP header name + + @return: C{name}, encoded if required, in Header-Case + """ + if canonicalName := self._canonicalHeaderCache.get(name): + return canonicalName + + bytes_name = name.encode("iso-8859-1") if isinstance(name, str) else name + + if not _istoken(bytes_name): + raise InvalidHeaderName(bytes_name) + + result = b"-".join([word.capitalize() for word in bytes_name.split(b"-")]) + + # Some headers have special capitalization: + if result in self._caseMappings: + result = self._caseMappings[result] + + # In general, we should only see a very small number of header + # variations in the real world, so caching them is fine. However, an + # attacker could generate infinite header variations to fill up RAM, so + # we cap how many we cache. The performance degradation from lack of + # caching won't be that bad, and legit traffic won't hit it. + if len(self._canonicalHeaderCache) < self._MAX_CACHED_HEADERS: + self._canonicalHeaderCache[name] = result + + return result + + +_nameEncoder = _NameEncoder() +""" +The global name encoder. +""" + + __all__ = ["Headers"] diff --git a/contrib/python/Twisted/py3/twisted/web/server.py b/contrib/python/Twisted/py3/twisted/web/server.py index cfcefad7f36..1a4318022b9 100644 --- a/contrib/python/Twisted/py3/twisted/web/server.py +++ b/contrib/python/Twisted/py3/twisted/web/server.py @@ -13,7 +13,6 @@ This is a web server which integrates with the twisted.internet infrastructure. value. """ - import copy import os import re @@ -192,8 +191,8 @@ class Request(Copyable, http.Request, components.Componentized): self.site = self.channel.site # set various default headers - self.setHeader(b"server", version) - self.setHeader(b"date", datetimeToString()) + self.setHeader(b"Server", version) + self.setHeader(b"Date", datetimeToString()) # Resource Identification self.prepath = [] @@ -228,8 +227,8 @@ class Request(Copyable, http.Request, components.Componentized): # is a Content-Length header set to 0, as empty bodies don't need # a content-type. needsCT = self.code not in (NOT_MODIFIED, NO_CONTENT) - contentType = self.responseHeaders.getRawHeaders(b"content-type") - contentLength = self.responseHeaders.getRawHeaders(b"content-length") + contentType = self.responseHeaders.getRawHeaders(b"Content-Type") + contentLength = self.responseHeaders.getRawHeaders(b"Content-Length") contentLengthZero = contentLength and (contentLength[0] == b"0") if ( @@ -239,7 +238,7 @@ class Request(Copyable, http.Request, components.Componentized): and not contentLengthZero ): self.responseHeaders.setRawHeaders( - b"content-type", [self.defaultContentType] + b"Content-Type", [self.defaultContentType] ) # Only let the write happen if we're not generating a HEAD response by @@ -298,7 +297,7 @@ class Request(Copyable, http.Request, components.Componentized): ) # Oh well, I guess we won't include the content length. else: - self.setHeader(b"content-length", b"%d" % (len(body),)) + self.setHeader(b"Content-Length", b"%d" % (len(body),)) self._inFakeHead = False self.method = b"HEAD" @@ -361,10 +360,10 @@ class Request(Copyable, http.Request, components.Componentized): slf=self, resrc=resrc, ) - self.setHeader(b"content-length", b"%d" % (len(body),)) + self.setHeader(b"Content-Length", b"%d" % (len(body),)) self.write(b"") else: - self.setHeader(b"content-length", b"%d" % (len(body),)) + self.setHeader(b"Content-Length", b"%d" % (len(body),)) self.write(body) self.finish() @@ -397,8 +396,8 @@ class Request(Copyable, http.Request, components.Componentized): ) self.setResponseCode(http.INTERNAL_SERVER_ERROR) - self.setHeader(b"content-type", b"text/html") - self.setHeader(b"content-length", b"%d" % (len(body),)) + self.setHeader(b"Content-Type", b"text/html") + self.setHeader(b"Content-Length", b"%d" % (len(body),)) self.write(body) self.finish() return reason @@ -605,16 +604,16 @@ class GzipEncoderFactory: request if so. """ acceptHeaders = b",".join( - request.requestHeaders.getRawHeaders(b"accept-encoding", []) + request.requestHeaders.getRawHeaders(b"Accept-Encoding", []) ) if self._gzipCheckRegex.search(acceptHeaders): - encoding = request.responseHeaders.getRawHeaders(b"content-encoding") + encoding = request.responseHeaders.getRawHeaders(b"Content-Encoding") if encoding: encoding = b",".join(encoding + [b"gzip"]) else: encoding = b"gzip" - request.responseHeaders.setRawHeaders(b"content-encoding", [encoding]) + request.responseHeaders.setRawHeaders(b"Content-Encoding", [encoding]) return _GzipEncoder(self.compressLevel, request) @@ -646,7 +645,7 @@ class _GzipEncoder: if not self._request.startedWriting: # Remove the content-length header, we can't honor it # because we compress on the fly. - self._request.responseHeaders.removeHeader(b"content-length") + self._request.responseHeaders.removeHeader(b"Content-Length") return self._zlibCompressor.compress(data) def finish(self): diff --git a/contrib/python/Twisted/py3/ya.make b/contrib/python/Twisted/py3/ya.make index d74fcea5c50..0ad42217967 100644 --- a/contrib/python/Twisted/py3/ya.make +++ b/contrib/python/Twisted/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(24.7.0) +VERSION(24.10.0) LICENSE(MIT) @@ -35,6 +35,7 @@ PY_SRCS( twisted/_threads/_threadworker.py twisted/_version.py twisted/application/__init__.py + twisted/application/_client_service.py twisted/application/app.py twisted/application/internet.py twisted/application/reactors.py @@ -388,6 +389,7 @@ PY_SRCS( twisted/trial/unittest.py twisted/trial/util.py twisted/web/__init__.py + twisted/web/_abnf.py twisted/web/_auth/__init__.py twisted/web/_auth/basic.py twisted/web/_auth/digest.py diff --git a/contrib/python/ipython/py3/.dist-info/METADATA b/contrib/python/ipython/py3/.dist-info/METADATA index b3a405d51d7..c1f48dd50c3 100644 --- a/contrib/python/ipython/py3/.dist-info/METADATA +++ b/contrib/python/ipython/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ipython -Version: 8.28.0 +Version: 8.29.0 Summary: IPython: Productive Interactive Computing Author: The IPython Development Team Author-email: [email protected] diff --git a/contrib/python/ipython/py3/IPython/core/completer.py b/contrib/python/ipython/py3/IPython/core/completer.py index c639ce07a12..8f843dd5844 100644 --- a/contrib/python/ipython/py3/IPython/core/completer.py +++ b/contrib/python/ipython/py3/IPython/core/completer.py @@ -159,7 +159,7 @@ By default results from all matchers are combined, in the order determined by their priority. Matchers can request to suppress results from subsequent matchers by setting ``suppress`` to ``True`` in the ``MatcherResult``. -When multiple matchers simultaneously request surpression, the results from of +When multiple matchers simultaneously request suppression, the results from of the matcher with higher priority will be returned. Sometimes it is desirable to suppress most but not all other matchers; @@ -2652,7 +2652,7 @@ class IPCompleter(Completer): ) can_close_quote = can_close_quote and self.auto_close_dict_keys - # fast path if closing qoute should be appended but not suffix is allowed + # fast path if closing quote should be appended but not suffix is allowed if not can_close_quote and not can_close_bracket and closing_quote: return [leading + k for k in matches] diff --git a/contrib/python/ipython/py3/IPython/core/completerlib.py b/contrib/python/ipython/py3/IPython/core/completerlib.py index 05f39e50159..4612b326ded 100644 --- a/contrib/python/ipython/py3/IPython/core/completerlib.py +++ b/contrib/python/ipython/py3/IPython/core/completerlib.py @@ -214,7 +214,7 @@ def is_possible_submodule(module, attr): try: obj = getattr(module, attr) except AttributeError: - # Is possilby an unimported submodule + # Is possibly an unimported submodule return True except TypeError: # https://github.com/ipython/ipython/issues/9678 diff --git a/contrib/python/ipython/py3/IPython/core/debugger.py b/contrib/python/ipython/py3/IPython/core/debugger.py index e7a0b8fb554..84d3de8c5b3 100644 --- a/contrib/python/ipython/py3/IPython/core/debugger.py +++ b/contrib/python/ipython/py3/IPython/core/debugger.py @@ -19,7 +19,7 @@ Global Configuration -------------------- The IPython debugger will by read the global ``~/.pdbrc`` file. -That is to say you can list all comands supported by ipdb in your `~/.pdbrc` +That is to say you can list all commands supported by ipdb in your `~/.pdbrc` configuration file, to globally configure pdb. Example:: @@ -177,7 +177,7 @@ def BdbQuit_excepthook(et, ev, tb, excepthook=None): parameter. """ raise ValueError( - "`BdbQuit_excepthook` is deprecated since version 5.1. It is still arround only because it is still imported by ipdb.", + "`BdbQuit_excepthook` is deprecated since version 5.1. It is still around only because it is still imported by ipdb.", ) diff --git a/contrib/python/ipython/py3/IPython/core/display.py b/contrib/python/ipython/py3/IPython/core/display.py index 5c4557b150f..c3c44016f41 100644 --- a/contrib/python/ipython/py3/IPython/core/display.py +++ b/contrib/python/ipython/py3/IPython/core/display.py @@ -21,13 +21,35 @@ from IPython.testing.skipdoctest import skip_doctest from . import display_functions -__all__ = ['display_pretty', 'display_html', 'display_markdown', - 'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', - 'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', - 'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', - 'GeoJSON', 'Javascript', 'Image', 'set_matplotlib_formats', - 'set_matplotlib_close', - 'Video'] +__all__ = [ + "display_pretty", + "display_html", + "display_markdown", + "display_svg", + "display_png", + "display_jpeg", + "display_webp", + "display_latex", + "display_json", + "display_javascript", + "display_pdf", + "DisplayObject", + "TextDisplayObject", + "Pretty", + "HTML", + "Markdown", + "Math", + "Latex", + "SVG", + "ProgressBar", + "JSON", + "GeoJSON", + "Javascript", + "Image", + "set_matplotlib_formats", + "set_matplotlib_close", + "Video", +] _deprecated_names = ["display", "clear_output", "publish_display_data", "update_display", "DisplayHandle"] @@ -200,6 +222,23 @@ def display_jpeg(*objs, **kwargs): _display_mimetype('image/jpeg', objs, **kwargs) +def display_webp(*objs, **kwargs): + """Display the WEBP representation of an object. + + Parameters + ---------- + *objs : object + The Python objects to display, or if raw=True raw JPEG data to + display. + raw : bool + Are the data objects raw data or Python objects that need to be + formatted before display? [default: False] + metadata : dict (optional) + Metadata to be associated with the specific mimetype output. + """ + _display_mimetype("image/webp", objs, **kwargs) + + def display_latex(*objs, **kwargs): """Display the LaTeX representation of an object. @@ -776,9 +815,14 @@ class Javascript(TextDisplayObject): r += _lib_t2*len(self.lib) return r -# constants for identifying png/jpeg data -_PNG = b'\x89PNG\r\n\x1a\n' -_JPEG = b'\xff\xd8' + +# constants for identifying png/jpeg/gif/webp data +_PNG = b"\x89PNG\r\n\x1a\n" +_JPEG = b"\xff\xd8" +_GIF1 = b"GIF87a" +_GIF2 = b"GIF89a" +_WEBP = b"WEBP" + def _pngxy(data): """read the (width, height) from a PNG header""" @@ -786,6 +830,7 @@ def _pngxy(data): # next 8 bytes are width/height return struct.unpack('>ii', data[ihdr+4:ihdr+12]) + def _jpegxy(data): """read the (width, height) from a JPEG header""" # adapted from http://www.64lines.com/jpeg-width-height @@ -805,22 +850,45 @@ def _jpegxy(data): h, w = struct.unpack('>HH', data[iSOF+5:iSOF+9]) return w, h + def _gifxy(data): """read the (width, height) from a GIF header""" return struct.unpack('<HH', data[6:10]) +def _webpxy(data): + """read the (width, height) from a WEBP header""" + if data[12:16] == b"VP8 ": + width, height = struct.unpack("<HH", data[24:30]) + width = width & 0x3FFF + height = height & 0x3FFF + return (width, height) + elif data[12:16] == b"VP8L": + size_info = struct.unpack("<I", data[21:25])[0] + width = 1 + ((size_info & 0x3F) << 8) | (size_info >> 24) + height = 1 + ( + (((size_info >> 8) & 0xF) << 10) + | (((size_info >> 14) & 0x3FC) << 2) + | ((size_info >> 22) & 0x3) + ) + return (width, height) + else: + raise ValueError("Not a valid WEBP header") + + class Image(DisplayObject): - _read_flags = 'rb' - _FMT_JPEG = u'jpeg' - _FMT_PNG = u'png' - _FMT_GIF = u'gif' - _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG, _FMT_GIF] + _read_flags = "rb" + _FMT_JPEG = "jpeg" + _FMT_PNG = "png" + _FMT_GIF = "gif" + _FMT_WEBP = "webp" + _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG, _FMT_GIF, _FMT_WEBP] _MIMETYPES = { - _FMT_PNG: 'image/png', - _FMT_JPEG: 'image/jpeg', - _FMT_GIF: 'image/gif', + _FMT_PNG: "image/png", + _FMT_JPEG: "image/jpeg", + _FMT_GIF: "image/gif", + _FMT_WEBP: "image/webp", } def __init__( @@ -837,7 +905,7 @@ class Image(DisplayObject): metadata=None, alt=None, ): - """Create a PNG/JPEG/GIF image object given raw data. + """Create a PNG/JPEG/GIF/WEBP image object given raw data. When this object is returned by an input cell or passed to the display function, it will result in the image being displayed @@ -858,7 +926,7 @@ class Image(DisplayObject): Images from a file are always embedded. format : unicode - The format of the image data (png/jpeg/jpg/gif). If a filename or URL is given + The format of the image data (png/jpeg/jpg/gif/webp). If a filename or URL is given for format will be inferred from the filename extension. embed : bool @@ -942,6 +1010,8 @@ class Image(DisplayObject): format = self._FMT_PNG elif ext == u'gif': format = self._FMT_GIF + elif ext == "webp": + format = self._FMT_WEBP else: format = ext.lower() elif isinstance(data, bytes): @@ -949,6 +1019,12 @@ class Image(DisplayObject): # only if format has not been specified. if data[:2] == _JPEG: format = self._FMT_JPEG + elif data[:8] == _PNG: + format = self._FMT_PNG + elif data[8:12] == _WEBP: + format = self._FMT_WEBP + elif data[:6] == _GIF1 or data[:6] == _GIF2: + format = self._FMT_GIF # failed to detect format, default png if format is None: diff --git a/contrib/python/ipython/py3/IPython/core/guarded_eval.py b/contrib/python/ipython/py3/IPython/core/guarded_eval.py index d8ac9928af4..39fe853f589 100644 --- a/contrib/python/ipython/py3/IPython/core/guarded_eval.py +++ b/contrib/python/ipython/py3/IPython/core/guarded_eval.py @@ -132,7 +132,7 @@ def _get_external(module_name: str, access_path: Sequence[str]): Raises: * `KeyError` if module is removed not found, and - * `AttributeError` if acess path does not match an exported object + * `AttributeError` if access path does not match an exported object """ member_type = sys.modules[module_name] for attr in access_path: @@ -235,7 +235,7 @@ class SelectivePolicy(EvaluationPolicy): accept = has_original_attr and has_original_attribute if accept: - # We still need to check for overriden properties. + # We still need to check for overridden properties. value_class = type(value) if not hasattr(value_class, attr): @@ -332,7 +332,7 @@ class EvaluationContext(NamedTuple): evaluation: Literal[ "forbidden", "minimal", "limited", "unsafe", "dangerous" ] = "forbidden" - #: Whether the evalution of code takes place inside of a subscript. + #: Whether the evaluation of code takes place inside of a subscript. #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. in_subscript: bool = False @@ -373,7 +373,7 @@ def guarded_eval(code: str, context: EvaluationContext): # getitem at all, for example it fails on simple `[0][1]` if context.in_subscript: - # syntatic sugar for ellipsis (:) is only available in susbcripts + # syntactic sugar for ellipsis (:) is only available in subscripts # so we need to trick the ast parser into thinking that we have # a subscript, but we need to be able to later recognise that we did # it so we can ignore the actual __getitem__ operation diff --git a/contrib/python/ipython/py3/IPython/core/inputsplitter.py b/contrib/python/ipython/py3/IPython/core/inputsplitter.py index af7a12e6e02..092f21408a4 100644 --- a/contrib/python/ipython/py3/IPython/core/inputsplitter.py +++ b/contrib/python/ipython/py3/IPython/core/inputsplitter.py @@ -97,7 +97,7 @@ def num_ini_spaces(s): """ warnings.warn( "`num_ini_spaces` is Pending Deprecation since IPython 8.17." - "It is considered fro removal in in future version. " + "It is considered for removal in in future version. " "Please open an issue if you believe it should be kept.", stacklevel=2, category=PendingDeprecationWarning, diff --git a/contrib/python/ipython/py3/IPython/core/interactiveshell.py b/contrib/python/ipython/py3/IPython/core/interactiveshell.py index d05cb451f83..07fb8077601 100644 --- a/contrib/python/ipython/py3/IPython/core/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/core/interactiveshell.py @@ -1627,7 +1627,7 @@ class InteractiveShell(SingletonConfigurable): Returns ------- parts_ok: bool - wether we were properly able to parse parts. + whether we were properly able to parse parts. parts: list of str extracted parts diff --git a/contrib/python/ipython/py3/IPython/core/magics/ast_mod.py b/contrib/python/ipython/py3/IPython/core/magics/ast_mod.py index fa547914437..dc3c5bc7683 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/ast_mod.py +++ b/contrib/python/ipython/py3/IPython/core/magics/ast_mod.py @@ -5,7 +5,7 @@ with ast-transformers it is not easy to directly manipulate ast. IPython has pre-code and post-code hooks, but are ran from within the IPython -machinery so may be inappropriate, for example for performance mesurement. +machinery so may be inappropriate, for example for performance measurement. This module give you tools to simplify this, and expose 2 classes: diff --git a/contrib/python/ipython/py3/IPython/core/magics/code.py b/contrib/python/ipython/py3/IPython/core/magics/code.py index 4f1574dcefd..834ca514736 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/code.py +++ b/contrib/python/ipython/py3/IPython/core/magics/code.py @@ -153,7 +153,9 @@ def strip_initial_indent(lines): for line in it: if line.startswith(indent): - yield line[len(indent):] + yield line[len(indent) :] + elif line in ("\n", "\r\n") or len(line) == 0: + yield line else: # Less indented than the first line - stop dedenting yield line diff --git a/contrib/python/ipython/py3/IPython/core/magics/execution.py b/contrib/python/ipython/py3/IPython/core/magics/execution.py index abfc4cbda76..3aa0a27fc27 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/execution.py +++ b/contrib/python/ipython/py3/IPython/core/magics/execution.py @@ -195,12 +195,16 @@ class ExecutionMagics(Magics): """Run a statement through the python code profiler. - Usage, in line mode: + **Usage, in line mode:** + %prun [options] statement - Usage, in cell mode: + **Usage, in cell mode:** + %%prun [options] [statement] + code... + code... In cell mode, the additional code lines are appended to the (possibly @@ -1028,11 +1032,16 @@ class ExecutionMagics(Magics): def timeit(self, line='', cell=None, local_ns=None): """Time execution of a Python statement or expression - Usage, in line mode: + **Usage, in line mode:** + %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement - or in cell mode: + + **or in cell mode:** + %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code + code + code... Time execution of a Python statement or expression using the timeit @@ -1046,6 +1055,7 @@ class ExecutionMagics(Magics): body has access to any variables created in the setup code. Options: + -n<N>: execute the given statement <N> times in a loop. If <N> is not provided, <N> is determined so as to get sufficient accuracy. @@ -1066,7 +1076,7 @@ class ExecutionMagics(Magics): -q: Quiet, do not print result. -o: return a TimeitResult that can be stored in a variable to inspect - the result in more details. + the result in more details. .. versionchanged:: 7.3 User variables are no longer expanded, diff --git a/contrib/python/ipython/py3/IPython/core/magics/packaging.py b/contrib/python/ipython/py3/IPython/core/magics/packaging.py index 09d4117270f..ed1c1274f3d 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/packaging.py +++ b/contrib/python/ipython/py3/IPython/core/magics/packaging.py @@ -162,3 +162,20 @@ class PackagingMagics(Magics): """ micromamba = _get_conda_like_executable("micromamba") self._run_command(micromamba, line) + + @line_magic + def uv(self, line): + """Run the uv package manager within the current kernel. + + Usage: + %uv pip install [pkgs] + """ + python = sys.executable + if sys.platform == "win32": + python = '"' + python + '"' + else: + python = shlex.quote(python) + + self.shell.system(" ".join([python, "-m", "uv", line])) + + print("Note: you may need to restart the kernel to use updated packages.") diff --git a/contrib/python/ipython/py3/IPython/core/oinspect.py b/contrib/python/ipython/py3/IPython/core/oinspect.py index 8aa8429e5d4..a4fdc28bde4 100644 --- a/contrib/python/ipython/py3/IPython/core/oinspect.py +++ b/contrib/python/ipython/py3/IPython/core/oinspect.py @@ -460,7 +460,7 @@ class Inspector(Colorable): mime_hooks = traitlets.Dict( config=True, - help="dictionary of mime to callable to add informations into help mimebundle dict", + help="dictionary of mime to callable to add information into help mimebundle dict", ).tag(config=True) def __init__( diff --git a/contrib/python/ipython/py3/IPython/core/page.py b/contrib/python/ipython/py3/IPython/core/page.py index 31b314ec460..2eb6c399b36 100644 --- a/contrib/python/ipython/py3/IPython/core/page.py +++ b/contrib/python/ipython/py3/IPython/core/page.py @@ -125,7 +125,7 @@ def _detect_screen_size(screen_lines_def): # print('***Screen size:',screen_lines_real,'lines x', # screen_cols,'columns.') # dbg -def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): +def pager_page(strng, start=0, screen_lines=0, pager_cmd=None) -> None: """Display a string, piping through a pager after a certain length. strng can be a mime-bundle dict, supplying multiple representations, @@ -239,7 +239,7 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): page_dumb(strng,screen_lines=screen_lines) -def page(data, start=0, screen_lines=0, pager_cmd=None): +def page(data, start: int = 0, screen_lines: int = 0, pager_cmd=None): """Display content in a pager, piping through a pager after a certain length. data can be a mime-bundle dict, supplying multiple representations, diff --git a/contrib/python/ipython/py3/IPython/core/prefilter.py b/contrib/python/ipython/py3/IPython/core/prefilter.py index a29df0c27ad..fc7b1c34161 100644 --- a/contrib/python/ipython/py3/IPython/core/prefilter.py +++ b/contrib/python/ipython/py3/IPython/core/prefilter.py @@ -512,8 +512,10 @@ class AutocallChecker(PrefilterChecker): callable(oinfo.obj) and (not self.exclude_regexp.match(line_info.the_rest)) and self.function_name_regexp.match(line_info.ifun) - and line_info.raw_the_rest.startswith(" ") - or not line_info.raw_the_rest.strip() + and ( + line_info.raw_the_rest.startswith(" ") + or not line_info.raw_the_rest.strip() + ) ): return self.prefilter_manager.get_handler_by_name("auto") else: diff --git a/contrib/python/ipython/py3/IPython/core/pylabtools.py b/contrib/python/ipython/py3/IPython/core/pylabtools.py index 5c926a9c106..ad2fc74e9ee 100644 --- a/contrib/python/ipython/py3/IPython/core/pylabtools.py +++ b/contrib/python/ipython/py3/IPython/core/pylabtools.py @@ -528,7 +528,7 @@ def _list_matplotlib_backends_and_gui_loops() -> list[str]: # Matplotlib and IPython do not always use the same gui framework name. -# Always use the approprate one of these conversion functions when passing a +# Always use the appropriate one of these conversion functions when passing a # gui framework name to/from Matplotlib. def _convert_gui_to_matplotlib(gui: str | None) -> str | None: if gui and gui.lower() == "osx": diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py index fb5a54da6ab..d9eb28aa955 100644 --- a/contrib/python/ipython/py3/IPython/core/release.py +++ b/contrib/python/ipython/py3/IPython/core/release.py @@ -16,7 +16,7 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 28 +_version_minor = 29 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" @@ -32,22 +32,13 @@ if _version_extra: version = __version__ # backwards compatibility name version_info = (_version_major, _version_minor, _version_patch, _version_extra) -# Change this when incrementing the kernel protocol version -kernel_protocol_version_info = (5, 0) -kernel_protocol_version = "%i.%i" % kernel_protocol_version_info license = "BSD-3-Clause" -authors = {'Fernando' : ('Fernando Perez','[email protected]'), - 'Janko' : ('Janko Hauser','[email protected]'), - 'Nathan' : ('Nathaniel Gray','[email protected]'), - 'Ville' : ('Ville Vainio','[email protected]'), - 'Brian' : ('Brian E Granger', '[email protected]'), - 'Min' : ('Min Ragan-Kelley', '[email protected]'), - 'Thomas' : ('Thomas A. Kluyver', '[email protected]'), - 'Jorgen' : ('Jorgen Stenarson', '[email protected]'), - 'Matthias' : ('Matthias Bussonnier', '[email protected]'), - } +authors = { + "Fernando": ("Fernando Perez", "[email protected]"), + "M": ("M Bussonnier", "[email protected]"), +} author = 'The IPython Development Team' diff --git a/contrib/python/ipython/py3/IPython/lib/pretty.py b/contrib/python/ipython/py3/IPython/lib/pretty.py index 8a24632d600..a232e4edf6f 100644 --- a/contrib/python/ipython/py3/IPython/lib/pretty.py +++ b/contrib/python/ipython/py3/IPython/lib/pretty.py @@ -541,7 +541,7 @@ class RawText: class CallExpression: """ Object which emits a line-wrapped call expression in the form `__name(*args, **kwargs)` """ def __init__(__self, __name, *args, **kwargs): - # dunders are to avoid clashes with kwargs, as python's name manging + # dunders are to avoid clashes with kwargs, as python's name managing # will kick in. self = __self self.name = __name @@ -555,7 +555,7 @@ class CallExpression: return inner def _repr_pretty_(self, p, cycle): - # dunders are to avoid clashes with kwargs, as python's name manging + # dunders are to avoid clashes with kwargs, as python's name managing # will kick in. started = False @@ -724,8 +724,15 @@ class _ReFlags: def _repr_pretty_(self, p, cycle): done_one = False - for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL', - 'UNICODE', 'VERBOSE', 'DEBUG'): + for flag in ( + "IGNORECASE", + "LOCALE", + "MULTILINE", + "DOTALL", + "UNICODE", + "VERBOSE", + "DEBUG", + ): if self.value & getattr(re, flag): if done_one: p.text('|') diff --git a/contrib/python/ipython/py3/IPython/terminal/shortcuts/auto_suggest.py b/contrib/python/ipython/py3/IPython/terminal/shortcuts/auto_suggest.py index 65f91577ce9..94a94a88c1e 100644 --- a/contrib/python/ipython/py3/IPython/terminal/shortcuts/auto_suggest.py +++ b/contrib/python/ipython/py3/IPython/terminal/shortcuts/auto_suggest.py @@ -77,7 +77,7 @@ class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory): def connect(self, pt_app: PromptSession): self._connected_apps.append(pt_app) # note: `on_text_changed` could be used for a bit different behaviour - # on character deletion (i.e. reseting history position on backspace) + # on character deletion (i.e. resetting history position on backspace) pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position) pt_app.default_buffer.on_cursor_position_changed.add_handler(self._dismiss) diff --git a/contrib/python/ipython/py3/IPython/terminal/shortcuts/filters.py b/contrib/python/ipython/py3/IPython/terminal/shortcuts/filters.py index 7c9d6a9c41d..8e7c8d037c9 100644 --- a/contrib/python/ipython/py3/IPython/terminal/shortcuts/filters.py +++ b/contrib/python/ipython/py3/IPython/terminal/shortcuts/filters.py @@ -207,7 +207,7 @@ class PassThrough(Filter): pass_through = PassThrough() # these one is callable and re-used multiple times hence needs to be -# only defined once beforhand so that transforming back to human-readable +# only defined once beforehand so that transforming back to human-readable # names works well in the documentation. default_buffer_focused = has_focus(DEFAULT_BUFFER) diff --git a/contrib/python/ipython/py3/IPython/testing/tools.py b/contrib/python/ipython/py3/IPython/testing/tools.py index b0303490482..aa54443ad06 100644 --- a/contrib/python/ipython/py3/IPython/testing/tools.py +++ b/contrib/python/ipython/py3/IPython/testing/tools.py @@ -36,7 +36,7 @@ from . import skipdoctest doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco @doctest_deco -def full_path(startPath,files): +def full_path(startPath: str, files: list[str]) -> list[str]: """Make full paths for all the listed files, based on startPath. Only the base part of startPath is kept, since this routine is typically @@ -49,7 +49,7 @@ def full_path(startPath,files): Initial path to use as the base for the results. This path is split using os.path.split() and only its first component is kept. - files : string or list + files : list One or more files. Examples @@ -61,13 +61,8 @@ def full_path(startPath,files): >>> full_path('/foo',['a.txt','b.txt']) ['/a.txt', '/b.txt'] - If a single file is given, the output is still a list:: - - >>> full_path('/foo','a.txt') - ['/a.txt'] """ - - files = list_strings(files) + assert isinstance(files, list) base = os.path.split(startPath)[0] return [ os.path.join(base,f) for f in files ] diff --git a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py index 304813b0f5a..261309bb28d 100644 --- a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py +++ b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py @@ -1,2 +1,2 @@ # GENERATED BY setup.py -commit = "a9c7369d7" +commit = "af19fb054" diff --git a/contrib/python/ipython/py3/IPython/utils/frame.py b/contrib/python/ipython/py3/IPython/utils/frame.py index 808906bda81..3d0c1b71897 100644 --- a/contrib/python/ipython/py3/IPython/utils/frame.py +++ b/contrib/python/ipython/py3/IPython/utils/frame.py @@ -15,6 +15,7 @@ Utilities for working with stack frames. #----------------------------------------------------------------------------- import sys +from typing import Any #----------------------------------------------------------------------------- # Code @@ -51,7 +52,7 @@ def extract_vars(*names,**kw): return dict((k,callerNS[k]) for k in names) -def extract_vars_above(*names): +def extract_vars_above(*names: list[str]): """Extract a set of variables by name from another frame. Similar to extractVars(), but with a specified depth of 1, so that names @@ -65,7 +66,7 @@ def extract_vars_above(*names): return dict((k,callerNS[k]) for k in names) -def debugx(expr,pre_msg=''): +def debugx(expr: str, pre_msg: str = ""): """Print the value of an expression from the caller's frame. Takes an expression, evaluates it in the caller's frame and prints both @@ -84,7 +85,8 @@ def debugx(expr,pre_msg=''): # deactivate it by uncommenting the following line, which makes it a no-op #def debugx(expr,pre_msg=''): pass -def extract_module_locals(depth=0): + +def extract_module_locals(depth: int = 0) -> tuple[Any, Any]: """Returns (module, locals) of the function `depth` frames away from the caller""" f = sys._getframe(depth + 1) global_ns = f.f_globals diff --git a/contrib/python/ipython/py3/IPython/utils/text.py b/contrib/python/ipython/py3/IPython/utils/text.py index 46b3bb0e46f..8f1d380fc5e 100644 --- a/contrib/python/ipython/py3/IPython/utils/text.py +++ b/contrib/python/ipython/py3/IPython/utils/text.py @@ -16,7 +16,20 @@ import warnings from string import Formatter from pathlib import Path -from typing import List, Dict, Tuple, Optional, cast, Sequence, Mapping, Any +from typing import ( + List, + Dict, + Tuple, + Optional, + cast, + Sequence, + Mapping, + Any, + Union, + Callable, + Iterator, + TypeVar, +) if sys.version_info < (3, 12): from typing_extensions import Self @@ -138,8 +151,13 @@ class SList(list): p = paths = property(get_paths) - def grep(self, pattern, prune = False, field = None): - """ Return all strings matching 'pattern' (a regex or callable) + def grep( + self, + pattern: Union[str, Callable[[Any], re.Match[str] | None]], + prune: bool = False, + field: Optional[int] = None, + ) -> Self: + """Return all strings matching 'pattern' (a regex or callable) This is case-insensitive. If prune is true, return all items NOT matching the pattern. @@ -154,7 +172,7 @@ class SList(list): a.grep('chm', field=-1) """ - def match_target(s): + def match_target(s: str) -> str: if field is None: return s parts = s.split() @@ -169,12 +187,12 @@ class SList(list): else: pred = pattern if not prune: - return SList([el for el in self if pred(match_target(el))]) + return type(self)([el for el in self if pred(match_target(el))]) else: - return SList([el for el in self if not pred(match_target(el))]) + return type(self)([el for el in self if not pred(match_target(el))]) - def fields(self, *fields): - """ Collect whitespace-separated fields from string list + def fields(self, *fields: List[str]) -> List[List[str]]: + """Collect whitespace-separated fields from string list Allows quick awk-like usage of string lists. @@ -209,8 +227,12 @@ class SList(list): return res - def sort(self,field= None, nums = False): - """ sort by specified fields (see fields()) + def sort( # type:ignore[override] + self, + field: Optional[List[str]] = None, + nums: bool = False, + ) -> Self: + """sort by specified fields (see fields()) Example:: @@ -236,7 +258,7 @@ class SList(list): dsu.sort() - return SList([t[1] for t in dsu]) + return type(self)([t[1] for t in dsu]) # FIXME: We need to reimplement type specific displayhook and then add this @@ -255,7 +277,7 @@ class SList(list): # print_slist = result_display.register(SList)(print_slist) -def indent(instr,nspaces=4, ntabs=0, flatten=False): +def indent(instr: str, nspaces: int = 4, ntabs: int = 0, flatten: bool = False) -> str: """Indent a string a given number of spaces or tabstops. indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces. @@ -275,7 +297,7 @@ def indent(instr,nspaces=4, ntabs=0, flatten=False): Returns ------- - str|unicode : string indented by ntabs and nspaces. + str : string indented by ntabs and nspaces. """ if instr is None: @@ -292,7 +314,7 @@ def indent(instr,nspaces=4, ntabs=0, flatten=False): return outstr -def list_strings(arg): +def list_strings(arg: Union[str, List[str]]) -> List[str]: """Always return a list of strings, given a string or list of strings as input. @@ -316,7 +338,7 @@ def list_strings(arg): return arg -def marquee(txt='',width=78,mark='*'): +def marquee(txt: str = "", width: int = 78, mark: str = "*") -> str: """Return the input string centered in a 'marquee'. Examples @@ -343,11 +365,12 @@ def marquee(txt='',width=78,mark='*'): ini_spaces_re = re.compile(r'^(\s+)') -def num_ini_spaces(strng): + +def num_ini_spaces(strng: str) -> int: """Return the number of initial spaces in a string""" warnings.warn( "`num_ini_spaces` is Pending Deprecation since IPython 8.17." - "It is considered fro removal in in future version. " + "It is considered for removal in in future version. " "Please open an issue if you believe it should be kept.", stacklevel=2, category=PendingDeprecationWarning, @@ -359,7 +382,7 @@ def num_ini_spaces(strng): return 0 -def format_screen(strng): +def format_screen(strng: str) -> str: """Format a string for screen printing. This removes some latex-type format codes.""" @@ -396,7 +419,7 @@ def dedent(text: str) -> str: return '\n'.join([first, rest]) -def wrap_paragraphs(text, ncols=80): +def wrap_paragraphs(text: str, ncols: int = 80) -> List[str]: """Wrap multiple paragraphs to fit a specified width. This is equivalent to textwrap.wrap, but with support for multiple @@ -408,7 +431,7 @@ def wrap_paragraphs(text, ncols=80): """ warnings.warn( "`wrap_paragraphs` is Pending Deprecation since IPython 8.17." - "It is considered fro removal in in future version. " + "It is considered for removal in in future version. " "Please open an issue if you believe it should be kept.", stacklevel=2, category=PendingDeprecationWarning, @@ -428,7 +451,7 @@ def wrap_paragraphs(text, ncols=80): return out_ps -def strip_email_quotes(text): +def strip_email_quotes(text: str) -> str: """Strip leading email quotation characters ('>'). Removes any combination of leading '>' interspersed with whitespace that @@ -478,7 +501,7 @@ def strip_email_quotes(text): return text -def strip_ansi(source): +def strip_ansi(source: str) -> str: """ Remove ansi escape codes from text. @@ -489,7 +512,7 @@ def strip_ansi(source): """ warnings.warn( "`strip_ansi` is Pending Deprecation since IPython 8.17." - "It is considered fro removal in in future version. " + "It is considered for removal in in future version. " "Please open an issue if you believe it should be kept.", stacklevel=2, category=PendingDeprecationWarning, @@ -519,7 +542,8 @@ class EvalFormatter(Formatter): In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello") Out[3]: 'll' """ - def get_field(self, name, args, kwargs): + + def get_field(self, name: str, args: Any, kwargs: Any) -> Tuple[Any, str]: v = eval(name, kwargs) return v, name @@ -606,11 +630,15 @@ class DollarFormatter(FullEvalFormatter): In [4]: f.format('$a or {b}', a=1, b=2) Out[4]: '1 or 2' """ - _dollar_pattern_ignore_single_quote = re.compile(r"(.*?)\$(\$?[\w\.]+)(?=([^']*'[^']*')*[^']*$)") - def parse(self, fmt_string): - for literal_txt, field_name, format_spec, conversion \ - in Formatter.parse(self, fmt_string): - + + _dollar_pattern_ignore_single_quote = re.compile( + r"(.*?)\$(\$?[\w\.]+)(?=([^']*'[^']*')*[^']*$)" + ) + + def parse(self, fmt_string: str) -> Iterator[Tuple[Any, Any, Any, Any]]: # type: ignore + for literal_txt, field_name, format_spec, conversion in Formatter.parse( + self, fmt_string + ): # Find $foo patterns in the literal text. continue_from = 0 txt = "" @@ -627,14 +655,17 @@ class DollarFormatter(FullEvalFormatter): # Re-yield the {foo} style pattern yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion) - def __repr__(self): + def __repr__(self) -> str: return "<DollarFormatter>" #----------------------------------------------------------------------------- # Utils to columnize a list of string #----------------------------------------------------------------------------- -def _col_chunks(l, max_rows, row_first=False): + +def _col_chunks( + l: List[int], max_rows: int, row_first: bool = False +) -> Iterator[List[int]]: """Yield successive max_rows-sized column chunks from l.""" if row_first: ncols = (len(l) // max_rows) + (len(l) % max_rows > 0) @@ -646,7 +677,7 @@ def _col_chunks(l, max_rows, row_first=False): def _find_optimal( - rlist: List[str], row_first: bool, separator_size: int, displaywidth: int + rlist: List[int], row_first: bool, separator_size: int, displaywidth: int ) -> Dict[str, Any]: """Calculate optimal info to columnize a list of string""" for max_rows in range(1, len(rlist) + 1): @@ -662,7 +693,10 @@ def _find_optimal( } -def _get_or_default(mylist, i, default=None): +T = TypeVar("T") + + +def _get_or_default(mylist: List[T], i: int, default: T) -> T: """return list item number, or default if don't exist""" if i >= len(mylist): return default @@ -727,7 +761,7 @@ def compute_item_matrix( """ warnings.warn( "`compute_item_matrix` is Pending Deprecation since IPython 8.17." - "It is considered fro removal in in future version. " + "It is considered for removal in in future version. " "Please open an issue if you believe it should be kept.", stacklevel=2, category=PendingDeprecationWarning, @@ -740,9 +774,31 @@ def compute_item_matrix( ) nrow, ncol = info["max_rows"], info["num_columns"] if row_first: - return ([[_get_or_default(items, r * ncol + c, default=empty) for c in range(ncol)] for r in range(nrow)], info) + return ( + [ + [ + _get_or_default( + items, r * ncol + c, default=empty + ) # type:ignore[misc] + for c in range(ncol) + ] + for r in range(nrow) + ], + info, + ) else: - return ([[_get_or_default(items, c * nrow + r, default=empty) for c in range(ncol)] for r in range(nrow)], info) + return ( + [ + [ + _get_or_default( + items, c * nrow + r, default=empty + ) # type:ignore[misc] + for c in range(ncol) + ] + for r in range(nrow) + ], + info, + ) def columnize( @@ -795,7 +851,9 @@ def columnize( return "\n".join(map(sjoin, fmatrix)) + "\n" -def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""): +def get_text_list( + list_: List[str], last_sep: str = " and ", sep: str = ", ", wrap_item_with: str = "" +) -> str: """ Return a string with a natural enumeration of items diff --git a/contrib/python/ipython/py3/ya.make b/contrib/python/ipython/py3/ya.make index b13b8d1baa8..71ae1e14b1a 100644 --- a/contrib/python/ipython/py3/ya.make +++ b/contrib/python/ipython/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.28.0) +VERSION(8.29.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/pip/.dist-info/METADATA b/contrib/python/pip/.dist-info/METADATA index 6141107f90b..9e5aa3a486f 100644 --- a/contrib/python/pip/.dist-info/METADATA +++ b/contrib/python/pip/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pip -Version: 24.2 +Version: 24.3.1 Summary: The PyPA recommended tool for installing Python packages. Author-email: The pip developers <[email protected]> License: MIT @@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Requires-Python: >=3.8 diff --git a/contrib/python/pip/AUTHORS.txt b/contrib/python/pip/AUTHORS.txt index dda2ac30f85..8ccefbc6e59 100644 --- a/contrib/python/pip/AUTHORS.txt +++ b/contrib/python/pip/AUTHORS.txt @@ -57,6 +57,7 @@ Anthony Sottile Antoine Musso Anton Ovchinnikov Anton Patrushev +Anton Zelenov Antonio Alvarado Hernandez Antony Lee Antti Kaihola @@ -225,6 +226,7 @@ Diego Ramirez DiegoCaraballo Dimitri Merejkowsky Dimitri Papadopoulos +Dimitri Papadopoulos Orfanos Dirk Stolle Dmitry Gladkov Dmitry Volodin @@ -690,6 +692,7 @@ snook92 socketubs Sorin Sbarnea Srinivas Nyayapati +Srishti Hegde Stavros Korokithakis Stefan Scherfke Stefano Rivera diff --git a/contrib/python/pip/patches/01-arcadia.patch b/contrib/python/pip/patches/01-arcadia.patch deleted file mode 100644 index 3bf8efa83cb..00000000000 --- a/contrib/python/pip/patches/01-arcadia.patch +++ /dev/null @@ -1,13 +0,0 @@ -DEVTOOLSSUPPORT-48054 ---- contrib/python/pip/pip/_vendor/distlib/scripts.py (index) -+++ contrib/python/pip/pip/_vendor/distlib/scripts.py (working tree) -@@ -62,9 +62,6 @@ if __name__ == '__main__': - distlib_package = __name__.rsplit('.', 1)[0] - - WRAPPERS = { -- r.name: r.bytes -- for r in finder(distlib_package).iterator("") -- if r.name.endswith(".exe") - } - - diff --git a/contrib/python/pip/pip/__init__.py b/contrib/python/pip/pip/__init__.py index 640e922f537..efefccffc7a 100644 --- a/contrib/python/pip/pip/__init__.py +++ b/contrib/python/pip/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "24.2" +__version__ = "24.3.1" def main(args: Optional[List[str]] = None) -> int: diff --git a/contrib/python/pip/pip/_internal/build_env.py b/contrib/python/pip/pip/_internal/build_env.py index be1e0ca85d2..0f1e2667caf 100644 --- a/contrib/python/pip/pip/_internal/build_env.py +++ b/contrib/python/pip/pip/_internal/build_env.py @@ -242,6 +242,10 @@ class BuildEnvironment: prefix.path, "--no-warn-script-location", "--disable-pip-version-check", + # The prefix specified two lines above, thus + # target from config file or env var should be ignored + "--target", + "", ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append("-vv") diff --git a/contrib/python/pip/pip/_internal/cli/index_command.py b/contrib/python/pip/pip/_internal/cli/index_command.py index 226f8da1e94..db105d0fef9 100644 --- a/contrib/python/pip/pip/_internal/cli/index_command.py +++ b/contrib/python/pip/pip/_internal/cli/index_command.py @@ -54,7 +54,7 @@ class SessionCommandMixin(CommandContextMixIn): def __init__(self) -> None: super().__init__() - self._session: Optional["PipSession"] = None + self._session: Optional[PipSession] = None @classmethod def _get_index_urls(cls, options: Values) -> Optional[List[str]]: diff --git a/contrib/python/pip/pip/_internal/cli/parser.py b/contrib/python/pip/pip/_internal/cli/parser.py index b7d7c1f600a..bc4aca032d4 100644 --- a/contrib/python/pip/pip/_internal/cli/parser.py +++ b/contrib/python/pip/pip/_internal/cli/parser.py @@ -6,7 +6,7 @@ import shutil import sys import textwrap from contextlib import suppress -from typing import Any, Dict, Generator, List, Optional, Tuple +from typing import Any, Dict, Generator, List, NoReturn, Optional, Tuple from pip._internal.cli.status_codes import UNKNOWN_ERROR from pip._internal.configuration import Configuration, ConfigurationError @@ -289,6 +289,6 @@ class ConfigOptionParser(CustomOptionParser): defaults[option.dest] = option.check_value(opt_str, default) return optparse.Values(defaults) - def error(self, msg: str) -> None: + def error(self, msg: str) -> NoReturn: self.print_usage(sys.stderr) self.exit(UNKNOWN_ERROR, f"{msg}\n") diff --git a/contrib/python/pip/pip/_internal/cli/progress_bars.py b/contrib/python/pip/pip/_internal/cli/progress_bars.py index 883359c9ce7..1236180c086 100644 --- a/contrib/python/pip/pip/_internal/cli/progress_bars.py +++ b/contrib/python/pip/pip/_internal/cli/progress_bars.py @@ -25,7 +25,7 @@ def _rich_progress_bar( iterable: Iterable[bytes], *, bar_type: str, - size: int, + size: Optional[int], ) -> Generator[bytes, None, None]: assert bar_type == "on", "This should only be used in the default mode." diff --git a/contrib/python/pip/pip/_internal/commands/list.py b/contrib/python/pip/pip/_internal/commands/list.py index 82fc46a118f..84943702410 100644 --- a/contrib/python/pip/pip/_internal/commands/list.py +++ b/contrib/python/pip/pip/_internal/commands/list.py @@ -176,7 +176,7 @@ class ListCommand(IndexGroupCommand): if options.excludes: skip.update(canonicalize_name(n) for n in options.excludes) - packages: "_ProcessedDists" = [ + packages: _ProcessedDists = [ cast("_DistWithLatestInfo", d) for d in get_environment(options.path).iter_installed_distributions( local_only=options.local, diff --git a/contrib/python/pip/pip/_internal/commands/search.py b/contrib/python/pip/pip/_internal/commands/search.py index e0d329d58ad..74b8d656b47 100644 --- a/contrib/python/pip/pip/_internal/commands/search.py +++ b/contrib/python/pip/pip/_internal/commands/search.py @@ -89,7 +89,7 @@ def transform_hits(hits: List[Dict[str, str]]) -> List["TransformedHit"]: packages with the list of versions stored inline. This converts the list from pypi into one we can use. """ - packages: Dict[str, "TransformedHit"] = OrderedDict() + packages: Dict[str, TransformedHit] = OrderedDict() for hit in hits: name = hit["name"] summary = hit["summary"] diff --git a/contrib/python/pip/pip/_internal/exceptions.py b/contrib/python/pip/pip/_internal/exceptions.py index 2587740f73a..45a876a850d 100644 --- a/contrib/python/pip/pip/_internal/exceptions.py +++ b/contrib/python/pip/pip/_internal/exceptions.py @@ -15,6 +15,8 @@ import sys from itertools import chain, groupby, repeat from typing import TYPE_CHECKING, Dict, Iterator, List, Literal, Optional, Union +from pip._vendor.packaging.requirements import InvalidRequirement +from pip._vendor.packaging.version import InvalidVersion from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult from pip._vendor.rich.markup import escape from pip._vendor.rich.text import Text @@ -429,7 +431,7 @@ class HashErrors(InstallationError): """Multiple HashError instances rolled into one for reporting""" def __init__(self) -> None: - self.errors: List["HashError"] = [] + self.errors: List[HashError] = [] def append(self, error: "HashError") -> None: self.errors.append(error) @@ -775,3 +777,33 @@ class LegacyDistutilsInstall(DiagnosticPipError): ), hint_stmt=None, ) + + +class InvalidInstalledPackage(DiagnosticPipError): + reference = "invalid-installed-package" + + def __init__( + self, + *, + dist: "BaseDistribution", + invalid_exc: Union[InvalidRequirement, InvalidVersion], + ) -> None: + installed_location = dist.installed_location + + if isinstance(invalid_exc, InvalidRequirement): + invalid_type = "requirement" + else: + invalid_type = "version" + + super().__init__( + message=Text( + f"Cannot process installed package {dist} " + + (f"in {installed_location!r} " if installed_location else "") + + f"because it has an invalid {invalid_type}:\n{invalid_exc.args[0]}" + ), + context=( + "Starting with pip 24.1, packages with invalid " + f"{invalid_type}s can not be processed." + ), + hint_stmt="To proceed this package must be uninstalled.", + ) diff --git a/contrib/python/pip/pip/_internal/index/sources.py b/contrib/python/pip/pip/_internal/index/sources.py index f4626d71ab4..3dafb30e6eb 100644 --- a/contrib/python/pip/pip/_internal/index/sources.py +++ b/contrib/python/pip/pip/_internal/index/sources.py @@ -6,7 +6,6 @@ from typing import Callable, Dict, Iterable, List, Optional, Tuple from pip._vendor.packaging.utils import ( InvalidSdistFilename, - InvalidVersion, InvalidWheelFilename, canonicalize_name, parse_sdist_filename, @@ -68,10 +67,10 @@ class _FlatDirectoryToUrls: # otherwise not worth considering as a package try: project_filename = parse_wheel_filename(entry.name)[0] - except (InvalidWheelFilename, InvalidVersion): + except InvalidWheelFilename: try: project_filename = parse_sdist_filename(entry.name)[0] - except (InvalidSdistFilename, InvalidVersion): + except InvalidSdistFilename: continue self._project_name_to_urls[project_filename].append(url) diff --git a/contrib/python/pip/pip/_internal/locations/_distutils.py b/contrib/python/pip/pip/_internal/locations/_distutils.py index 0e18c6e1e14..3d856256986 100644 --- a/contrib/python/pip/pip/_internal/locations/_distutils.py +++ b/contrib/python/pip/pip/_internal/locations/_distutils.py @@ -21,7 +21,7 @@ from distutils.cmd import Command as DistutilsCommand from distutils.command.install import SCHEME_KEYS from distutils.command.install import install as distutils_install_command from distutils.sysconfig import get_python_lib -from typing import Dict, List, Optional, Union, cast +from typing import Dict, List, Optional, Union from pip._internal.models.scheme import Scheme from pip._internal.utils.compat import WINDOWS @@ -64,7 +64,7 @@ def distutils_scheme( obj: Optional[DistutilsCommand] = None obj = d.get_command_obj("install", create=True) assert obj is not None - i = cast(distutils_install_command, obj) + i: distutils_install_command = obj # NOTE: setting user or home has the side-effect of creating the home dir # or user base for installations during finalize_options() # ideally, we'd prefer a scheme class that has no side-effects. @@ -78,7 +78,7 @@ def distutils_scheme( i.root = root or i.root i.finalize_options() - scheme = {} + scheme: Dict[str, str] = {} for key in SCHEME_KEYS: scheme[key] = getattr(i, "install_" + key) diff --git a/contrib/python/pip/pip/_internal/metadata/importlib/_envs.py b/contrib/python/pip/pip/_internal/metadata/importlib/_envs.py index 70cb7a6009a..4d906fd3149 100644 --- a/contrib/python/pip/pip/_internal/metadata/importlib/_envs.py +++ b/contrib/python/pip/pip/_internal/metadata/importlib/_envs.py @@ -150,7 +150,7 @@ def _emit_egg_deprecation(location: Optional[str]) -> None: deprecated( reason=f"Loading egg at {location} is deprecated.", replacement="to use pip for package installation", - gone_in="24.3", + gone_in="25.1", issue=12330, ) diff --git a/contrib/python/pip/pip/_internal/models/wheel.py b/contrib/python/pip/pip/_internal/models/wheel.py index 36d4d2e785c..ea8560089d3 100644 --- a/contrib/python/pip/pip/_internal/models/wheel.py +++ b/contrib/python/pip/pip/_internal/models/wheel.py @@ -6,8 +6,13 @@ import re from typing import Dict, Iterable, List from pip._vendor.packaging.tags import Tag +from pip._vendor.packaging.utils import ( + InvalidWheelFilename as PackagingInvalidWheelName, +) +from pip._vendor.packaging.utils import parse_wheel_filename from pip._internal.exceptions import InvalidWheelFilename +from pip._internal.utils.deprecation import deprecated class Wheel: @@ -29,9 +34,29 @@ class Wheel: raise InvalidWheelFilename(f"{filename} is not a valid wheel filename.") self.filename = filename self.name = wheel_info.group("name").replace("_", "-") - # we'll assume "_" means "-" due to wheel naming scheme - # (https://github.com/pypa/pip/issues/1150) - self.version = wheel_info.group("ver").replace("_", "-") + _version = wheel_info.group("ver") + if "_" in _version: + try: + parse_wheel_filename(filename) + except PackagingInvalidWheelName as e: + deprecated( + reason=( + f"Wheel filename {filename!r} is not correctly normalised. " + "Future versions of pip will raise the following error:\n" + f"{e.args[0]}\n\n" + ), + replacement=( + "to rename the wheel to use a correctly normalised " + "name (this may require updating the version in " + "the project metadata)" + ), + gone_in="25.1", + issue=12938, + ) + + _version = _version.replace("_", "-") + + self.version = _version self.build_tag = wheel_info.group("build") self.pyversions = wheel_info.group("pyver").split(".") self.abis = wheel_info.group("abi").split(".") diff --git a/contrib/python/pip/pip/_internal/network/lazy_wheel.py b/contrib/python/pip/pip/_internal/network/lazy_wheel.py index 82ec50d5106..03f883c1fc4 100644 --- a/contrib/python/pip/pip/_internal/network/lazy_wheel.py +++ b/contrib/python/pip/pip/_internal/network/lazy_wheel.py @@ -159,7 +159,7 @@ class LazyZipOverHTTP: try: # For read-only ZIP files, ZipFile only needs # methods read, seek, seekable and tell. - ZipFile(self) # type: ignore + ZipFile(self) except BadZipFile: pass else: diff --git a/contrib/python/pip/pip/_internal/req/constructors.py b/contrib/python/pip/pip/_internal/req/constructors.py index d73236e05c6..56a964f3177 100644 --- a/contrib/python/pip/pip/_internal/req/constructors.py +++ b/contrib/python/pip/pip/_internal/req/constructors.py @@ -80,7 +80,7 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme assert ( pre is not None and post is not None ), f"regex group selection for requirement {req} failed, this should never happen" - extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else "" + extras: str = "[{}]".format(",".join(sorted(new_extras)) if new_extras else "") return get_requirement(f"{pre}{extras}{post}") diff --git a/contrib/python/pip/pip/_internal/req/req_file.py b/contrib/python/pip/pip/_internal/req/req_file.py index 53ad8674cd8..eb2a1f69921 100644 --- a/contrib/python/pip/pip/_internal/req/req_file.py +++ b/contrib/python/pip/pip/_internal/req/req_file.py @@ -329,10 +329,15 @@ class RequirementsFileParser: self, filename: str, constraint: bool ) -> Generator[ParsedLine, None, None]: """Parse a given file, yielding parsed lines.""" - yield from self._parse_and_recurse(filename, constraint) + yield from self._parse_and_recurse( + filename, constraint, [{os.path.abspath(filename): None}] + ) def _parse_and_recurse( - self, filename: str, constraint: bool + self, + filename: str, + constraint: bool, + parsed_files_stack: List[Dict[str, Optional[str]]], ) -> Generator[ParsedLine, None, None]: for line in self._parse_file(filename, constraint): if not line.is_requirement and ( @@ -353,12 +358,30 @@ class RequirementsFileParser: # original file and nested file are paths elif not SCHEME_RE.search(req_path): # do a join so relative paths work - req_path = os.path.join( - os.path.dirname(filename), - req_path, + # and then abspath so that we can identify recursive references + req_path = os.path.abspath( + os.path.join( + os.path.dirname(filename), + req_path, + ) ) - - yield from self._parse_and_recurse(req_path, nested_constraint) + parsed_files = parsed_files_stack[0] + if req_path in parsed_files: + initial_file = parsed_files[req_path] + tail = ( + f" and again in {initial_file}" + if initial_file is not None + else "" + ) + raise RequirementsFileParseError( + f"{req_path} recursively references itself in {filename}{tail}" + ) + # Keeping a track where was each file first included in + new_parsed_files = parsed_files.copy() + new_parsed_files[req_path] = filename + yield from self._parse_and_recurse( + req_path, nested_constraint, [new_parsed_files, *parsed_files_stack] + ) else: yield line diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py index d30d477be68..6617644fe53 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/candidates.py @@ -9,6 +9,7 @@ from pip._vendor.packaging.version import Version from pip._internal.exceptions import ( HashError, InstallationSubprocessError, + InvalidInstalledPackage, MetadataInconsistent, MetadataInvalid, ) @@ -398,8 +399,12 @@ class AlreadyInstalledCandidate(Candidate): def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: if not with_requires: return - for r in self.dist.iter_dependencies(): - yield from self._factory.make_requirements_from_spec(str(r), self._ireq) + + try: + for r in self.dist.iter_dependencies(): + yield from self._factory.make_requirements_from_spec(str(r), self._ireq) + except InvalidRequirement as exc: + raise InvalidInstalledPackage(dist=self.dist, invalid_exc=exc) from None def get_install_requirement(self) -> Optional[InstallRequirement]: return None diff --git a/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py b/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py index 145bdbf71a1..dc6e2e12e1f 100644 --- a/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py +++ b/contrib/python/pip/pip/_internal/resolution/resolvelib/factory.py @@ -23,13 +23,14 @@ from typing import ( from pip._vendor.packaging.requirements import InvalidRequirement from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name -from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import InvalidVersion, Version from pip._vendor.resolvelib import ResolutionImpossible from pip._internal.cache import CacheEntry, WheelCache from pip._internal.exceptions import ( DistributionNotFound, InstallationError, + InvalidInstalledPackage, MetadataInconsistent, MetadataInvalid, UnsupportedPythonVersion, @@ -283,10 +284,15 @@ class Factory: installed_dist = self._installed_dists[name] except KeyError: return None - # Don't use the installed distribution if its version does not fit - # the current dependency graph. - if not specifier.contains(installed_dist.version, prereleases=True): - return None + + try: + # Don't use the installed distribution if its version + # does not fit the current dependency graph. + if not specifier.contains(installed_dist.version, prereleases=True): + return None + except InvalidVersion as e: + raise InvalidInstalledPackage(dist=installed_dist, invalid_exc=e) + candidate = self._make_candidate_from_dist( dist=installed_dist, extras=extras, diff --git a/contrib/python/pip/pip/_internal/utils/compatibility_tags.py b/contrib/python/pip/pip/_internal/utils/compatibility_tags.py index b6ed9a78e55..2e7b7450dce 100644 --- a/contrib/python/pip/pip/_internal/utils/compatibility_tags.py +++ b/contrib/python/pip/pip/_internal/utils/compatibility_tags.py @@ -12,10 +12,11 @@ from pip._vendor.packaging.tags import ( generic_tags, interpreter_name, interpreter_version, + ios_platforms, mac_platforms, ) -_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") +_apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") def version_info_to_nodot(version_info: Tuple[int, ...]) -> str: @@ -24,7 +25,7 @@ def version_info_to_nodot(version_info: Tuple[int, ...]) -> str: def _mac_platforms(arch: str) -> List[str]: - match = _osx_arch_pat.match(arch) + match = _apple_arch_pat.match(arch) if match: name, major, minor, actual_arch = match.groups() mac_version = (int(major), int(minor)) @@ -43,6 +44,26 @@ def _mac_platforms(arch: str) -> List[str]: return arches +def _ios_platforms(arch: str) -> List[str]: + match = _apple_arch_pat.match(arch) + if match: + name, major, minor, actual_multiarch = match.groups() + ios_version = (int(major), int(minor)) + arches = [ + # Since we have always only checked that the platform starts + # with "ios", for backwards-compatibility we extract the + # actual prefix provided by the user in case they provided + # something like "ioscustom_". It may be good to remove + # this as undocumented or deprecate it in the future. + "{}_{}".format(name, arch[len("ios_") :]) + for arch in ios_platforms(ios_version, actual_multiarch) + ] + else: + # arch pattern didn't match (?!) + arches = [arch] + return arches + + def _custom_manylinux_platforms(arch: str) -> List[str]: arches = [arch] arch_prefix, arch_sep, arch_suffix = arch.partition("_") @@ -68,6 +89,8 @@ def _get_custom_platforms(arch: str) -> List[str]: arch_prefix, arch_sep, arch_suffix = arch.partition("_") if arch.startswith("macosx"): arches = _mac_platforms(arch) + elif arch.startswith("ios"): + arches = _ios_platforms(arch) elif arch_prefix in ["manylinux2014", "manylinux2010"]: arches = _custom_manylinux_platforms(arch) else: diff --git a/contrib/python/pip/pip/_internal/utils/misc.py b/contrib/python/pip/pip/_internal/utils/misc.py index 3707e872684..c0a3e4d3b9d 100644 --- a/contrib/python/pip/pip/_internal/utils/misc.py +++ b/contrib/python/pip/pip/_internal/utils/misc.py @@ -129,12 +129,7 @@ def rmtree( onexc = _onerror_ignore if onexc is None: onexc = _onerror_reraise - handler: OnErr = partial( - # `[func, path, Union[ExcInfo, BaseException]] -> Any` is equivalent to - # `Union[([func, path, ExcInfo] -> Any), ([func, path, BaseException] -> Any)]`. - cast(Union[OnExc, OnErr], rmtree_errorhandler), - onexc=onexc, - ) + handler: OnErr = partial(rmtree_errorhandler, onexc=onexc) if sys.version_info >= (3, 12): # See https://docs.python.org/3.12/whatsnew/3.12.html#shutil. shutil.rmtree(dir, onexc=handler) # type: ignore @@ -555,7 +550,7 @@ class HiddenText: # This is useful for testing. def __eq__(self, other: Any) -> bool: - if type(self) != type(other): + if type(self) is not type(other): return False # The string being used for redaction doesn't also have to match, diff --git a/contrib/python/pip/pip/_vendor/certifi/__init__.py b/contrib/python/pip/pip/_vendor/certifi/__init__.py index d321f1bc3ab..f61d77fa382 100644 --- a/contrib/python/pip/pip/_vendor/certifi/__init__.py +++ b/contrib/python/pip/pip/_vendor/certifi/__init__.py @@ -1,4 +1,4 @@ from .core import contents, where __all__ = ["contents", "where"] -__version__ = "2024.07.04" +__version__ = "2024.08.30" diff --git a/contrib/python/pip/pip/_vendor/certifi/cacert.pem b/contrib/python/pip/pip/_vendor/certifi/cacert.pem index a6581589ba1..3c165a1b85e 100644 --- a/contrib/python/pip/pip/_vendor/certifi/cacert.pem +++ b/contrib/python/pip/pip/_vendor/certifi/cacert.pem @@ -4796,3 +4796,134 @@ PQQDAwNoADBlAjAdfKR7w4l1M+E7qUW/Runpod3JIha3RxEL2Jq68cgLcFBTApFw hVmpHqTm6iMxoAACMQD94vizrxa5HnPEluPBMBnYfubDl94cT7iJLzPrSA8Z94dG XSaQpYXFuXqUPoeovQA= -----END CERTIFICATE----- + +# Issuer: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA +# Label: "TWCA CYBER Root CA" +# Serial: 85076849864375384482682434040119489222 +# MD5 Fingerprint: 0b:33:a0:97:52:95:d4:a9:fd:bb:db:6e:a3:55:5b:51 +# SHA1 Fingerprint: f6:b1:1c:1a:83:38:e9:7b:db:b3:a8:c8:33:24:e0:2d:9c:7f:26:66 +# SHA256 Fingerprint: 3f:63:bb:28:14:be:17:4e:c8:b6:43:9c:f0:8d:6d:56:f0:b7:c4:05:88:3a:56:48:a3:34:42:4d:6b:3e:c5:58 +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIQQAE0jMIAAAAAAAAAATzyxjANBgkqhkiG9w0BAQwFADBQ +MQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FOLUNBMRAwDgYDVQQLEwdSb290 +IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3QgQ0EwHhcNMjIxMTIyMDY1NDI5 +WhcNNDcxMTIyMTU1OTU5WjBQMQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FO +LUNBMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3Qg +Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDG+Moe2Qkgfh1sTs6P +40czRJzHyWmqOlt47nDSkvgEs1JSHWdyKKHfi12VCv7qze33Kc7wb3+szT3vsxxF +avcokPFhV8UMxKNQXd7UtcsZyoC5dc4pztKFIuwCY8xEMCDa6pFbVuYdHNWdZsc/ +34bKS1PE2Y2yHer43CdTo0fhYcx9tbD47nORxc5zb87uEB8aBs/pJ2DFTxnk684i +JkXXYJndzk834H/nY62wuFm40AZoNWDTNq5xQwTxaWV4fPMf88oon1oglWa0zbfu +j3ikRRjpJi+NmykosaS3Om251Bw4ckVYsV7r8Cibt4LK/c/WMw+f+5eesRycnupf +Xtuq3VTpMCEobY5583WSjCb+3MX2w7DfRFlDo7YDKPYIMKoNM+HvnKkHIuNZW0CP +2oi3aQiotyMuRAlZN1vH4xfyIutuOVLF3lSnmMlLIJXcRolftBL5hSmO68gnFSDA +S9TMfAxsNAwmmyYxpjyn9tnQS6Jk/zuZQXLB4HCX8SS7K8R0IrGsayIyJNN4KsDA +oS/xUgXJP+92ZuJF2A09rZXIx4kmyA+upwMu+8Ff+iDhcK2wZSA3M2Cw1a/XDBzC +kHDXShi8fgGwsOsVHkQGzaRP6AzRwyAQ4VRlnrZR0Bp2a0JaWHY06rc3Ga4udfmW +5cFZ95RXKSWNOkyrTZpB0F8mAwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSdhWEUfMFib5do5E83QOGt4A1WNzAd +BgNVHQ4EFgQUnYVhFHzBYm+XaORPN0DhreANVjcwDQYJKoZIhvcNAQEMBQADggIB +AGSPesRiDrWIzLjHhg6hShbNcAu3p4ULs3a2D6f/CIsLJc+o1IN1KriWiLb73y0t +tGlTITVX1olNc79pj3CjYcya2x6a4CD4bLubIp1dhDGaLIrdaqHXKGnK/nZVekZn +68xDiBaiA9a5F/gZbG0jAn/xX9AKKSM70aoK7akXJlQKTcKlTfjF/biBzysseKNn +TKkHmvPfXvt89YnNdJdhEGoHK4Fa0o635yDRIG4kqIQnoVesqlVYL9zZyvpoBJ7t +RCT5dEA7IzOrg1oYJkK2bVS1FmAwbLGg+LhBoF1JSdJlBTrq/p1hvIbZv97Tujqx +f36SNI7JAG7cmL3c7IAFrQI932XtCwP39xaEBDG6k5TY8hL4iuO/Qq+n1M0RFxbI +Qh0UqEL20kCGoE8jypZFVmAGzbdVAaYBlGX+bgUJurSkquLvWL69J1bY73NxW0Qz +8ppy6rBePm6pUlvscG21h483XjyMnM7k8M4MZ0HMzvaAq07MTFb1wWFZk7Q+ptq4 +NxKfKjLji7gh7MMrZQzvIt6IKTtM1/r+t+FHvpw+PoP7UV31aPcuIYXcv/Fa4nzX +xeSDwWrruoBa3lwtcHb4yOWHh8qgnaHlIhInD0Q9HWzq1MKLL295q39QpsQZp6F6 +t5b5wR9iWqJDB0BeJsas7a5wFsWqynKKTbDPAYsDP27X +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA12 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA12 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA12" +# Serial: 587887345431707215246142177076162061960426065942 +# MD5 Fingerprint: c6:89:ca:64:42:9b:62:08:49:0b:1e:7f:e9:07:3d:e8 +# SHA1 Fingerprint: 7a:22:1e:3d:de:1b:06:ac:9e:c8:47:70:16:8e:3c:e5:f7:6b:06:f4 +# SHA256 Fingerprint: 3f:03:4b:b5:70:4d:44:b2:d0:85:45:a0:20:57:de:93:eb:f3:90:5f:ce:72:1a:cb:c7:30:c0:6d:da:ee:90:4e +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUZvnHwa/swlG07VOX5uaCwysckBYwDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u +LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExMjAeFw0yMDA0MDgw +NTM2NDZaFw00MDA0MDgwNTM2NDZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD +eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS +b290IENBMTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6OcE3emhF +KxS06+QT61d1I02PJC0W6K6OyX2kVzsqdiUzg2zqMoqUm048luT9Ub+ZyZN+v/mt +p7JIKwccJ/VMvHASd6SFVLX9kHrko+RRWAPNEHl57muTH2SOa2SroxPjcf59q5zd +J1M3s6oYwlkm7Fsf0uZlfO+TvdhYXAvA42VvPMfKWeP+bl+sg779XSVOKik71gur +FzJ4pOE+lEa+Ym6b3kaosRbnhW70CEBFEaCeVESE99g2zvVQR9wsMJvuwPWW0v4J +hscGWa5Pro4RmHvzC1KqYiaqId+OJTN5lxZJjfU+1UefNzFJM3IFTQy2VYzxV4+K +h9GtxRESOaCtAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD +AgEGMB0GA1UdDgQWBBRXNPN0zwRL1SXm8UC2LEzZLemgrTANBgkqhkiG9w0BAQsF +AAOCAQEAPrvbFxbS8hQBICw4g0utvsqFepq2m2um4fylOqyttCg6r9cBg0krY6Ld +mmQOmFxv3Y67ilQiLUoT865AQ9tPkbeGGuwAtEGBpE/6aouIs3YIcipJQMPTw4WJ +mBClnW8Zt7vPemVV2zfrPIpyMpcemik+rY3moxtt9XUa5rBouVui7mlHJzWhhpmA +8zNL4WukJsPvdFlseqJkth5Ew1DgDzk9qTPxpfPSvWKErI4cqc1avTc7bgoitPQV +55FYxTpE05Uo2cBl6XLK0A+9H7MV2anjpEcJnuDLN/v9vZfVvhgaaaI5gdka9at/ +yOPiZwud9AzqVN/Ssq+xIvEg37xEHA== +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA14" +# Serial: 575790784512929437950770173562378038616896959179 +# MD5 Fingerprint: 71:0d:72:fa:92:19:65:5e:89:04:ac:16:33:f0:bc:d5 +# SHA1 Fingerprint: dd:50:c0:f7:79:b3:64:2e:74:a2:b8:9d:9f:d3:40:dd:bb:f0:f2:4f +# SHA256 Fingerprint: 4b:00:9c:10:34:49:4f:9a:b5:6b:ba:3b:a1:d6:27:31:fc:4d:20:d8:95:5a:dc:ec:10:a9:25:60:72:61:e3:38 +-----BEGIN CERTIFICATE----- +MIIFcjCCA1qgAwIBAgIUZNtaDCBO6Ncpd8hQJ6JaJ90t8sswDQYJKoZIhvcNAQEM +BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u +LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNDAeFw0yMDA0MDgw +NzA2MTlaFw00NTA0MDgwNzA2MTlaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD +eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS +b290IENBMTQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDF0nqh1oq/ +FjHQmNE6lPxauG4iwWL3pwon71D2LrGeaBLwbCRjOfHw3xDG3rdSINVSW0KZnvOg +vlIfX8xnbacuUKLBl422+JX1sLrcneC+y9/3OPJH9aaakpUqYllQC6KxNedlsmGy +6pJxaeQp8E+BgQQ8sqVb1MWoWWd7VRxJq3qdwudzTe/NCcLEVxLbAQ4jeQkHO6Lo +/IrPj8BGJJw4J+CDnRugv3gVEOuGTgpa/d/aLIJ+7sr2KeH6caH3iGicnPCNvg9J +kdjqOvn90Ghx2+m1K06Ckm9mH+Dw3EzsytHqunQG+bOEkJTRX45zGRBdAuVwpcAQ +0BB8b8VYSbSwbprafZX1zNoCr7gsfXmPvkPx+SgojQlD+Ajda8iLLCSxjVIHvXib +y8posqTdDEx5YMaZ0ZPxMBoH064iwurO8YQJzOAUbn8/ftKChazcqRZOhaBgy/ac +18izju3Gm5h1DVXoX+WViwKkrkMpKBGk5hIwAUt1ax5mnXkvpXYvHUC0bcl9eQjs +0Wq2XSqypWa9a4X0dFbD9ed1Uigspf9mR6XU/v6eVL9lfgHWMI+lNpyiUBzuOIAB +SMbHdPTGrMNASRZhdCyvjG817XsYAFs2PJxQDcqSMxDxJklt33UkN4Ii1+iW/RVL +ApY+B3KVfqs9TC7XyvDf4Fg/LS8EmjijAQIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUBpOjCl4oaTeqYR3r6/wtbyPk +86AwDQYJKoZIhvcNAQEMBQADggIBAJaAcgkGfpzMkwQWu6A6jZJOtxEaCnFxEM0E +rX+lRVAQZk5KQaID2RFPeje5S+LGjzJmdSX7684/AykmjbgWHfYfM25I5uj4V7Ib +ed87hwriZLoAymzvftAj63iP/2SbNDefNWWipAA9EiOWWF3KY4fGoweITedpdopT +zfFP7ELyk+OZpDc8h7hi2/DsHzc/N19DzFGdtfCXwreFamgLRB7lUe6TzktuhsHS +DCRZNhqfLJGP4xjblJUK7ZGqDpncllPjYYPGFrojutzdfhrGe0K22VoF3Jpf1d+4 +2kd92jjbrDnVHmtsKheMYc2xbXIBw8MgAGJoFjHVdqqGuw6qnsb58Nn4DSEC5MUo +FlkRudlpcyqSeLiSV5sI8jrlL5WwWLdrIBRtFO8KvH7YVdiI2i/6GaX7i+B/OfVy +K4XELKzvGUWSTLNhB9xNH27SgRNcmvMSZ4PPmz+Ln52kuaiWA3rF7iDeM9ovnhp6 +dB7h7sxaOgTdsxoEqBRjrLdHEoOabPXm6RUVkRqEGQ6UROcSjiVbgGcZ3GOTEAtl +Lor6CZpO2oYofaphNdgOpygau1LgePhsumywbrmHXumZNTfxPWQrqaA0k89jL9WB +365jJ6UeTo3cKXhZ+PmhIIynJkBugnLNeLLIjzwec+fBH7/PzqUqm9tEZDKgu39c +JRNItX+S +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA15" +# Serial: 126083514594751269499665114766174399806381178503 +# MD5 Fingerprint: 13:30:fc:c4:62:a6:a9:de:b5:c1:68:af:b5:d2:31:47 +# SHA1 Fingerprint: cb:ba:83:c8:c1:5a:5d:f1:f9:73:6f:ca:d7:ef:28:13:06:4a:07:7d +# SHA256 Fingerprint: e7:78:f0:f0:95:fe:84:37:29:cd:1a:00:82:17:9e:53:14:a9:c2:91:44:28:05:e1:fb:1d:8f:b6:b8:88:6c:3a +-----BEGIN CERTIFICATE----- +MIICIzCCAamgAwIBAgIUFhXHw9hJp75pDIqI7fBw+d23PocwCgYIKoZIzj0EAwMw +UTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28uLCBM +dGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNTAeFw0yMDA0MDgwODMy +NTZaFw00NTA0MDgwODMyNTZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpDeWJl +cnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBSb290 +IENBMTUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQLUHSNZDKZmbPSYAi4Io5GdCx4 +wCtELW1fHcmuS1Iggz24FG1Th2CeX2yF2wYUleDHKP+dX+Sq8bOLbe1PL0vJSpSR +ZHX+AezB2Ot6lHhWGENfa4HL9rzatAy2KZMIaY+jQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTrQciu/NWeUUj1vYv0hyCTQSvT +9DAKBggqhkjOPQQDAwNoADBlAjEA2S6Jfl5OpBEHvVnCB96rMjhTKkZEBhd6zlHp +4P9mLQlO4E/0BdGF9jVg3PVys0Z9AjBEmEYagoUeYWmJSwdLZrWeqrqgHkHZAXQ6 +bkU6iYAZezKYVWOr62Nuk22rGwlgMU4= +-----END CERTIFICATE----- diff --git a/contrib/python/pip/pip/_vendor/distlib/__init__.py b/contrib/python/pip/pip/_vendor/distlib/__init__.py index e999438fe94..bf0d6c6d30e 100644 --- a/contrib/python/pip/pip/_vendor/distlib/__init__.py +++ b/contrib/python/pip/pip/_vendor/distlib/__init__.py @@ -6,7 +6,7 @@ # import logging -__version__ = '0.3.8' +__version__ = '0.3.9' class DistlibException(Exception): diff --git a/contrib/python/pip/pip/_vendor/distlib/compat.py b/contrib/python/pip/pip/_vendor/distlib/compat.py index e93dc27a3eb..ca561dd2e37 100644 --- a/contrib/python/pip/pip/_vendor/distlib/compat.py +++ b/contrib/python/pip/pip/_vendor/distlib/compat.py @@ -217,8 +217,7 @@ except ImportError: # pragma: no cover # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. def _access_check(fn, mode): - return (os.path.exists(fn) and os.access(fn, mode) - and not os.path.isdir(fn)) + return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) # If we're given a path with a directory part, look it up directly rather # than referring to PATH directories. This includes checking relative to the diff --git a/contrib/python/pip/pip/_vendor/distlib/database.py b/contrib/python/pip/pip/_vendor/distlib/database.py index eb3765f193b..c0f896a7d85 100644 --- a/contrib/python/pip/pip/_vendor/distlib/database.py +++ b/contrib/python/pip/pip/_vendor/distlib/database.py @@ -20,14 +20,12 @@ import zipimport from . import DistlibException, resources from .compat import StringIO from .version import get_scheme, UnsupportedVersionError -from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, - LEGACY_METADATA_FILENAME) -from .util import (parse_requirement, cached_property, parse_name_and_version, - read_exports, write_exports, CSVReader, CSVWriter) +from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME) +from .util import (parse_requirement, cached_property, parse_name_and_version, read_exports, write_exports, CSVReader, + CSVWriter) __all__ = [ - 'Distribution', 'BaseInstalledDistribution', 'InstalledDistribution', - 'EggInfoDistribution', 'DistributionPath' + 'Distribution', 'BaseInstalledDistribution', 'InstalledDistribution', 'EggInfoDistribution', 'DistributionPath' ] logger = logging.getLogger(__name__) @@ -35,8 +33,7 @@ logger = logging.getLogger(__name__) EXPORTS_FILENAME = 'pydist-exports.json' COMMANDS_FILENAME = 'pydist-commands.json' -DIST_FILES = ('INSTALLER', METADATA_FILENAME, 'RECORD', 'REQUESTED', - 'RESOURCES', EXPORTS_FILENAME, 'SHARED') +DIST_FILES = ('INSTALLER', METADATA_FILENAME, 'RECORD', 'REQUESTED', 'RESOURCES', EXPORTS_FILENAME, 'SHARED') DISTINFO_EXT = '.dist-info' @@ -134,13 +131,9 @@ class DistributionPath(object): continue try: if self._include_dist and entry.endswith(DISTINFO_EXT): - possible_filenames = [ - METADATA_FILENAME, WHEEL_METADATA_FILENAME, - LEGACY_METADATA_FILENAME - ] + possible_filenames = [METADATA_FILENAME, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME] for metadata_filename in possible_filenames: - metadata_path = posixpath.join( - entry, metadata_filename) + metadata_path = posixpath.join(entry, metadata_filename) pydist = finder.find(metadata_path) if pydist: break @@ -148,15 +141,11 @@ class DistributionPath(object): continue with contextlib.closing(pydist.as_stream()) as stream: - metadata = Metadata(fileobj=stream, - scheme='legacy') + metadata = Metadata(fileobj=stream, scheme='legacy') logger.debug('Found %s', r.path) seen.add(r.path) - yield new_dist_class(r.path, - metadata=metadata, - env=self) - elif self._include_egg and entry.endswith( - ('.egg-info', '.egg')): + yield new_dist_class(r.path, metadata=metadata, env=self) + elif self._include_egg and entry.endswith(('.egg-info', '.egg')): logger.debug('Found %s', r.path) seen.add(r.path) yield old_dist_class(r.path, self) @@ -274,8 +263,7 @@ class DistributionPath(object): try: matcher = self._scheme.matcher('%s (%s)' % (name, version)) except ValueError: - raise DistlibException('invalid name or version: %r, %r' % - (name, version)) + raise DistlibException('invalid name or version: %r, %r' % (name, version)) for dist in self.get_distributions(): # We hit a problem on Travis where enum34 was installed and doesn't @@ -390,10 +378,8 @@ class Distribution(object): def _get_requirements(self, req_attr): md = self.metadata reqts = getattr(md, req_attr) - logger.debug('%s: got requirements %r from metadata: %r', self.name, - req_attr, reqts) - return set( - md.get_requirements(reqts, extras=self.extras, env=self.context)) + logger.debug('%s: got requirements %r from metadata: %r', self.name, req_attr, reqts) + return set(md.get_requirements(reqts, extras=self.extras, env=self.context)) @property def run_requires(self): @@ -469,8 +455,7 @@ class Distribution(object): if type(other) is not type(self): result = False else: - result = (self.name == other.name and self.version == other.version - and self.source_url == other.source_url) + result = (self.name == other.name and self.version == other.version and self.source_url == other.source_url) return result def __hash__(self): @@ -561,8 +546,7 @@ class InstalledDistribution(BaseInstalledDistribution): if r is None: r = finder.find(LEGACY_METADATA_FILENAME) if r is None: - raise ValueError('no %s found in %s' % - (METADATA_FILENAME, path)) + raise ValueError('no %s found in %s' % (METADATA_FILENAME, path)) with contextlib.closing(r.as_stream()) as stream: metadata = Metadata(fileobj=stream, scheme='legacy') @@ -580,8 +564,7 @@ class InstalledDistribution(BaseInstalledDistribution): self.modules = data.splitlines() def __repr__(self): - return '<InstalledDistribution %r %s at %r>' % ( - self.name, self.version, self.path) + return '<InstalledDistribution %r %s at %r>' % (self.name, self.version, self.path) def __str__(self): return "%s %s" % (self.name, self.version) @@ -703,8 +686,7 @@ class InstalledDistribution(BaseInstalledDistribution): size = '%d' % os.path.getsize(path) with open(path, 'rb') as fp: hash_value = self.get_hash(fp.read()) - if path.startswith(base) or (base_under_prefix - and path.startswith(prefix)): + if path.startswith(base) or (base_under_prefix and path.startswith(prefix)): path = os.path.relpath(path, base) writer.writerow((path, hash_value, size)) @@ -746,8 +728,7 @@ class InstalledDistribution(BaseInstalledDistribution): with open(path, 'rb') as f: actual_hash = self.get_hash(f.read(), hasher) if actual_hash != hash_value: - mismatches.append( - (path, 'hash', hash_value, actual_hash)) + mismatches.append((path, 'hash', hash_value, actual_hash)) return mismatches @cached_property @@ -829,9 +810,8 @@ class InstalledDistribution(BaseInstalledDistribution): # it's an absolute path? distinfo_dirname, path = path.split(os.sep)[-2:] if distinfo_dirname != self.path.split(os.sep)[-1]: - raise DistlibException( - 'dist-info file %r does not belong to the %r %s ' - 'distribution' % (path, self.name, self.version)) + raise DistlibException('dist-info file %r does not belong to the %r %s ' + 'distribution' % (path, self.name, self.version)) # The file must be relative if path not in DIST_FILES: @@ -857,8 +837,7 @@ class InstalledDistribution(BaseInstalledDistribution): yield path def __eq__(self, other): - return (isinstance(other, InstalledDistribution) - and self.path == other.path) + return (isinstance(other, InstalledDistribution) and self.path == other.path) # See http://docs.python.org/reference/datamodel#object.__hash__ __hash__ = object.__hash__ @@ -911,8 +890,7 @@ class EggInfoDistribution(BaseInstalledDistribution): if not line: # pragma: no cover continue if line.startswith('['): # pragma: no cover - logger.warning( - 'Unexpected line: quitting requirement scan: %r', line) + logger.warning('Unexpected line: quitting requirement scan: %r', line) break r = parse_requirement(line) if not r: # pragma: no cover @@ -954,13 +932,11 @@ class EggInfoDistribution(BaseInstalledDistribution): else: # FIXME handle the case where zipfile is not available zipf = zipimport.zipimporter(path) - fileobj = StringIO( - zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8')) + fileobj = StringIO(zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8')) metadata = Metadata(fileobj=fileobj, scheme='legacy') try: data = zipf.get_data('EGG-INFO/requires.txt') - tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode( - 'utf-8') + tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode('utf-8') requires = parse_requires_data(data.decode('utf-8')) except IOError: requires = None @@ -990,8 +966,7 @@ class EggInfoDistribution(BaseInstalledDistribution): return metadata def __repr__(self): - return '<EggInfoDistribution %r %s at %r>' % (self.name, self.version, - self.path) + return '<EggInfoDistribution %r %s at %r>' % (self.name, self.version, self.path) def __str__(self): return "%s %s" % (self.name, self.version) @@ -1083,8 +1058,7 @@ class EggInfoDistribution(BaseInstalledDistribution): yield line def __eq__(self, other): - return (isinstance(other, EggInfoDistribution) - and self.path == other.path) + return (isinstance(other, EggInfoDistribution) and self.path == other.path) # See http://docs.python.org/reference/datamodel#object.__hash__ __hash__ = object.__hash__ @@ -1184,8 +1158,7 @@ class DependencyGraph(object): disconnected.append(dist) for other, label in adjs: if label is not None: - f.write('"%s" -> "%s" [label="%s"]\n' % - (dist.name, other.name, label)) + f.write('"%s" -> "%s" [label="%s"]\n' % (dist.name, other.name, label)) else: f.write('"%s" -> "%s"\n' % (dist.name, other.name)) if not skip_disconnected and len(disconnected) > 0: @@ -1225,8 +1198,7 @@ class DependencyGraph(object): # Remove from the adjacency list of others for k, v in alist.items(): alist[k] = [(d, r) for d, r in v if d not in to_remove] - logger.debug('Moving to result: %s', - ['%s (%s)' % (d.name, d.version) for d in to_remove]) + logger.debug('Moving to result: %s', ['%s (%s)' % (d.name, d.version) for d in to_remove]) result.extend(to_remove) return result, list(alist.keys()) @@ -1261,15 +1233,13 @@ def make_graph(dists, scheme='default'): # now make the edges for dist in dists: - requires = (dist.run_requires | dist.meta_requires - | dist.build_requires | dist.dev_requires) + requires = (dist.run_requires | dist.meta_requires | dist.build_requires | dist.dev_requires) for req in requires: try: matcher = scheme.matcher(req) except UnsupportedVersionError: # XXX compat-mode if cannot read the version - logger.warning('could not read version %r - using name only', - req) + logger.warning('could not read version %r - using name only', req) name = req.split()[0] matcher = scheme.matcher(name) diff --git a/contrib/python/pip/pip/_vendor/distlib/locators.py b/contrib/python/pip/pip/_vendor/distlib/locators.py index f9f0788fc2a..222c1bf3e90 100644 --- a/contrib/python/pip/pip/_vendor/distlib/locators.py +++ b/contrib/python/pip/pip/_vendor/distlib/locators.py @@ -19,15 +19,12 @@ except ImportError: # pragma: no cover import zlib from . import DistlibException -from .compat import (urljoin, urlparse, urlunparse, url2pathname, pathname2url, - queue, quote, unescape, build_opener, - HTTPRedirectHandler as BaseRedirectHandler, text_type, - Request, HTTPError, URLError) +from .compat import (urljoin, urlparse, urlunparse, url2pathname, pathname2url, queue, quote, unescape, build_opener, + HTTPRedirectHandler as BaseRedirectHandler, text_type, Request, HTTPError, URLError) from .database import Distribution, DistributionPath, make_dist from .metadata import Metadata, MetadataInvalidError -from .util import (cached_property, ensure_slash, split_filename, get_project_data, - parse_requirement, parse_name_and_version, ServerProxy, - normalize_name) +from .util import (cached_property, ensure_slash, split_filename, get_project_data, parse_requirement, + parse_name_and_version, ServerProxy, normalize_name) from .version import get_scheme, UnsupportedVersionError from .wheel import Wheel, is_compatible @@ -58,6 +55,7 @@ class RedirectHandler(BaseRedirectHandler): """ A class to work around a bug in some Python 3.2.x releases. """ + # There's a bug in the base version for some 3.2.x # (e.g. 3.2.2 on Ubuntu Oneiric). If a Location header # returns e.g. /abc, it bails because it says the scheme '' @@ -80,8 +78,7 @@ class RedirectHandler(BaseRedirectHandler): headers.replace_header(key, newurl) else: headers[key] = newurl - return BaseRedirectHandler.http_error_302(self, req, fp, code, msg, - headers) + return BaseRedirectHandler.http_error_302(self, req, fp, code, msg, headers) http_error_301 = http_error_303 = http_error_307 = http_error_302 @@ -92,7 +89,7 @@ class Locator(object): """ source_extensions = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz') binary_extensions = ('.egg', '.exe', '.whl') - excluded_extensions = ('.pdf',) + excluded_extensions = ('.pdf', ) # A list of tags indicating which wheels you want to match. The default # value of None matches against the tags compatible with the running @@ -100,7 +97,7 @@ class Locator(object): # instance to a list of tuples (pyver, abi, arch) which you want to match. wheel_tags = None - downloadable_extensions = source_extensions + ('.whl',) + downloadable_extensions = source_extensions + ('.whl', ) def __init__(self, scheme='default'): """ @@ -200,8 +197,7 @@ class Locator(object): is_downloadable = basename.endswith(self.downloadable_extensions) if is_wheel: compatible = is_compatible(Wheel(basename), self.wheel_tags) - return (t.scheme == 'https', 'pypi.org' in t.netloc, - is_downloadable, is_wheel, compatible, basename) + return (t.scheme == 'https', 'pypi.org' in t.netloc, is_downloadable, is_wheel, compatible, basename) def prefer_url(self, url1, url2): """ @@ -239,14 +235,14 @@ class Locator(object): If it is, a dictionary is returned with keys "name", "version", "filename" and "url"; otherwise, None is returned. """ + def same_project(name1, name2): return normalize_name(name1) == normalize_name(name2) result = None scheme, netloc, path, params, query, frag = urlparse(url) if frag.lower().startswith('egg='): # pragma: no cover - logger.debug('%s: version hint in fragment: %r', - project_name, frag) + logger.debug('%s: version hint in fragment: %r', project_name, frag) m = HASHER_HASH.match(frag) if m: algo, digest = m.groups() @@ -270,10 +266,8 @@ class Locator(object): 'name': wheel.name, 'version': wheel.version, 'filename': wheel.filename, - 'url': urlunparse((scheme, netloc, origpath, - params, query, '')), - 'python-version': ', '.join( - ['.'.join(list(v[2:])) for v in wheel.pyver]), + 'url': urlunparse((scheme, netloc, origpath, params, query, '')), + 'python-version': ', '.join(['.'.join(list(v[2:])) for v in wheel.pyver]), } except Exception: # pragma: no cover logger.warning('invalid path for wheel: %s', path) @@ -294,8 +288,7 @@ class Locator(object): 'name': name, 'version': version, 'filename': filename, - 'url': urlunparse((scheme, netloc, origpath, - params, query, '')), + 'url': urlunparse((scheme, netloc, origpath, params, query, '')), } if pyver: # pragma: no cover result['python-version'] = pyver @@ -371,7 +364,7 @@ class Locator(object): self.matcher = matcher = scheme.matcher(r.requirement) logger.debug('matcher: %s (%s)', matcher, type(matcher).__name__) versions = self.get_project(r.name) - if len(versions) > 2: # urls and digests keys are present + if len(versions) > 2: # urls and digests keys are present # sometimes, versions are invalid slist = [] vcls = matcher.version_class @@ -412,6 +405,7 @@ class PyPIRPCLocator(Locator): This locator uses XML-RPC to locate distributions. It therefore cannot be used with simple mirrors (that only mirror file content). """ + def __init__(self, url, **kwargs): """ Initialise an instance. @@ -461,6 +455,7 @@ class PyPIJSONLocator(Locator): This locator uses PyPI's JSON interface. It's very limited in functionality and probably not worth using. """ + def __init__(self, url, **kwargs): super(PyPIJSONLocator, self).__init__(**kwargs) self.base_url = ensure_slash(url) @@ -498,7 +493,7 @@ class PyPIJSONLocator(Locator): # Now get other releases for version, infos in d['releases'].items(): if version == md.version: - continue # already done + continue # already done omd = Metadata(scheme=self.scheme) omd.name = md.name omd.version = version @@ -511,6 +506,8 @@ class PyPIJSONLocator(Locator): odist.digests[url] = self._get_digest(info) result['urls'].setdefault(version, set()).add(url) result['digests'][url] = self._get_digest(info) + + # for info in urls: # md.source_url = info['url'] # dist.digest = self._get_digest(info) @@ -534,7 +531,8 @@ class Page(object): # or immediately followed by a "rel" attribute. The attribute values can be # declared with double quotes, single quotes or no quotes - which leads to # the length of the expression. - _href = re.compile(""" + _href = re.compile( + """ (rel\\s*=\\s*(?:"(?P<rel1>[^"]*)"|'(?P<rel2>[^']*)'|(?P<rel3>[^>\\s\n]*))\\s+)? href\\s*=\\s*(?:"(?P<url1>[^"]*)"|'(?P<url2>[^']*)'|(?P<url3>[^>\\s\n]*)) (\\s+rel\\s*=\\s*(?:"(?P<rel4>[^"]*)"|'(?P<rel5>[^']*)'|(?P<rel6>[^>\\s\n]*)))? @@ -561,17 +559,16 @@ href\\s*=\\s*(?:"(?P<url1>[^"]*)"|'(?P<url2>[^']*)'|(?P<url3>[^>\\s\n]*)) about their "rel" attribute, for determining which ones to treat as downloads and which ones to queue for further scraping. """ + def clean(url): "Tidy up an URL." scheme, netloc, path, params, query, frag = urlparse(url) - return urlunparse((scheme, netloc, quote(path), - params, query, frag)) + return urlunparse((scheme, netloc, quote(path), params, query, frag)) result = set() for match in self._href.finditer(self.data): d = match.groupdict('') - rel = (d['rel1'] or d['rel2'] or d['rel3'] or - d['rel4'] or d['rel5'] or d['rel6']) + rel = (d['rel1'] or d['rel2'] or d['rel3'] or d['rel4'] or d['rel5'] or d['rel6']) url = d['url1'] or d['url2'] or d['url3'] url = urljoin(self.base_url, url) url = unescape(url) @@ -645,7 +642,7 @@ class SimpleScrapingLocator(Locator): # Note that you need two loops, since you can't say which # thread will get each sentinel for t in self._threads: - self._to_fetch.put(None) # sentinel + self._to_fetch.put(None) # sentinel for t in self._threads: t.join() self._threads = [] @@ -693,7 +690,7 @@ class SimpleScrapingLocator(Locator): info = self.convert_url_to_download_info(url, self.project_name) logger.debug('process_download: %s -> %s', url, info) if info: - with self._lock: # needed because self.result is shared + with self._lock: # needed because self.result is shared self._update_version_data(self.result, info) return info @@ -703,8 +700,7 @@ class SimpleScrapingLocator(Locator): particular "rel" attribute should be queued for scraping. """ scheme, netloc, path, _, _, _ = urlparse(link) - if path.endswith(self.source_extensions + self.binary_extensions + - self.excluded_extensions): + if path.endswith(self.source_extensions + self.binary_extensions + self.excluded_extensions): result = False elif self.skip_externals and not link.startswith(self.base_url): result = False @@ -722,8 +718,7 @@ class SimpleScrapingLocator(Locator): result = False else: result = True - logger.debug('should_queue: %s (%s) from %s -> %s', link, rel, - referrer, result) + logger.debug('should_queue: %s (%s) from %s -> %s', link, rel, referrer, result) return result def _fetch(self): @@ -738,14 +733,13 @@ class SimpleScrapingLocator(Locator): try: if url: page = self.get_page(url) - if page is None: # e.g. after an error + if page is None: # e.g. after an error continue for link, rel in page.links: if link not in self._seen: try: self._seen.add(link) - if (not self._process_download(link) and - self._should_queue(link, url, rel)): + if (not self._process_download(link) and self._should_queue(link, url, rel)): logger.debug('Queueing %s from %s', link, url) self._to_fetch.put(link) except MetadataInvalidError: # e.g. invalid versions @@ -793,7 +787,7 @@ class SimpleScrapingLocator(Locator): data = resp.read() encoding = headers.get('Content-Encoding') if encoding: - decoder = self.decoders[encoding] # fail if not found + decoder = self.decoders[encoding] # fail if not found data = decoder(data) encoding = 'utf-8' m = CHARSET.search(content_type) @@ -802,7 +796,7 @@ class SimpleScrapingLocator(Locator): try: data = data.decode(encoding) except UnicodeError: # pragma: no cover - data = data.decode('latin-1') # fallback + data = data.decode('latin-1') # fallback result = Page(data, final_url) self._page_cache[final_url] = result except HTTPError as e: @@ -815,7 +809,7 @@ class SimpleScrapingLocator(Locator): except Exception as e: # pragma: no cover logger.exception('Fetch failed: %s: %s', url, e) finally: - self._page_cache[url] = result # even if None (failure) + self._page_cache[url] = result # even if None (failure) return result _distname_re = re.compile('<a href=[^>]*>([^<]+)<') @@ -869,9 +863,7 @@ class DirectoryLocator(Locator): for fn in files: if self.should_include(fn, root): fn = os.path.join(root, fn) - url = urlunparse(('file', '', - pathname2url(os.path.abspath(fn)), - '', '', '')) + url = urlunparse(('file', '', pathname2url(os.path.abspath(fn)), '', '', '')) info = self.convert_url_to_download_info(url, name) if info: self._update_version_data(result, info) @@ -888,9 +880,7 @@ class DirectoryLocator(Locator): for fn in files: if self.should_include(fn, root): fn = os.path.join(root, fn) - url = urlunparse(('file', '', - pathname2url(os.path.abspath(fn)), - '', '', '')) + url = urlunparse(('file', '', pathname2url(os.path.abspath(fn)), '', '', '')) info = self.convert_url_to_download_info(url, None) if info: result.add(info['name']) @@ -906,6 +896,7 @@ class JSONLocator(Locator): require archive downloads before dependencies can be determined! As you might imagine, that can be slow. """ + def get_distribution_names(self): """ Return all the distribution names known to this locator. @@ -922,9 +913,9 @@ class JSONLocator(Locator): # We don't store summary in project metadata as it makes # the data bigger for no benefit during dependency # resolution - dist = make_dist(data['name'], info['version'], - summary=data.get('summary', - 'Placeholder for summary'), + dist = make_dist(data['name'], + info['version'], + summary=data.get('summary', 'Placeholder for summary'), scheme=self.scheme) md = dist.metadata md.source_url = info['url'] @@ -943,6 +934,7 @@ class DistPathLocator(Locator): This locator finds installed distributions in a path. It can be useful for adding to an :class:`AggregatingLocator`. """ + def __init__(self, distpath, **kwargs): """ Initialise an instance. @@ -960,8 +952,12 @@ class DistPathLocator(Locator): else: result = { dist.version: dist, - 'urls': {dist.version: set([dist.source_url])}, - 'digests': {dist.version: set([None])} + 'urls': { + dist.version: set([dist.source_url]) + }, + 'digests': { + dist.version: set([None]) + } } return result @@ -970,6 +966,7 @@ class AggregatingLocator(Locator): """ This class allows you to chain and/or merge a list of locators. """ + def __init__(self, *locators, **kwargs): """ Initialise an instance. @@ -1058,10 +1055,9 @@ class AggregatingLocator(Locator): # We use a legacy scheme simply because most of the dists on PyPI use legacy # versions which don't conform to PEP 440. default_locator = AggregatingLocator( - # JSONLocator(), # don't use as PEP 426 is withdrawn - SimpleScrapingLocator('https://pypi.org/simple/', - timeout=3.0), - scheme='legacy') + # JSONLocator(), # don't use as PEP 426 is withdrawn + SimpleScrapingLocator('https://pypi.org/simple/', timeout=3.0), + scheme='legacy') locate = default_locator.locate @@ -1137,7 +1133,7 @@ class DependencyFinder(object): :return: A set of distribution which can fulfill the requirement. """ matcher = self.get_matcher(reqt) - name = matcher.key # case-insensitive + name = matcher.key # case-insensitive result = set() provided = self.provided if name in provided: @@ -1179,8 +1175,7 @@ class DependencyFinder(object): unmatched.add(s) if unmatched: # can't replace other with provider - problems.add(('cantreplace', provider, other, - frozenset(unmatched))) + problems.add(('cantreplace', provider, other, frozenset(unmatched))) result = False else: # can replace other with provider @@ -1233,8 +1228,7 @@ class DependencyFinder(object): dist = odist = requirement logger.debug('passed %s as requirement', odist) else: - dist = odist = self.locator.locate(requirement, - prereleases=prereleases) + dist = odist = self.locator.locate(requirement, prereleases=prereleases) if dist is None: raise DistlibException('Unable to locate %r' % requirement) logger.debug('located %s', odist) @@ -1244,7 +1238,7 @@ class DependencyFinder(object): install_dists = set([odist]) while todo: dist = todo.pop() - name = dist.key # case-insensitive + name = dist.key # case-insensitive if name not in self.dists_by_name: self.add_distribution(dist) else: @@ -1281,8 +1275,7 @@ class DependencyFinder(object): providers.add(provider) if r in ireqts and dist in install_dists: install_dists.add(provider) - logger.debug('Adding %s to install_dists', - provider.name_and_version) + logger.debug('Adding %s to install_dists', provider.name_and_version) for p in providers: name = p.key if name not in self.dists_by_name: @@ -1297,7 +1290,6 @@ class DependencyFinder(object): for dist in dists: dist.build_time_dependency = dist not in install_dists if dist.build_time_dependency: - logger.debug('%s is a build-time dependency only.', - dist.name_and_version) + logger.debug('%s is a build-time dependency only.', dist.name_and_version) logger.debug('find done for %s', odist) return dists, problems diff --git a/contrib/python/pip/pip/_vendor/distlib/markers.py b/contrib/python/pip/pip/_vendor/distlib/markers.py index 1514d460e70..3f5632be47c 100644 --- a/contrib/python/pip/pip/_vendor/distlib/markers.py +++ b/contrib/python/pip/pip/_vendor/distlib/markers.py @@ -23,8 +23,7 @@ from .version import LegacyVersion as LV __all__ = ['interpret'] -_VERSION_PATTERN = re.compile( - r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")') +_VERSION_PATTERN = re.compile(r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")') _VERSION_MARKERS = {'python_version', 'python_full_version'} @@ -82,13 +81,12 @@ class Evaluator(object): elhs = expr['lhs'] erhs = expr['rhs'] if _is_literal(expr['lhs']) and _is_literal(expr['rhs']): - raise SyntaxError('invalid comparison: %s %s %s' % - (elhs, op, erhs)) + raise SyntaxError('invalid comparison: %s %s %s' % (elhs, op, erhs)) lhs = self.evaluate(elhs, context) rhs = self.evaluate(erhs, context) - if ((_is_version_marker(elhs) or _is_version_marker(erhs)) - and op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')): + if ((_is_version_marker(elhs) or _is_version_marker(erhs)) and + op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')): lhs = LV(lhs) rhs = LV(rhs) elif _is_version_marker(elhs) and op in ('in', 'not in'): @@ -111,8 +109,7 @@ def default_context(): return version if hasattr(sys, 'implementation'): - implementation_version = format_full_version( - sys.implementation.version) + implementation_version = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: implementation_version = '0' @@ -156,11 +153,9 @@ def interpret(marker, execution_context=None): try: expr, rest = parse_marker(marker) except Exception as e: - raise SyntaxError('Unable to interpret marker syntax: %s: %s' % - (marker, e)) + raise SyntaxError('Unable to interpret marker syntax: %s: %s' % (marker, e)) if rest and rest[0] != '#': - raise SyntaxError('unexpected trailing data in marker: %s: %s' % - (marker, rest)) + raise SyntaxError('unexpected trailing data in marker: %s: %s' % (marker, rest)) context = dict(DEFAULT_CONTEXT) if execution_context: context.update(execution_context) diff --git a/contrib/python/pip/pip/_vendor/distlib/metadata.py b/contrib/python/pip/pip/_vendor/distlib/metadata.py index 7189aeef229..ce9a34b3e24 100644 --- a/contrib/python/pip/pip/_vendor/distlib/metadata.py +++ b/contrib/python/pip/pip/_vendor/distlib/metadata.py @@ -15,7 +15,6 @@ import json import logging import re - from . import DistlibException, __version__ from .compat import StringIO, string_types, text_type from .markers import interpret @@ -40,6 +39,7 @@ class MetadataUnrecognizedVersionError(DistlibException): class MetadataInvalidError(DistlibException): """A metadata value is invalid""" + # public API of this module __all__ = ['Metadata', 'PKG_INFO_ENCODING', 'PKG_INFO_PREFERRED_VERSION'] @@ -52,53 +52,38 @@ PKG_INFO_PREFERRED_VERSION = '1.1' _LINE_PREFIX_1_2 = re.compile('\n \\|') _LINE_PREFIX_PRE_1_2 = re.compile('\n ') -_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', - 'Summary', 'Description', - 'Keywords', 'Home-page', 'Author', 'Author-email', - 'License') +_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Summary', 'Description', 'Keywords', 'Home-page', + 'Author', 'Author-email', 'License') -_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', - 'Supported-Platform', 'Summary', 'Description', - 'Keywords', 'Home-page', 'Author', 'Author-email', - 'License', 'Classifier', 'Download-URL', 'Obsoletes', +_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', 'License', 'Classifier', 'Download-URL', 'Obsoletes', 'Provides', 'Requires') -_314_MARKERS = ('Obsoletes', 'Provides', 'Requires', 'Classifier', - 'Download-URL') +_314_MARKERS = ('Obsoletes', 'Provides', 'Requires', 'Classifier', 'Download-URL') -_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', - 'Supported-Platform', 'Summary', 'Description', - 'Keywords', 'Home-page', 'Author', 'Author-email', - 'Maintainer', 'Maintainer-email', 'License', - 'Classifier', 'Download-URL', 'Obsoletes-Dist', - 'Project-URL', 'Provides-Dist', 'Requires-Dist', +_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', + 'Classifier', 'Download-URL', 'Obsoletes-Dist', 'Project-URL', 'Provides-Dist', 'Requires-Dist', 'Requires-Python', 'Requires-External') -_345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', - 'Obsoletes-Dist', 'Requires-External', 'Maintainer', - 'Maintainer-email', 'Project-URL') +_345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', 'Obsoletes-Dist', 'Requires-External', + 'Maintainer', 'Maintainer-email', 'Project-URL') -_426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', - 'Supported-Platform', 'Summary', 'Description', - 'Keywords', 'Home-page', 'Author', 'Author-email', - 'Maintainer', 'Maintainer-email', 'License', - 'Classifier', 'Download-URL', 'Obsoletes-Dist', - 'Project-URL', 'Provides-Dist', 'Requires-Dist', - 'Requires-Python', 'Requires-External', 'Private-Version', - 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension', - 'Provides-Extra') +_426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', + 'Classifier', 'Download-URL', 'Obsoletes-Dist', 'Project-URL', 'Provides-Dist', 'Requires-Dist', + 'Requires-Python', 'Requires-External', 'Private-Version', 'Obsoleted-By', 'Setup-Requires-Dist', + 'Extension', 'Provides-Extra') -_426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', - 'Setup-Requires-Dist', 'Extension') +_426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') # See issue #106: Sometimes 'Requires' and 'Provides' occur wrongly in # the metadata. Include them in the tuple literal below to allow them # (for now). # Ditto for Obsoletes - see issue #140. -_566_FIELDS = _426_FIELDS + ('Description-Content-Type', - 'Requires', 'Provides', 'Obsoletes') +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires', 'Provides', 'Obsoletes') -_566_MARKERS = ('Description-Content-Type',) +_566_MARKERS = ('Description-Content-Type', ) _643_MARKERS = ('Dynamic', 'License-File') @@ -135,6 +120,7 @@ def _version2fieldlist(version): def _best_version(fields): """Detect the best version depending on the fields used.""" + def _has_marker(keys, markers): return any(marker in keys for marker in markers) @@ -163,12 +149,12 @@ def _best_version(fields): possible_versions.remove('2.2') logger.debug('Removed 2.2 due to %s', key) # if key not in _426_FIELDS and '2.0' in possible_versions: - # possible_versions.remove('2.0') - # logger.debug('Removed 2.0 due to %s', key) + # possible_versions.remove('2.0') + # logger.debug('Removed 2.0 due to %s', key) # possible_version contains qualified versions if len(possible_versions) == 1: - return possible_versions[0] # found ! + return possible_versions[0] # found ! elif len(possible_versions) == 0: logger.debug('Out of options - unknown metadata set: %s', fields) raise MetadataConflictError('Unknown metadata set') @@ -199,28 +185,25 @@ def _best_version(fields): if is_2_1: return '2.1' # if is_2_2: - # return '2.2' + # return '2.2' return '2.2' + # This follows the rules about transforming keys as described in # https://www.python.org/dev/peps/pep-0566/#id17 -_ATTR2FIELD = { - name.lower().replace("-", "_"): name for name in _ALL_FIELDS -} +_ATTR2FIELD = {name.lower().replace("-", "_"): name for name in _ALL_FIELDS} _FIELD2ATTR = {field: attr for attr, field in _ATTR2FIELD.items()} _PREDICATE_FIELDS = ('Requires-Dist', 'Obsoletes-Dist', 'Provides-Dist') -_VERSIONS_FIELDS = ('Requires-Python',) -_VERSION_FIELDS = ('Version',) -_LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes', - 'Requires', 'Provides', 'Obsoletes-Dist', - 'Provides-Dist', 'Requires-Dist', 'Requires-External', - 'Project-URL', 'Supported-Platform', 'Setup-Requires-Dist', +_VERSIONS_FIELDS = ('Requires-Python', ) +_VERSION_FIELDS = ('Version', ) +_LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes', 'Requires', 'Provides', 'Obsoletes-Dist', 'Provides-Dist', + 'Requires-Dist', 'Requires-External', 'Project-URL', 'Supported-Platform', 'Setup-Requires-Dist', 'Provides-Extra', 'Extension', 'License-File') -_LISTTUPLEFIELDS = ('Project-URL',) +_LISTTUPLEFIELDS = ('Project-URL', ) -_ELEMENTSFIELD = ('Keywords',) +_ELEMENTSFIELD = ('Keywords', ) _UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description') @@ -252,10 +235,10 @@ class LegacyMetadata(object): - *mapping* is a dict-like object - *scheme* is a version scheme name """ + # TODO document the mapping API and UNKNOWN default key - def __init__(self, path=None, fileobj=None, mapping=None, - scheme='default'): + def __init__(self, path=None, fileobj=None, mapping=None, scheme='default'): if [path, fileobj, mapping].count(None) < 2: raise TypeError('path, fileobj and mapping are exclusive') self._fields = {} @@ -290,8 +273,7 @@ class LegacyMetadata(object): raise KeyError(name) def __contains__(self, name): - return (name in self._fields or - self._convert_name(name) in self._fields) + return (name in self._fields or self._convert_name(name) in self._fields) def _convert_name(self, name): if name in _ALL_FIELDS: @@ -319,12 +301,12 @@ class LegacyMetadata(object): # Public API # -# dependencies = property(_get_dependencies, _set_dependencies) - def get_fullname(self, filesafe=False): - """Return the distribution name with version. + """ + Return the distribution name with version. - If filesafe is true, return a filename-escaped form.""" + If filesafe is true, return a filename-escaped form. + """ return _get_name_and_version(self['Name'], self['Version'], filesafe) def is_field(self, name): @@ -415,6 +397,7 @@ class LegacyMetadata(object): Keys that don't match a metadata field or that have an empty value are dropped. """ + def _set(key, value): if key in _ATTR2FIELD and value: self.set(self._convert_name(key), value) @@ -437,14 +420,12 @@ class LegacyMetadata(object): """Control then set a metadata field.""" name = self._convert_name(name) - if ((name in _ELEMENTSFIELD or name == 'Platform') and - not isinstance(value, (list, tuple))): + if ((name in _ELEMENTSFIELD or name == 'Platform') and not isinstance(value, (list, tuple))): if isinstance(value, string_types): value = [v.strip() for v in value.split(',')] else: value = [] - elif (name in _LISTFIELDS and - not isinstance(value, (list, tuple))): + elif (name in _LISTFIELDS and not isinstance(value, (list, tuple))): if isinstance(value, string_types): value = [value] else: @@ -458,18 +439,14 @@ class LegacyMetadata(object): for v in value: # check that the values are valid if not scheme.is_valid_matcher(v.split(';')[0]): - logger.warning( - "'%s': '%s' is not valid (field '%s')", - project_name, v, name) + logger.warning("'%s': '%s' is not valid (field '%s')", project_name, v, name) # FIXME this rejects UNKNOWN, is that right? elif name in _VERSIONS_FIELDS and value is not None: if not scheme.is_valid_constraint_list(value): - logger.warning("'%s': '%s' is not a valid version (field '%s')", - project_name, value, name) + logger.warning("'%s': '%s' is not a valid version (field '%s')", project_name, value, name) elif name in _VERSION_FIELDS and value is not None: if not scheme.is_valid_version(value): - logger.warning("'%s': '%s' is not a valid version (field '%s')", - project_name, value, name) + logger.warning("'%s': '%s' is not a valid version (field '%s')", project_name, value, name) if name in _UNICODEFIELDS: if name == 'Description': @@ -539,10 +516,8 @@ class LegacyMetadata(object): return True for fields, controller in ((_PREDICATE_FIELDS, are_valid_constraints), - (_VERSIONS_FIELDS, - scheme.is_valid_constraint_list), - (_VERSION_FIELDS, - scheme.is_valid_version)): + (_VERSIONS_FIELDS, scheme.is_valid_constraint_list), (_VERSION_FIELDS, + scheme.is_valid_version)): for field in fields: value = self.get(field, None) if value is not None and not controller(value): @@ -598,8 +573,7 @@ class LegacyMetadata(object): return [(key, self[key]) for key in self.keys()] def __repr__(self): - return '<%s %s %s>' % (self.__class__.__name__, self.name, - self.version) + return '<%s %s %s>' % (self.__class__.__name__, self.name, self.version) METADATA_FILENAME = 'pydist.json' @@ -631,7 +605,7 @@ class Metadata(object): MANDATORY_KEYS = { 'name': (), 'version': (), - 'summary': ('legacy',), + 'summary': ('legacy', ), } INDEX_KEYS = ('name version license summary description author ' @@ -644,22 +618,21 @@ class Metadata(object): SYNTAX_VALIDATORS = { 'metadata_version': (METADATA_VERSION_MATCHER, ()), - 'name': (NAME_MATCHER, ('legacy',)), - 'version': (VERSION_MATCHER, ('legacy',)), - 'summary': (SUMMARY_MATCHER, ('legacy',)), - 'dynamic': (FIELDNAME_MATCHER, ('legacy',)), + 'name': (NAME_MATCHER, ('legacy', )), + 'version': (VERSION_MATCHER, ('legacy', )), + 'summary': (SUMMARY_MATCHER, ('legacy', )), + 'dynamic': (FIELDNAME_MATCHER, ('legacy', )), } __slots__ = ('_legacy', '_data', 'scheme') - def __init__(self, path=None, fileobj=None, mapping=None, - scheme='default'): + def __init__(self, path=None, fileobj=None, mapping=None, scheme='default'): if [path, fileobj, mapping].count(None) < 2: raise TypeError('path, fileobj and mapping are exclusive') self._legacy = None self._data = None self.scheme = scheme - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() if mapping is not None: try: self._validate_mapping(mapping, scheme) @@ -693,8 +666,7 @@ class Metadata(object): # The ValueError comes from the json.load - if that # succeeds and we get a validation error, we want # that to propagate - self._legacy = LegacyMetadata(fileobj=StringIO(data), - scheme=scheme) + self._legacy = LegacyMetadata(fileobj=StringIO(data), scheme=scheme) self.validate() common_keys = set(('name', 'version', 'license', 'keywords', 'summary')) @@ -732,8 +704,7 @@ class Metadata(object): result = self._legacy.get(lk) else: value = None if maker is None else maker() - if key not in ('commands', 'exports', 'modules', 'namespaces', - 'classifiers'): + if key not in ('commands', 'exports', 'modules', 'namespaces', 'classifiers'): result = self._data.get(key, value) else: # special cases for PEP 459 @@ -770,8 +741,7 @@ class Metadata(object): m = pattern.match(value) if not m: raise MetadataInvalidError("'%s' is an invalid value for " - "the '%s' property" % (value, - key)) + "the '%s' property" % (value, key)) def __setattr__(self, key, value): self._validate_value(key, value) @@ -783,8 +753,7 @@ class Metadata(object): if lk is None: raise NotImplementedError self._legacy[lk] = value - elif key not in ('commands', 'exports', 'modules', 'namespaces', - 'classifiers'): + elif key not in ('commands', 'exports', 'modules', 'namespaces', 'classifiers'): self._data[key] = value else: # special cases for PEP 459 @@ -872,8 +841,7 @@ class Metadata(object): # A recursive call, but it should terminate since 'test' # has been removed from the extras reqts = self._data.get('%s_requires' % key, []) - result.extend(self.get_requirements(reqts, extras=extras, - env=env)) + result.extend(self.get_requirements(reqts, extras=extras, env=env)) return result @property @@ -914,8 +882,7 @@ class Metadata(object): if self._legacy: missing, warnings = self._legacy.check(True) if missing or warnings: - logger.warning('Metadata: missing: %s, warnings: %s', - missing, warnings) + logger.warning('Metadata: missing: %s, warnings: %s', missing, warnings) else: self._validate_mapping(self._data, self.scheme) @@ -932,9 +899,8 @@ class Metadata(object): 'metadata_version': self.METADATA_VERSION, 'generator': self.GENERATOR, } - lmd = self._legacy.todict(True) # skip missing ones - for k in ('name', 'version', 'license', 'summary', 'description', - 'classifier'): + lmd = self._legacy.todict(True) # skip missing ones + for k in ('name', 'version', 'license', 'summary', 'description', 'classifier'): if k in lmd: if k == 'classifier': nk = 'classifiers' @@ -945,14 +911,13 @@ class Metadata(object): if kw == ['']: kw = [] result['keywords'] = kw - keys = (('requires_dist', 'run_requires'), - ('setup_requires_dist', 'build_requires')) + keys = (('requires_dist', 'run_requires'), ('setup_requires_dist', 'build_requires')) for ok, nk in keys: if ok in lmd and lmd[ok]: result[nk] = [{'requires': lmd[ok]}] result['provides'] = self.provides - author = {} - maintainer = {} + # author = {} + # maintainer = {} return result LEGACY_MAPPING = { @@ -969,6 +934,7 @@ class Metadata(object): } def _to_legacy(self): + def process_entries(entries): reqts = set() for e in entries: @@ -1037,12 +1003,10 @@ class Metadata(object): else: d = self._data if fileobj: - json.dump(d, fileobj, ensure_ascii=True, indent=2, - sort_keys=True) + json.dump(d, fileobj, ensure_ascii=True, indent=2, sort_keys=True) else: with codecs.open(path, 'w', 'utf-8') as f: - json.dump(d, f, ensure_ascii=True, indent=2, - sort_keys=True) + json.dump(d, f, ensure_ascii=True, indent=2, sort_keys=True) def add_requirements(self, requirements): if self._legacy: @@ -1055,7 +1019,7 @@ class Metadata(object): always = entry break if always is None: - always = { 'requires': requirements } + always = {'requires': requirements} run_requires.insert(0, always) else: rset = set(always['requires']) | set(requirements) @@ -1064,5 +1028,4 @@ class Metadata(object): def __repr__(self): name = self.name or '(no name)' version = self.version or 'no version' - return '<%s %s %s (%s)>' % (self.__class__.__name__, - self.metadata_version, name, version) + return '<%s %s %s (%s)>' % (self.__class__.__name__, self.metadata_version, name, version) diff --git a/contrib/python/pip/pip/_vendor/distlib/scripts.py b/contrib/python/pip/pip/_vendor/distlib/scripts.py index e5681ff72e1..b1fc705b7e6 100644 --- a/contrib/python/pip/pip/_vendor/distlib/scripts.py +++ b/contrib/python/pip/pip/_vendor/distlib/scripts.py @@ -15,8 +15,7 @@ from zipfile import ZipInfo from .compat import sysconfig, detect_encoding, ZipFile from .resources import finder -from .util import (FileOperator, get_export_entry, convert_path, - get_executable, get_platform, in_venv) +from .util import (FileOperator, get_export_entry, convert_path, get_executable, get_platform, in_venv) logger = logging.getLogger(__name__) @@ -57,12 +56,16 @@ if __name__ == '__main__': # location where it was imported from. So we load everything into memory in # advance. -# Issue 31: don't hardcode an absolute package name, but -# determine it relative to the current package -distlib_package = __name__.rsplit('.', 1)[0] +if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'): + # Issue 31: don't hardcode an absolute package name, but + # determine it relative to the current package + DISTLIB_PACKAGE = __name__.rsplit('.', 1)[0] -WRAPPERS = { -} + WRAPPERS = { + r.name: r.bytes + for r in finder(DISTLIB_PACKAGE).iterator("") + if r.name.endswith(".exe") + } def enquote_executable(executable): @@ -94,25 +97,18 @@ class ScriptMaker(object): executable = None # for shebangs - def __init__(self, - source_dir, - target_dir, - add_launchers=True, - dry_run=False, - fileop=None): + def __init__(self, source_dir, target_dir, add_launchers=True, dry_run=False, fileop=None): self.source_dir = source_dir self.target_dir = target_dir self.add_launchers = add_launchers self.force = False self.clobber = False # It only makes sense to set mode bits on POSIX. - self.set_mode = (os.name == 'posix') or (os.name == 'java' - and os._name == 'posix') + self.set_mode = (os.name == 'posix') or (os.name == 'java' and os._name == 'posix') self.variants = set(('', 'X.Y')) self._fileop = fileop or FileOperator(dry_run) - self._is_nt = os.name == 'nt' or (os.name == 'java' - and os._name == 'nt') + self._is_nt = os.name == 'nt' or (os.name == 'java' and os._name == 'nt') self.version_info = sys.version_info def _get_alternate_executable(self, executable, options): @@ -161,6 +157,12 @@ class ScriptMaker(object): """ if os.name != 'posix': simple_shebang = True + elif getattr(sys, "cross_compiling", False): + # In a cross-compiling environment, the shebang will likely be a + # script; this *must* be invoked with the "safe" version of the + # shebang, or else using os.exec() to run the entry script will + # fail, raising "OSError 8 [Errno 8] Exec format error". + simple_shebang = False else: # Add 3 for '#!' prefix and newline suffix. shebang_length = len(executable) + len(post_interp) + 3 @@ -168,15 +170,14 @@ class ScriptMaker(object): max_shebang_length = 512 else: max_shebang_length = 127 - simple_shebang = ((b' ' not in executable) - and (shebang_length <= max_shebang_length)) + simple_shebang = ((b' ' not in executable) and (shebang_length <= max_shebang_length)) if simple_shebang: result = b'#!' + executable + post_interp + b'\n' else: result = b'#!/bin/sh\n' result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n' - result += b"' '''" + result += b"' '''\n" return result def _get_shebang(self, encoding, post_interp=b'', options=None): @@ -187,21 +188,17 @@ class ScriptMaker(object): elif not sysconfig.is_python_build(): executable = get_executable() elif in_venv(): # pragma: no cover - executable = os.path.join( - sysconfig.get_path('scripts'), - 'python%s' % sysconfig.get_config_var('EXE')) + executable = os.path.join(sysconfig.get_path('scripts'), 'python%s' % sysconfig.get_config_var('EXE')) else: # pragma: no cover if os.name == 'nt': # for Python builds from source on Windows, no Python executables with # a version suffix are created, so we use python.exe - executable = os.path.join( - sysconfig.get_config_var('BINDIR'), - 'python%s' % (sysconfig.get_config_var('EXE'))) + executable = os.path.join(sysconfig.get_config_var('BINDIR'), + 'python%s' % (sysconfig.get_config_var('EXE'))) else: executable = os.path.join( sysconfig.get_config_var('BINDIR'), - 'python%s%s' % (sysconfig.get_config_var('VERSION'), - sysconfig.get_config_var('EXE'))) + 'python%s%s' % (sysconfig.get_config_var('VERSION'), sysconfig.get_config_var('EXE'))) if options: executable = self._get_alternate_executable(executable, options) @@ -225,8 +222,8 @@ class ScriptMaker(object): # check that the shebang is decodable using utf-8. executable = executable.encode('utf-8') # in case of IronPython, play safe and enable frames support - if (sys.platform == 'cli' and '-X:Frames' not in post_interp - and '-X:FullFrames' not in post_interp): # pragma: no cover + if (sys.platform == 'cli' and '-X:Frames' not in post_interp and + '-X:FullFrames' not in post_interp): # pragma: no cover post_interp += b' -X:Frames' shebang = self._build_shebang(executable, post_interp) # Python parser starts to read a script using UTF-8 until @@ -237,8 +234,7 @@ class ScriptMaker(object): try: shebang.decode('utf-8') except UnicodeDecodeError: # pragma: no cover - raise ValueError('The shebang (%r) is not decodable from utf-8' % - shebang) + raise ValueError('The shebang (%r) is not decodable from utf-8' % shebang) # If the script is encoded to a custom encoding (use a # #coding:xxx cookie), the shebang has to be decodable from # the script encoding too. @@ -247,15 +243,12 @@ class ScriptMaker(object): shebang.decode(encoding) except UnicodeDecodeError: # pragma: no cover raise ValueError('The shebang (%r) is not decodable ' - 'from the script encoding (%r)' % - (shebang, encoding)) + 'from the script encoding (%r)' % (shebang, encoding)) return shebang def _get_script_text(self, entry): return self.script_template % dict( - module=entry.prefix, - import_name=entry.suffix.split('.')[0], - func=entry.suffix) + module=entry.prefix, import_name=entry.suffix.split('.')[0], func=entry.suffix) manifest = _DEFAULT_MANIFEST @@ -265,9 +258,6 @@ class ScriptMaker(object): def _write_script(self, names, shebang, script_bytes, filenames, ext): use_launcher = self.add_launchers and self._is_nt - linesep = os.linesep.encode('utf-8') - if not shebang.endswith(linesep): - shebang += linesep if not use_launcher: script_bytes = shebang + script_bytes else: # pragma: no cover @@ -280,8 +270,7 @@ class ScriptMaker(object): source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') if source_date_epoch: date_time = time.gmtime(int(source_date_epoch))[:6] - zinfo = ZipInfo(filename='__main__.py', - date_time=date_time) + zinfo = ZipInfo(filename='__main__.py', date_time=date_time) zf.writestr(zinfo, script_bytes) else: zf.writestr('__main__.py', script_bytes) @@ -312,8 +301,7 @@ class ScriptMaker(object): except Exception: pass # still in use - ignore error else: - if self._is_nt and not outname.endswith( - '.' + ext): # pragma: no cover + if self._is_nt and not outname.endswith('.' + ext): # pragma: no cover outname = '%s.%s' % (outname, ext) if os.path.exists(outname) and not self.clobber: logger.warning('Skipping existing file %s', outname) @@ -332,9 +320,7 @@ class ScriptMaker(object): if 'X' in self.variants: result.add('%s%s' % (name, self.version_info[0])) if 'X.Y' in self.variants: - result.add('%s%s%s.%s' % - (name, self.variant_separator, self.version_info[0], - self.version_info[1])) + result.add('%s%s%s.%s' % (name, self.variant_separator, self.version_info[0], self.version_info[1])) return result def _make_script(self, entry, filenames, options=None): @@ -389,8 +375,7 @@ class ScriptMaker(object): self._fileop.set_executable_mode([outname]) filenames.append(outname) else: - logger.info('copying and adjusting %s -> %s', script, - self.target_dir) + logger.info('copying and adjusting %s -> %s', script, self.target_dir) if not self._fileop.dry_run: encoding, lines = detect_encoding(f.readline) f.seek(0) @@ -412,8 +397,7 @@ class ScriptMaker(object): def dry_run(self, value): self._fileop.dry_run = value - if os.name == 'nt' or (os.name == 'java' - and os._name == 'nt'): # pragma: no cover + if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'): # pragma: no cover # Executable launcher support. # Launchers are from https://bitbucket.org/vinay.sajip/simple_launcher/ @@ -426,7 +410,7 @@ class ScriptMaker(object): name = '%s%s%s.exe' % (kind, bits, platform_suffix) if name not in WRAPPERS: msg = ('Unable to find resource %s in package %s' % - (name, distlib_package)) + (name, DISTLIB_PACKAGE)) raise ValueError(msg) return WRAPPERS[name] diff --git a/contrib/python/pip/pip/_vendor/distlib/util.py b/contrib/python/pip/pip/_vendor/distlib/util.py index ba58858d0fb..0d5bd7a8bf3 100644 --- a/contrib/python/pip/pip/_vendor/distlib/util.py +++ b/contrib/python/pip/pip/_vendor/distlib/util.py @@ -31,11 +31,9 @@ except ImportError: # pragma: no cover import time from . import DistlibException -from .compat import (string_types, text_type, shutil, raw_input, StringIO, - cache_from_source, urlopen, urljoin, httplib, xmlrpclib, - HTTPHandler, BaseConfigurator, valid_ident, - Container, configparser, URLError, ZipFile, fsdecode, - unquote, urlparse) +from .compat import (string_types, text_type, shutil, raw_input, StringIO, cache_from_source, urlopen, urljoin, httplib, + xmlrpclib, HTTPHandler, BaseConfigurator, valid_ident, Container, configparser, URLError, ZipFile, + fsdecode, unquote, urlparse) logger = logging.getLogger(__name__) @@ -88,8 +86,7 @@ def parse_marker(marker_string): else: m = STRING_CHUNK.match(remaining) if not m: - raise SyntaxError('error in string literal: %s' % - remaining) + raise SyntaxError('error in string literal: %s' % remaining) parts.append(m.groups()[0]) remaining = remaining[m.end():] else: @@ -210,8 +207,7 @@ def parse_requirement(req): ver_remaining = ver_remaining[m.end():] m = VERSION_IDENTIFIER.match(ver_remaining) if not m: - raise SyntaxError('invalid version: %s' % - ver_remaining) + raise SyntaxError('invalid version: %s' % ver_remaining) v = m.groups()[0] versions.append((op, v)) ver_remaining = ver_remaining[m.end():] @@ -224,8 +220,7 @@ def parse_requirement(req): break m = COMPARE_OP.match(ver_remaining) if not m: - raise SyntaxError('invalid constraint: %s' % - ver_remaining) + raise SyntaxError('invalid constraint: %s' % ver_remaining) if not versions: versions = None return versions, ver_remaining @@ -235,8 +230,7 @@ def parse_requirement(req): else: i = remaining.find(')', 1) if i < 0: - raise SyntaxError('unterminated parenthesis: %s' % - remaining) + raise SyntaxError('unterminated parenthesis: %s' % remaining) s = remaining[1:i] remaining = remaining[i + 1:].lstrip() # As a special diversion from PEP 508, allow a version number @@ -267,14 +261,8 @@ def parse_requirement(req): if not versions: rs = distname else: - rs = '%s %s' % (distname, ', '.join( - ['%s %s' % con for con in versions])) - return Container(name=distname, - extras=extras, - constraints=versions, - marker=mark_expr, - url=uri, - requirement=rs) + rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions])) + return Container(name=distname, extras=extras, constraints=versions, marker=mark_expr, url=uri, requirement=rs) def get_resources_dests(resources_root, rules): @@ -524,8 +512,7 @@ class FileOperator(object): second will have the same "age". """ if not os.path.exists(source): - raise DistlibException("file '%r' does not exist" % - os.path.abspath(source)) + raise DistlibException("file '%r' does not exist" % os.path.abspath(source)) if not os.path.exists(target): return True @@ -601,12 +588,7 @@ class FileOperator(object): if self.record: self.dirs_created.add(path) - def byte_compile(self, - path, - optimize=False, - force=False, - prefix=None, - hashed_invalidation=False): + def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): dpath = cache_from_source(path, not optimize) logger.info('Byte-compiling %s to %s', path, dpath) if not self.dry_run: @@ -617,12 +599,11 @@ class FileOperator(object): assert path.startswith(prefix) diagpath = path[len(prefix):] compile_kwargs = {} - if hashed_invalidation and hasattr(py_compile, - 'PycInvalidationMode'): - compile_kwargs[ - 'invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH - py_compile.compile(path, dpath, diagpath, True, - **compile_kwargs) # raise error + if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): + if not isinstance(hashed_invalidation, py_compile.PycInvalidationMode): + hashed_invalidation = py_compile.PycInvalidationMode.CHECKED_HASH + compile_kwargs['invalidation_mode'] = hashed_invalidation + py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error self.record_as_written(dpath) return dpath @@ -716,16 +697,14 @@ class ExportEntry(object): return resolve(self.prefix, self.suffix) def __repr__(self): # pragma: no cover - return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix, - self.suffix, self.flags) + return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix, self.suffix, self.flags) def __eq__(self, other): if not isinstance(other, ExportEntry): result = False else: - result = (self.name == other.name and self.prefix == other.prefix - and self.suffix == other.suffix - and self.flags == other.flags) + result = (self.name == other.name and self.prefix == other.prefix and self.suffix == other.suffix and + self.flags == other.flags) return result __hash__ = object.__hash__ @@ -810,7 +789,7 @@ def get_cache_base(suffix=None): return os.path.join(result, suffix) -def path_to_cache_dir(path): +def path_to_cache_dir(path, use_abspath=True): """ Convert an absolute path to a directory name for use in a cache. @@ -820,7 +799,7 @@ def path_to_cache_dir(path): #. Any occurrence of ``os.sep`` is replaced with ``'--'``. #. ``'.cache'`` is appended. """ - d, p = os.path.splitdrive(os.path.abspath(path)) + d, p = os.path.splitdrive(os.path.abspath(path) if use_abspath else path) if d: d = d.replace(':', '---') p = p.replace(os.sep, '--') @@ -865,9 +844,8 @@ def is_string_sequence(seq): return result -PROJECT_NAME_AND_VERSION = re.compile( - '([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-' - '([a-z0-9_.+-]+)', re.I) +PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-' + '([a-z0-9_.+-]+)', re.I) PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)') @@ -1003,11 +981,11 @@ class Cache(object): logger.warning('Directory \'%s\' is not private', base) self.base = os.path.abspath(os.path.normpath(base)) - def prefix_to_dir(self, prefix): + def prefix_to_dir(self, prefix, use_abspath=True): """ Converts a resource prefix to a directory name in the cache. """ - return path_to_cache_dir(prefix) + return path_to_cache_dir(prefix, use_abspath=use_abspath) def clear(self): """ @@ -1092,8 +1070,7 @@ class EventMixin(object): logger.exception('Exception during event publication') value = None result.append(value) - logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, - args, kwargs, result) + logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, args, kwargs, result) return result @@ -1145,8 +1122,7 @@ class Sequencer(object): raise ValueError('%r not a successor of %r' % (succ, pred)) def is_step(self, step): - return (step in self._preds or step in self._succs - or step in self._nodes) + return (step in self._preds or step in self._succs or step in self._nodes) def get_steps(self, final): if not self.is_step(final): @@ -1242,8 +1218,7 @@ class Sequencer(object): # Unarchiving functionality for zip, tar, tgz, tbz, whl # -ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', - '.whl') +ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', '.whl') def unarchive(archive_filename, dest_dir, format=None, check=True): @@ -1474,8 +1449,7 @@ def _iglob(path_glob): if ssl: - from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, - CertificateError) + from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, CertificateError) # # HTTPSConnection which verifies certificates/matches domains @@ -1487,8 +1461,7 @@ if ssl: # noinspection PyPropertyAccess def connect(self): - sock = socket.create_connection((self.host, self.port), - self.timeout) + sock = socket.create_connection((self.host, self.port), self.timeout) if getattr(self, '_tunnel_host', False): self.sock = sock self._tunnel() @@ -1543,9 +1516,8 @@ if ssl: return self.do_open(self._conn_maker, req) except URLError as e: if 'certificate verify failed' in str(e.reason): - raise CertificateError( - 'Unable to verify server certificate ' - 'for %s' % req.host) + raise CertificateError('Unable to verify server certificate ' + 'for %s' % req.host) else: raise @@ -1561,9 +1533,8 @@ if ssl: class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler): def http_open(self, req): - raise URLError( - 'Unexpected HTTP request on what should be a secure ' - 'connection: %s' % req) + raise URLError('Unexpected HTTP request on what should be a secure ' + 'connection: %s' % req) # @@ -1598,8 +1569,7 @@ if ssl: kwargs['timeout'] = self.timeout if not self._connection or host != self._connection[0]: self._extra_headers = eh - self._connection = host, httplib.HTTPSConnection( - h, None, **kwargs) + self._connection = host, httplib.HTTPSConnection(h, None, **kwargs) return self._connection[1] @@ -1789,10 +1759,7 @@ class SubprocessMixin(object): stream.close() def run_command(self, cmd, **kwargs): - p = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - **kwargs) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout')) t1.start() t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr')) @@ -1847,10 +1814,7 @@ class PyPIRCFile(object): if 'distutils' in sections: # let's get the list of servers index_servers = config.get('distutils', 'index-servers') - _servers = [ - server.strip() for server in index_servers.split('\n') - if server.strip() != '' - ] + _servers = [server.strip() for server in index_servers.split('\n') if server.strip() != ''] if _servers == []: # nothing set, let's try to get the default pypi if 'pypi' in sections: @@ -1861,9 +1825,7 @@ class PyPIRCFile(object): result['username'] = config.get(server, 'username') # optional params - for key, default in (('repository', - self.DEFAULT_REPOSITORY), - ('realm', self.DEFAULT_REALM), + for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM), ('password', None)): if config.has_option(server, key): result[key] = config.get(server, key) @@ -1873,11 +1835,9 @@ class PyPIRCFile(object): # work around people having "repository" for the "pypi" # section of their config set to the HTTP (rather than # HTTPS) URL - if (server == 'pypi' and repository - in (self.DEFAULT_REPOSITORY, 'pypi')): + if (server == 'pypi' and repository in (self.DEFAULT_REPOSITORY, 'pypi')): result['repository'] = self.DEFAULT_REPOSITORY - elif (result['server'] != repository - and result['repository'] != repository): + elif (result['server'] != repository and result['repository'] != repository): result = {} elif 'server-login' in sections: # old format @@ -2003,8 +1963,7 @@ def get_host_platform(): from distutils import sysconfig except ImportError: import sysconfig - osname, release, machine = _osx_support.get_platform_osx( - sysconfig.get_config_vars(), osname, release, machine) + osname, release, machine = _osx_support.get_platform_osx(sysconfig.get_config_vars(), osname, release, machine) return '%s-%s-%s' % (osname, release, machine) diff --git a/contrib/python/pip/pip/_vendor/distlib/version.py b/contrib/python/pip/pip/_vendor/distlib/version.py index 14171ac938d..d70a96ef51e 100644 --- a/contrib/python/pip/pip/_vendor/distlib/version.py +++ b/contrib/python/pip/pip/_vendor/distlib/version.py @@ -619,8 +619,7 @@ class LegacyVersion(Version): def is_prerelease(self): result = False for x in self._parts: - if (isinstance(x, string_types) and x.startswith('*') and - x < '*final'): + if (isinstance(x, string_types) and x.startswith('*') and x < '*final'): result = True break return result diff --git a/contrib/python/pip/pip/_vendor/distlib/wheel.py b/contrib/python/pip/pip/_vendor/distlib/wheel.py index 4a5a30e1d8d..62ab10fb3ad 100644 --- a/contrib/python/pip/pip/_vendor/distlib/wheel.py +++ b/contrib/python/pip/pip/_vendor/distlib/wheel.py @@ -25,9 +25,8 @@ from . import __version__, DistlibException from .compat import sysconfig, ZipFile, fsdecode, text_type, filter from .database import InstalledDistribution from .metadata import Metadata, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME -from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache, - cached_property, get_cache_base, read_exports, tempdir, - get_platform) +from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache, cached_property, get_cache_base, + read_exports, tempdir, get_platform) from .version import NormalizedVersion, UnsupportedVersionError logger = logging.getLogger(__name__) @@ -88,8 +87,7 @@ FILENAME_RE = re.compile( \.whl$ ''', re.IGNORECASE | re.VERBOSE) -NAME_VERSION_RE = re.compile( - r''' +NAME_VERSION_RE = re.compile(r''' (?P<nm>[^-]+) -(?P<vn>\d+[^-]*) (-(?P<bn>\d+[^-]*))?$ @@ -235,8 +233,7 @@ class Wheel(object): arch = '.'.join(self.arch) # replace - with _ as a local version separator version = self.version.replace('-', '_') - return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, pyver, - abi, arch) + return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, pyver, abi, arch) @property def exists(self): @@ -334,8 +331,7 @@ class Wheel(object): try: hasher = getattr(hashlib, hash_kind) except AttributeError: - raise DistlibException('Unsupported hash algorithm: %r' % - hash_kind) + raise DistlibException('Unsupported hash algorithm: %r' % hash_kind) result = hasher(data).digest() result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii') return hash_kind, result @@ -513,7 +509,7 @@ class Wheel(object): installed, and the headers, scripts, data and dist-info metadata are not written. If kwarg ``bytecode_hashed_invalidation`` is True, written bytecode will try to use file-hash based invalidation (PEP-552) on - supported interpreter versions (CPython 2.7+). + supported interpreter versions (CPython 3.7+). The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. @@ -522,8 +518,7 @@ class Wheel(object): dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) - bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', - False) + bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -602,8 +597,7 @@ class Wheel(object): if lib_only and u_arcname.startswith((info_pfx, data_pfx)): logger.debug('lib_only: skipping %s', u_arcname) continue - is_script = (u_arcname.startswith(script_pfx) - and not u_arcname.endswith('.exe')) + is_script = (u_arcname.startswith(script_pfx) and not u_arcname.endswith('.exe')) if u_arcname.startswith(data_pfx): _, where, rp = u_arcname.split('/', 2) @@ -622,8 +616,7 @@ class Wheel(object): # So ... manually preserve permission bits as given in zinfo if os.name == 'posix': # just set the normal permission bits - os.chmod(outfile, - (zinfo.external_attr >> 16) & 0x1FF) + os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF) outfiles.append(outfile) # Double check the digest of the written file if not dry_run and row[1]: @@ -636,15 +629,12 @@ class Wheel(object): '%s' % outfile) if bc and outfile.endswith('.py'): try: - pyc = fileop.byte_compile( - outfile, - hashed_invalidation=bc_hashed_invalidation) + pyc = fileop.byte_compile(outfile, hashed_invalidation=bc_hashed_invalidation) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, # but log it and perhaps warn the user - logger.warning('Byte-compilation failed', - exc_info=True) + logger.warning('Byte-compilation failed', exc_info=True) else: fn = os.path.basename(convert_path(arcname)) workname = os.path.join(workdir, fn) @@ -732,8 +722,7 @@ class Wheel(object): outfiles.append(p) # Write RECORD - dist.write_installed_files(outfiles, paths['prefix'], - dry_run) + dist.write_installed_files(outfiles, paths['prefix'], dry_run) return dist except Exception: # pragma: no cover logger.exception('installation failed.') @@ -746,8 +735,7 @@ class Wheel(object): global cache if cache is None: # Use native string to avoid issues on 2.x: see Python #20140. - base = os.path.join(get_cache_base(), str('dylib-cache'), - '%s.%s' % sys.version_info[:2]) + base = os.path.join(get_cache_base(), str('dylib-cache'), '%s.%s' % sys.version_info[:2]) cache = Cache(base) return cache @@ -764,7 +752,7 @@ class Wheel(object): wf = wrapper(bf) extensions = json.load(wf) cache = self._get_dylib_cache() - prefix = cache.prefix_to_dir(pathname) + prefix = cache.prefix_to_dir(self.filename, use_abspath=False) cache_base = os.path.join(cache.base, prefix) if not os.path.isdir(cache_base): os.makedirs(cache_base) @@ -774,8 +762,7 @@ class Wheel(object): extract = True else: file_time = os.stat(dest).st_mtime - file_time = datetime.datetime.fromtimestamp( - file_time) + file_time = datetime.datetime.fromtimestamp(file_time) info = zf.getinfo(relpath) wheel_time = datetime.datetime(*info.date_time) extract = wheel_time > file_time @@ -924,12 +911,10 @@ class Wheel(object): else: parts = [int(s) for s in version[i + 1:].split('.')] parts[-1] += 1 - updated = '%s+%s' % (version[:i], '.'.join( - str(i) for i in parts)) + updated = '%s+%s' % (version[:i], '.'.join(str(i) for i in parts)) except UnsupportedVersionError: - logger.debug( - 'Cannot update non-compliant (PEP-440) ' - 'version %r', version) + logger.debug('Cannot update non-compliant (PEP-440) ' + 'version %r', version) if updated: md = Metadata(path=path) md.version = updated @@ -971,14 +956,11 @@ class Wheel(object): update_version(current_version, path) # Decide where the new wheel goes. if dest_dir is None: - fd, newpath = tempfile.mkstemp(suffix='.whl', - prefix='wheel-update-', - dir=workdir) + fd, newpath = tempfile.mkstemp(suffix='.whl', prefix='wheel-update-', dir=workdir) os.close(fd) else: if not os.path.isdir(dest_dir): - raise DistlibException('Not a directory: %r' % - dest_dir) + raise DistlibException('Not a directory: %r' % dest_dir) newpath = os.path.join(dest_dir, self.filename) archive_paths = list(path_map.items()) distinfo = os.path.join(workdir, info_dir) @@ -1005,11 +987,20 @@ def compatible_tags(): """ Return (pyver, abi, arch) tuples compatible with this Python. """ - versions = [VER_SUFFIX] - major = VER_SUFFIX[0] - for minor in range(sys.version_info[1] - 1, -1, -1): - versions.append(''.join([major, str(minor)])) + class _Version: + def __init__(self, major, minor): + self.major = major + self.major_minor = (major, minor) + self.string = ''.join((str(major), str(minor))) + + def __str__(self): + return self.string + + versions = [ + _Version(sys.version_info.major, minor_version) + for minor_version in range(sys.version_info.minor, -1, -1) + ] abis = [] for suffix in _get_suffixes(): if suffix.startswith('.abi'): @@ -1045,35 +1036,45 @@ def compatible_tags(): minor -= 1 # Most specific - our Python version, ABI and arch - for abi in abis: - for arch in arches: - result.append((''.join((IMP_PREFIX, versions[0])), abi, arch)) - # manylinux - if abi != 'none' and sys.platform.startswith('linux'): - arch = arch.replace('linux_', '') - parts = _get_glibc_version() - if len(parts) == 2: - if parts >= (2, 5): - result.append((''.join((IMP_PREFIX, versions[0])), abi, - 'manylinux1_%s' % arch)) - if parts >= (2, 12): - result.append((''.join((IMP_PREFIX, versions[0])), abi, - 'manylinux2010_%s' % arch)) - if parts >= (2, 17): - result.append((''.join((IMP_PREFIX, versions[0])), abi, - 'manylinux2014_%s' % arch)) - result.append( - (''.join((IMP_PREFIX, versions[0])), abi, - 'manylinux_%s_%s_%s' % (parts[0], parts[1], arch))) + for i, version_object in enumerate(versions): + version = str(version_object) + add_abis = [] + + if i == 0: + add_abis = abis + + if IMP_PREFIX == 'cp' and version_object.major_minor >= (3, 2): + limited_api_abi = 'abi' + str(version_object.major) + if limited_api_abi not in add_abis: + add_abis.append(limited_api_abi) + + for abi in add_abis: + for arch in arches: + result.append((''.join((IMP_PREFIX, version)), abi, arch)) + # manylinux + if abi != 'none' and sys.platform.startswith('linux'): + arch = arch.replace('linux_', '') + parts = _get_glibc_version() + if len(parts) == 2: + if parts >= (2, 5): + result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux1_%s' % arch)) + if parts >= (2, 12): + result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux2010_%s' % arch)) + if parts >= (2, 17): + result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux2014_%s' % arch)) + result.append((''.join( + (IMP_PREFIX, version)), abi, 'manylinux_%s_%s_%s' % (parts[0], parts[1], arch))) # where no ABI / arch dependency, but IMP_PREFIX dependency - for i, version in enumerate(versions): + for i, version_object in enumerate(versions): + version = str(version_object) result.append((''.join((IMP_PREFIX, version)), 'none', 'any')) if i == 0: result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any')) # no IMP_PREFIX, ABI or arch dependency - for i, version in enumerate(versions): + for i, version_object in enumerate(versions): + version = str(version_object) result.append((''.join(('py', version)), 'none', 'any')) if i == 0: result.append((''.join(('py', version[0])), 'none', 'any')) diff --git a/contrib/python/pip/pip/_vendor/packaging/tags.py b/contrib/python/pip/pip/_vendor/packaging/tags.py index 6667d299085..703f0ed53c0 100644 --- a/contrib/python/pip/pip/_vendor/packaging/tags.py +++ b/contrib/python/pip/pip/_vendor/packaging/tags.py @@ -25,7 +25,7 @@ from . import _manylinux, _musllinux logger = logging.getLogger(__name__) PythonVersion = Sequence[int] -MacVersion = Tuple[int, int] +AppleVersion = Tuple[int, int] INTERPRETER_SHORT_NAMES: dict[str, str] = { "python": "py", # Generic. @@ -363,7 +363,7 @@ def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str: return "i386" -def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> list[str]: +def _mac_binary_formats(version: AppleVersion, cpu_arch: str) -> list[str]: formats = [cpu_arch] if cpu_arch == "x86_64": if version < (10, 4): @@ -396,7 +396,7 @@ def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> list[str]: def mac_platforms( - version: MacVersion | None = None, arch: str | None = None + version: AppleVersion | None = None, arch: str | None = None ) -> Iterator[str]: """ Yields the platform tags for a macOS system. @@ -408,7 +408,7 @@ def mac_platforms( """ version_str, _, cpu_arch = platform.mac_ver() if version is None: - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) + version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2]))) if version == (10, 16): # When built against an older macOS SDK, Python will report macOS 10.16 # instead of the real version. @@ -424,7 +424,7 @@ def mac_platforms( stdout=subprocess.PIPE, text=True, ).stdout - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) + version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2]))) else: version = version if arch is None: @@ -483,6 +483,63 @@ def mac_platforms( ) +def ios_platforms( + version: AppleVersion | None = None, multiarch: str | None = None +) -> Iterator[str]: + """ + Yields the platform tags for an iOS system. + + :param version: A two-item tuple specifying the iOS version to generate + platform tags for. Defaults to the current iOS version. + :param multiarch: The CPU architecture+ABI to generate platform tags for - + (the value used by `sys.implementation._multiarch` e.g., + `arm64_iphoneos` or `x84_64_iphonesimulator`). Defaults to the current + multiarch value. + """ + if version is None: + # if iOS is the current platform, ios_ver *must* be defined. However, + # it won't exist for CPython versions before 3.13, which causes a mypy + # error. + _, release, _, _ = platform.ios_ver() # type: ignore[attr-defined] + version = cast("AppleVersion", tuple(map(int, release.split(".")[:2]))) + + if multiarch is None: + multiarch = sys.implementation._multiarch + multiarch = multiarch.replace("-", "_") + + ios_platform_template = "ios_{major}_{minor}_{multiarch}" + + # Consider any iOS major.minor version from the version requested, down to + # 12.0. 12.0 is the first iOS version that is known to have enough features + # to support CPython. Consider every possible minor release up to X.9. There + # highest the minor has ever gone is 8 (14.8 and 15.8) but having some extra + # candidates that won't ever match doesn't really hurt, and it saves us from + # having to keep an explicit list of known iOS versions in the code. Return + # the results descending order of version number. + + # If the requested major version is less than 12, there won't be any matches. + if version[0] < 12: + return + + # Consider the actual X.Y version that was requested. + yield ios_platform_template.format( + major=version[0], minor=version[1], multiarch=multiarch + ) + + # Consider every minor version from X.0 to the minor version prior to the + # version requested by the platform. + for minor in range(version[1] - 1, -1, -1): + yield ios_platform_template.format( + major=version[0], minor=minor, multiarch=multiarch + ) + + for major in range(version[0] - 1, 11, -1): + for minor in range(9, -1, -1): + yield ios_platform_template.format( + major=major, minor=minor, multiarch=multiarch + ) + + def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: linux = _normalize_string(sysconfig.get_platform()) if not linux.startswith("linux_"): @@ -512,6 +569,8 @@ def platform_tags() -> Iterator[str]: """ if platform.system() == "Darwin": return mac_platforms() + elif platform.system() == "iOS": + return ios_platforms() elif platform.system() == "Linux": return _linux_platforms() else: diff --git a/contrib/python/pip/pip/_vendor/truststore/__init__.py b/contrib/python/pip/pip/_vendor/truststore/__init__.py index 86368145a7e..e468bf8cebd 100644 --- a/contrib/python/pip/pip/_vendor/truststore/__init__.py +++ b/contrib/python/pip/pip/_vendor/truststore/__init__.py @@ -5,9 +5,32 @@ import sys as _sys if _sys.version_info < (3, 10): raise ImportError("truststore requires Python 3.10 or later") +# Detect Python runtimes which don't implement SSLObject.get_unverified_chain() API +# This API only became public in Python 3.13 but was available in CPython and PyPy since 3.10. +if _sys.version_info < (3, 13): + try: + import ssl as _ssl + except ImportError: + raise ImportError("truststore requires the 'ssl' module") + else: + _sslmem = _ssl.MemoryBIO() + _sslobj = _ssl.create_default_context().wrap_bio( + _sslmem, + _sslmem, + ) + try: + while not hasattr(_sslobj, "get_unverified_chain"): + _sslobj = _sslobj._sslobj # type: ignore[attr-defined] + except AttributeError: + raise ImportError( + "truststore requires peer certificate chain APIs to be available" + ) from None + + del _ssl, _sslobj, _sslmem # noqa: F821 + from ._api import SSLContext, extract_from_ssl, inject_into_ssl # noqa: E402 del _api, _sys # type: ignore[name-defined] # noqa: F821 __all__ = ["SSLContext", "inject_into_ssl", "extract_from_ssl"] -__version__ = "0.9.1" +__version__ = "0.10.0" diff --git a/contrib/python/pip/pip/_vendor/truststore/_api.py b/contrib/python/pip/pip/_vendor/truststore/_api.py index b1ea3b05c6d..aeb023af756 100644 --- a/contrib/python/pip/pip/_vendor/truststore/_api.py +++ b/contrib/python/pip/pip/_vendor/truststore/_api.py @@ -169,6 +169,9 @@ class SSLContext(_truststore_SSLContext_super_class): # type: ignore[misc] def cert_store_stats(self) -> dict[str, int]: raise NotImplementedError() + def set_default_verify_paths(self) -> None: + self._ctx.set_default_verify_paths() + @typing.overload def get_ca_certs( self, binary_form: typing.Literal[False] = ... diff --git a/contrib/python/pip/pip/_vendor/truststore/_macos.py b/contrib/python/pip/pip/_vendor/truststore/_macos.py index b234ffec723..34503077244 100644 --- a/contrib/python/pip/pip/_vendor/truststore/_macos.py +++ b/contrib/python/pip/pip/_vendor/truststore/_macos.py @@ -25,6 +25,8 @@ if _mac_version_info < (10, 8): f"Only OS X 10.8 and newer are supported, not {_mac_version_info[0]}.{_mac_version_info[1]}" ) +_is_macos_version_10_14_or_later = _mac_version_info >= (10, 14) + def _load_cdll(name: str, macos10_16_path: str) -> CDLL: """Loads a CDLL by name, falling back to known path on 10.16+""" @@ -115,6 +117,12 @@ try: ] Security.SecTrustGetTrustResult.restype = OSStatus + Security.SecTrustEvaluate.argtypes = [ + SecTrustRef, + POINTER(SecTrustResultType), + ] + Security.SecTrustEvaluate.restype = OSStatus + Security.SecTrustRef = SecTrustRef # type: ignore[attr-defined] Security.SecTrustResultType = SecTrustResultType # type: ignore[attr-defined] Security.OSStatus = OSStatus # type: ignore[attr-defined] @@ -197,8 +205,19 @@ try: CoreFoundation.CFStringRef = CFStringRef # type: ignore[attr-defined] CoreFoundation.CFErrorRef = CFErrorRef # type: ignore[attr-defined] -except AttributeError: - raise ImportError("Error initializing ctypes") from None +except AttributeError as e: + raise ImportError(f"Error initializing ctypes: {e}") from None + +# SecTrustEvaluateWithError is macOS 10.14+ +if _is_macos_version_10_14_or_later: + try: + Security.SecTrustEvaluateWithError.argtypes = [ + SecTrustRef, + POINTER(CFErrorRef), + ] + Security.SecTrustEvaluateWithError.restype = c_bool + except AttributeError as e: + raise ImportError(f"Error initializing ctypes: {e}") from None def _handle_osstatus(result: OSStatus, _: typing.Any, args: typing.Any) -> typing.Any: @@ -258,6 +277,7 @@ Security.SecTrustCreateWithCertificates.errcheck = _handle_osstatus # type: ign Security.SecTrustSetAnchorCertificates.errcheck = _handle_osstatus # type: ignore[assignment] Security.SecTrustSetAnchorCertificatesOnly.errcheck = _handle_osstatus # type: ignore[assignment] Security.SecTrustGetTrustResult.errcheck = _handle_osstatus # type: ignore[assignment] +Security.SecTrustEvaluate.errcheck = _handle_osstatus # type: ignore[assignment] class CFConst: @@ -365,9 +385,10 @@ def _verify_peercerts_impl( certs = None policies = None trust = None - cf_error = None try: - if server_hostname is not None: + # Only set a hostname on the policy if we're verifying the hostname + # on the leaf certificate. + if server_hostname is not None and ssl_context.check_hostname: cf_str_hostname = None try: cf_str_hostname = _bytes_to_cf_string(server_hostname.encode("ascii")) @@ -431,69 +452,120 @@ def _verify_peercerts_impl( # We always want system certificates. Security.SecTrustSetAnchorCertificatesOnly(trust, False) - cf_error = CoreFoundation.CFErrorRef() - sec_trust_eval_result = Security.SecTrustEvaluateWithError( - trust, ctypes.byref(cf_error) - ) - # sec_trust_eval_result is a bool (0 or 1) - # where 1 means that the certs are trusted. - if sec_trust_eval_result == 1: - is_trusted = True - elif sec_trust_eval_result == 0: - is_trusted = False + # macOS 10.13 and earlier don't support SecTrustEvaluateWithError() + # so we use SecTrustEvaluate() which means we need to construct error + # messages ourselves. + if _is_macos_version_10_14_or_later: + _verify_peercerts_impl_macos_10_14(ssl_context, trust) else: - raise ssl.SSLError( - f"Unknown result from Security.SecTrustEvaluateWithError: {sec_trust_eval_result!r}" - ) + _verify_peercerts_impl_macos_10_13(ssl_context, trust) + finally: + if policies: + CoreFoundation.CFRelease(policies) + if trust: + CoreFoundation.CFRelease(trust) - cf_error_code = 0 - if not is_trusted: - cf_error_code = CoreFoundation.CFErrorGetCode(cf_error) - # If the error is a known failure that we're - # explicitly okay with from SSLContext configuration - # we can set is_trusted accordingly. - if ssl_context.verify_mode != ssl.CERT_REQUIRED and ( - cf_error_code == CFConst.errSecNotTrusted - or cf_error_code == CFConst.errSecCertificateExpired - ): - is_trusted = True - elif ( - not ssl_context.check_hostname - and cf_error_code == CFConst.errSecHostNameMismatch - ): - is_trusted = True +def _verify_peercerts_impl_macos_10_13( + ssl_context: ssl.SSLContext, sec_trust_ref: typing.Any +) -> None: + """Verify using 'SecTrustEvaluate' API for macOS 10.13 and earlier. + macOS 10.14 added the 'SecTrustEvaluateWithError' API. + """ + sec_trust_result_type = Security.SecTrustResultType() + Security.SecTrustEvaluate(sec_trust_ref, ctypes.byref(sec_trust_result_type)) - # If we're still not trusted then we start to - # construct and raise the SSLCertVerificationError. - if not is_trusted: - cf_error_string_ref = None - try: - cf_error_string_ref = CoreFoundation.CFErrorCopyDescription(cf_error) + try: + sec_trust_result_type_as_int = int(sec_trust_result_type.value) + except (ValueError, TypeError): + sec_trust_result_type_as_int = -1 + + # Apple doesn't document these values in their own API docs. + # See: https://github.com/xybp888/iOS-SDKs/blob/master/iPhoneOS13.0.sdk/System/Library/Frameworks/Security.framework/Headers/SecTrust.h#L84 + if ( + ssl_context.verify_mode == ssl.CERT_REQUIRED + and sec_trust_result_type_as_int not in (1, 4) + ): + # Note that we're not able to ignore only hostname errors + # for macOS 10.13 and earlier, so check_hostname=False will + # still return an error. + sec_trust_result_type_to_message = { + 0: "Invalid trust result type", + # 1: "Trust evaluation succeeded", + 2: "User confirmation required", + 3: "User specified that certificate is not trusted", + # 4: "Trust result is unspecified", + 5: "Recoverable trust failure occurred", + 6: "Fatal trust failure occurred", + 7: "Other error occurred, certificate may be revoked", + } + error_message = sec_trust_result_type_to_message.get( + sec_trust_result_type_as_int, + f"Unknown trust result: {sec_trust_result_type_as_int}", + ) - # Can this ever return 'None' if there's a CFError? - cf_error_message = ( - _cf_string_ref_to_str(cf_error_string_ref) - or "Certificate verification failed" - ) + err = ssl.SSLCertVerificationError(error_message) + err.verify_message = error_message + err.verify_code = sec_trust_result_type_as_int + raise err - # TODO: Not sure if we need the SecTrustResultType for anything? - # We only care whether or not it's a success or failure for now. - sec_trust_result_type = Security.SecTrustResultType() - Security.SecTrustGetTrustResult( - trust, ctypes.byref(sec_trust_result_type) - ) - err = ssl.SSLCertVerificationError(cf_error_message) - err.verify_message = cf_error_message - err.verify_code = cf_error_code - raise err - finally: - if cf_error_string_ref: - CoreFoundation.CFRelease(cf_error_string_ref) +def _verify_peercerts_impl_macos_10_14( + ssl_context: ssl.SSLContext, sec_trust_ref: typing.Any +) -> None: + """Verify using 'SecTrustEvaluateWithError' API for macOS 10.14+.""" + cf_error = CoreFoundation.CFErrorRef() + sec_trust_eval_result = Security.SecTrustEvaluateWithError( + sec_trust_ref, ctypes.byref(cf_error) + ) + # sec_trust_eval_result is a bool (0 or 1) + # where 1 means that the certs are trusted. + if sec_trust_eval_result == 1: + is_trusted = True + elif sec_trust_eval_result == 0: + is_trusted = False + else: + raise ssl.SSLError( + f"Unknown result from Security.SecTrustEvaluateWithError: {sec_trust_eval_result!r}" + ) - finally: - if policies: - CoreFoundation.CFRelease(policies) - if trust: - CoreFoundation.CFRelease(trust) + cf_error_code = 0 + if not is_trusted: + cf_error_code = CoreFoundation.CFErrorGetCode(cf_error) + + # If the error is a known failure that we're + # explicitly okay with from SSLContext configuration + # we can set is_trusted accordingly. + if ssl_context.verify_mode != ssl.CERT_REQUIRED and ( + cf_error_code == CFConst.errSecNotTrusted + or cf_error_code == CFConst.errSecCertificateExpired + ): + is_trusted = True + + # If we're still not trusted then we start to + # construct and raise the SSLCertVerificationError. + if not is_trusted: + cf_error_string_ref = None + try: + cf_error_string_ref = CoreFoundation.CFErrorCopyDescription(cf_error) + + # Can this ever return 'None' if there's a CFError? + cf_error_message = ( + _cf_string_ref_to_str(cf_error_string_ref) + or "Certificate verification failed" + ) + + # TODO: Not sure if we need the SecTrustResultType for anything? + # We only care whether or not it's a success or failure for now. + sec_trust_result_type = Security.SecTrustResultType() + Security.SecTrustGetTrustResult( + sec_trust_ref, ctypes.byref(sec_trust_result_type) + ) + + err = ssl.SSLCertVerificationError(cf_error_message) + err.verify_message = cf_error_message + err.verify_code = cf_error_code + raise err + finally: + if cf_error_string_ref: + CoreFoundation.CFRelease(cf_error_string_ref) diff --git a/contrib/python/pip/pip/_vendor/truststore/_windows.py b/contrib/python/pip/pip/_vendor/truststore/_windows.py index 3d00d467f99..a9bf9abdfc8 100644 --- a/contrib/python/pip/pip/_vendor/truststore/_windows.py +++ b/contrib/python/pip/pip/_vendor/truststore/_windows.py @@ -212,6 +212,7 @@ CERT_CHAIN_POLICY_IGNORE_INVALID_POLICY_FLAG = 0x00000080 CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS = 0x00000F00 CERT_CHAIN_POLICY_ALLOW_TESTROOT_FLAG = 0x00008000 CERT_CHAIN_POLICY_TRUST_TESTROOT_FLAG = 0x00004000 +SECURITY_FLAG_IGNORE_CERT_CN_INVALID = 0x00001000 AUTHTYPE_SERVER = 2 CERT_CHAIN_POLICY_SSL = 4 FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 @@ -443,6 +444,10 @@ def _get_and_verify_cert_chain( ) ssl_extra_cert_chain_policy_para.dwAuthType = AUTHTYPE_SERVER ssl_extra_cert_chain_policy_para.fdwChecks = 0 + if ssl_context.check_hostname is False: + ssl_extra_cert_chain_policy_para.fdwChecks = ( + SECURITY_FLAG_IGNORE_CERT_CN_INVALID + ) if server_hostname: ssl_extra_cert_chain_policy_para.pwszServerName = c_wchar_p(server_hostname) @@ -452,8 +457,6 @@ def _get_and_verify_cert_chain( ) if ssl_context.verify_mode == ssl.CERT_NONE: chain_policy.dwFlags |= CERT_CHAIN_POLICY_VERIFY_MODE_NONE_FLAGS - if not ssl_context.check_hostname: - chain_policy.dwFlags |= CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG chain_policy.cbSize = sizeof(chain_policy) pPolicyPara = pointer(chain_policy) diff --git a/contrib/python/pip/pip/_vendor/urllib3/_version.py b/contrib/python/pip/pip/_vendor/urllib3/_version.py index 85e725eaf4d..d49df2a0c54 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/_version.py +++ b/contrib/python/pip/pip/_vendor/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.18" +__version__ = "1.26.20" diff --git a/contrib/python/pip/pip/_vendor/urllib3/connection.py b/contrib/python/pip/pip/_vendor/urllib3/connection.py index 54b96b19154..de35b63d670 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/connection.py +++ b/contrib/python/pip/pip/_vendor/urllib3/connection.py @@ -68,7 +68,7 @@ port_by_scheme = {"http": 80, "https": 443} # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2022, 1, 1) +RECENT_DATE = datetime.date(2024, 1, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") @@ -437,7 +437,7 @@ class HTTPSConnection(HTTPConnection): and self.ssl_version is None and hasattr(self.sock, "version") and self.sock.version() in {"TLSv1", "TLSv1.1"} - ): + ): # Defensive: warnings.warn( "Negotiating TLSv1/TLSv1.1 by default is deprecated " "and will be disabled in urllib3 v2.0.0. Connecting to " diff --git a/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py b/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py index 5a6adcbdc75..0872ed77011 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py +++ b/contrib/python/pip/pip/_vendor/urllib3/connectionpool.py @@ -423,12 +423,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass except IOError as e: # Python 2 and macOS/Linux - # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS + # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE/ECONNRESET are needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ if e.errno not in { errno.EPIPE, errno.ESHUTDOWN, errno.EPROTOTYPE, + errno.ECONNRESET, }: raise @@ -768,7 +769,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # so we try to cover our bases here! message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) return ( - "wrong version number" in message or "unknown protocol" in message + "wrong version number" in message + or "unknown protocol" in message + or "record layer failure" in message ) # Try to detect a common user error with proxies which is to diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/retry.py b/contrib/python/pip/pip/_vendor/urllib3/util/retry.py index 60ef6c4f3f9..9a1e90d0b23 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/retry.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/retry.py @@ -235,7 +235,9 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) #: Maximum backoff time. DEFAULT_BACKOFF_MAX = 120 diff --git a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py index 2b45d391d4d..0a6a0e06a0d 100644 --- a/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py +++ b/contrib/python/pip/pip/_vendor/urllib3/util/ssl_.py @@ -1,11 +1,11 @@ from __future__ import absolute_import +import hashlib import hmac import os import sys import warnings from binascii import hexlify, unhexlify -from hashlib import md5, sha1, sha256 from ..exceptions import ( InsecurePlatformWarning, @@ -24,7 +24,10 @@ IS_SECURETRANSPORT = False ALPN_PROTOCOLS = ["http/1.1"] # Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} +HASHFUNC_MAP = { + length: getattr(hashlib, algorithm, None) + for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256")) +} def _const_compare_digest_backport(a, b): @@ -191,9 +194,15 @@ def assert_fingerprint(cert, fingerprint): fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) - hashfunc = HASHFUNC_MAP.get(digest_length) - if not hashfunc: + if digest_length not in HASHFUNC_MAP: raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) + hashfunc = HASHFUNC_MAP.get(digest_length) + if hashfunc is None: + raise SSLError( + "Hash function implementation unavailable for fingerprint length: {0}".format( + digest_length + ) + ) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) diff --git a/contrib/python/pip/pip/_vendor/vendor.txt b/contrib/python/pip/pip/_vendor/vendor.txt index fd92690602f..2ba053a6e54 100644 --- a/contrib/python/pip/pip/_vendor/vendor.txt +++ b/contrib/python/pip/pip/_vendor/vendor.txt @@ -1,18 +1,18 @@ CacheControl==0.14.0 -distlib==0.3.8 +distlib==0.3.9 distro==1.9.0 msgpack==1.0.8 packaging==24.1 platformdirs==4.2.2 pyproject-hooks==1.0.0 requests==2.32.3 - certifi==2024.7.4 + certifi==2024.8.30 idna==3.7 - urllib3==1.26.18 + urllib3==1.26.20 rich==13.7.1 pygments==2.18.0 typing_extensions==4.12.2 resolvelib==1.0.1 setuptools==70.3.0 tomli==2.0.1 -truststore==0.9.1 +truststore==0.10.0 diff --git a/contrib/python/pip/ya.make b/contrib/python/pip/ya.make index 258dd0af73d..64b3b100b72 100644 --- a/contrib/python/pip/ya.make +++ b/contrib/python/pip/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(24.2) +VERSION(24.3.1) LICENSE(MIT) diff --git a/contrib/python/prettytable/py3/.dist-info/METADATA b/contrib/python/prettytable/py3/.dist-info/METADATA index 6df980cfd89..e44b80cd6eb 100644 --- a/contrib/python/prettytable/py3/.dist-info/METADATA +++ b/contrib/python/prettytable/py3/.dist-info/METADATA @@ -1,18 +1,18 @@ Metadata-Version: 2.3 Name: prettytable -Version: 3.11.0 +Version: 3.12.0 Summary: A simple Python library for easily displaying tabular data in a visually appealing ASCII table format -Project-URL: Changelog, https://github.com/jazzband/prettytable/releases -Project-URL: Homepage, https://github.com/jazzband/prettytable -Project-URL: Source, https://github.com/jazzband/prettytable +Project-URL: Changelog, https://github.com/prettytable/prettytable/releases +Project-URL: Funding, https://tidelift.com/subscription/pkg/pypi-prettytable?utm_source=pypi-prettytable&utm_medium=pypi +Project-URL: Homepage, https://github.com/prettytable/prettytable +Project-URL: Source, https://github.com/prettytable/prettytable Author-email: Luke Maurits <[email protected]> -Maintainer: Jazzband -License: BSD (3 clause) +Maintainer: Hugo van Kemenade +License-Expression: BSD-3-Clause License-File: LICENSE Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Text Processing Classifier: Typing :: Typed -Requires-Python: >=3.8 +Requires-Python: >=3.9 Requires-Dist: wcwidth Provides-Extra: tests Requires-Dist: pytest; extra == 'tests' @@ -32,13 +32,13 @@ Description-Content-Type: text/markdown # PrettyTable -[](https://jazzband.co/) [](https://pypi.org/project/prettytable/) [](https://pypi.org/project/prettytable/) [](https://pypistats.org/packages/prettytable) -[](https://github.com/jazzband/prettytable/actions) -[](https://codecov.io/gh/jazzband/prettytable) -[](https://github.com/psf/black) +[](https://github.com/prettytable/prettytable/actions) +[](https://codecov.io/gh/prettytable/prettytable) +[](https://github.com/psf/black) +[](https://tidelift.com/subscription/pkg/pypi-prettytable?utm_source=pypi-prettytable&utm_medium=badge) PrettyTable lets you print tables in an attractive ASCII form: @@ -64,11 +64,11 @@ Install via pip: Install latest development version: - python -m pip install -U git+https://github.com/jazzband/prettytable + python -m pip install -U git+https://github.com/prettytable/prettytable Or from `requirements.txt`: - -e git://github.com/jazzband/prettytable.git#egg=prettytable + -e git://github.com/prettytable/prettytable.git#egg=prettytable ## Tutorial on how to use the PrettyTable API diff --git a/contrib/python/prettytable/py3/README.md b/contrib/python/prettytable/py3/README.md index bb365317da2..0c89cbefbca 100644 --- a/contrib/python/prettytable/py3/README.md +++ b/contrib/python/prettytable/py3/README.md @@ -1,12 +1,12 @@ # PrettyTable -[](https://jazzband.co/) [](https://pypi.org/project/prettytable/) [](https://pypi.org/project/prettytable/) [](https://pypistats.org/packages/prettytable) -[](https://github.com/jazzband/prettytable/actions) -[](https://codecov.io/gh/jazzband/prettytable) -[](https://github.com/psf/black) +[](https://github.com/prettytable/prettytable/actions) +[](https://codecov.io/gh/prettytable/prettytable) +[](https://github.com/psf/black) +[](https://tidelift.com/subscription/pkg/pypi-prettytable?utm_source=pypi-prettytable&utm_medium=badge) PrettyTable lets you print tables in an attractive ASCII form: @@ -32,11 +32,11 @@ Install via pip: Install latest development version: - python -m pip install -U git+https://github.com/jazzband/prettytable + python -m pip install -U git+https://github.com/prettytable/prettytable Or from `requirements.txt`: - -e git://github.com/jazzband/prettytable.git#egg=prettytable + -e git://github.com/prettytable/prettytable.git#egg=prettytable ## Tutorial on how to use the PrettyTable API diff --git a/contrib/python/prettytable/py3/prettytable/__init__.py b/contrib/python/prettytable/py3/prettytable/__init__.py index 7f9bbe27eb5..29ffbe8d974 100644 --- a/contrib/python/prettytable/py3/prettytable/__init__.py +++ b/contrib/python/prettytable/py3/prettytable/__init__.py @@ -2,21 +2,27 @@ from __future__ import annotations from typing import Any -from .prettytable import ( - ALL, - DEFAULT, - DOUBLE_BORDER, - FRAME, - HEADER, - MARKDOWN, - MSWORD_FRIENDLY, - NONE, - ORGMODE, - PLAIN_COLUMNS, - RANDOM, - SINGLE_BORDER, +from ._version import __version__ +from .prettytable import ( # noqa: F401 + _DEPRECATED_ALL, + _DEPRECATED_DEFAULT, + _DEPRECATED_DOUBLE_BORDER, + _DEPRECATED_FRAME, + _DEPRECATED_HEADER, + _DEPRECATED_MARKDOWN, + _DEPRECATED_MSWORD_FRIENDLY, + _DEPRECATED_NONE, + _DEPRECATED_ORGMODE, + _DEPRECATED_PLAIN_COLUMNS, + _DEPRECATED_RANDOM, + _DEPRECATED_SINGLE_BORDER, + HRuleStyle, PrettyTable, + RowType, TableHandler, + TableStyle, + VRuleStyle, + _warn_deprecation, from_csv, from_db_cursor, from_html, @@ -37,21 +43,20 @@ __all__ = [ "ORGMODE", "PLAIN_COLUMNS", "RANDOM", + "HRuleStyle", "PrettyTable", + "RowType", "TableHandler", + "TableStyle", + "VRuleStyle", "from_csv", "from_db_cursor", "from_html", "from_html_one", "from_json", + "__version__", ] def __getattr__(name: str) -> Any: - if name == "__version__": - import importlib.metadata - - return importlib.metadata.version(__name__) - - msg = f"module '{__name__}' has no attribute '{name}'" - raise AttributeError(msg) + return _warn_deprecation(name, module_globals=globals()) diff --git a/contrib/python/prettytable/py3/prettytable/_version.py b/contrib/python/prettytable/py3/prettytable/_version.py new file mode 100644 index 00000000000..9e1a89958d8 --- /dev/null +++ b/contrib/python/prettytable/py3/prettytable/_version.py @@ -0,0 +1,16 @@ +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '3.12.0' +__version_tuple__ = version_tuple = (3, 12, 0) diff --git a/contrib/python/prettytable/py3/prettytable/colortable.py b/contrib/python/prettytable/py3/prettytable/colortable.py index 3df50c63d88..52e1fa20a4c 100644 --- a/contrib/python/prettytable/py3/prettytable/colortable.py +++ b/contrib/python/prettytable/py3/prettytable/colortable.py @@ -45,12 +45,54 @@ class Theme: class Themes: DEFAULT = Theme() + DYSLEXIA_FRIENDLY = Theme( + default_color="38;5;223", + vertical_color="38;5;22", + horizontal_color="38;5;22", + junction_color="38;5;58", + ) + EARTH = Theme( + default_color="33", + vertical_color="38;5;94", + horizontal_color="38;5;22", + junction_color="38;5;130", + ) + GLARE_REDUCTION = Theme( + default_color="38;5;252", + vertical_color="38;5;240", + horizontal_color="38;5;240", + junction_color="38;5;246", + ) + HIGH_CONTRAST = Theme( + default_color="97", + vertical_color="91", + horizontal_color="94", + junction_color="93", + ) + LAVENDER = Theme( + default_color="38;5;183", + vertical_color="35", + horizontal_color="38;5;147", + junction_color="38;5;219", + ) OCEAN = Theme( default_color="96", vertical_color="34", horizontal_color="34", junction_color="36", ) + OCEAN_DEEP = Theme( + default_color="96", + vertical_color="34", + horizontal_color="36", + junction_color="94", + ) + PASTEL = Theme( + default_color="38;5;223", + vertical_color="38;5;152", + horizontal_color="38;5;187", + junction_color="38;5;157", + ) class ColorTable(PrettyTable): diff --git a/contrib/python/prettytable/py3/prettytable/prettytable.py b/contrib/python/prettytable/py3/prettytable/prettytable.py index 7f5dacb58af..818a5203bbc 100644 --- a/contrib/python/prettytable/py3/prettytable/prettytable.py +++ b/contrib/python/prettytable/py3/prettytable/prettytable.py @@ -35,30 +35,120 @@ from __future__ import annotations import io import re +import warnings +from collections.abc import Callable, Iterable, Mapping, Sequence +from enum import IntEnum from html.parser import HTMLParser -from typing import Any +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast -# hrule styles -FRAME = 0 -ALL = 1 -NONE = 2 -HEADER = 3 +if TYPE_CHECKING: + from sqlite3 import Cursor + + from _typeshed import SupportsRichComparison + from typing_extensions import Self, TypeAlias + + +class HRuleStyle(IntEnum): + FRAME = 0 + ALL = 1 + NONE = 2 + HEADER = 3 + + +class VRuleStyle(IntEnum): + FRAME = 0 + ALL = 1 + NONE = 2 + + +class TableStyle(IntEnum): + DEFAULT = 10 + MSWORD_FRIENDLY = 11 + PLAIN_COLUMNS = 12 + MARKDOWN = 13 + ORGMODE = 14 + DOUBLE_BORDER = 15 + SINGLE_BORDER = 16 + RANDOM = 20 + + +# keep for backwards compatibility +_DEPRECATED_FRAME: Final = 0 +_DEPRECATED_ALL: Final = 1 +_DEPRECATED_NONE: Final = 2 +_DEPRECATED_HEADER: Final = 3 +_DEPRECATED_DEFAULT: Final = TableStyle.DEFAULT +_DEPRECATED_MSWORD_FRIENDLY: Final = TableStyle.MSWORD_FRIENDLY +_DEPRECATED_PLAIN_COLUMNS: Final = TableStyle.PLAIN_COLUMNS +_DEPRECATED_MARKDOWN: Final = TableStyle.MARKDOWN +_DEPRECATED_ORGMODE: Final = TableStyle.ORGMODE +_DEPRECATED_DOUBLE_BORDER: Final = TableStyle.DOUBLE_BORDER +_DEPRECATED_SINGLE_BORDER: Final = TableStyle.SINGLE_BORDER +_DEPRECATED_RANDOM: Final = TableStyle.RANDOM +# -------------------------------- + +BASE_ALIGN_VALUE: Final = "base_align_value" + +RowType: TypeAlias = list[Any] +AlignType: TypeAlias = Literal["l", "c", "r"] +VAlignType: TypeAlias = Literal["t", "m", "b"] +HeaderStyleType: TypeAlias = Literal["cap", "title", "upper", "lower", None] + + +class OptionsType(TypedDict): + title: str | None + start: int + end: int | None + fields: Sequence[str | None] | None + header: bool + border: bool + preserve_internal_border: bool + sortby: str | None + reversesort: bool + sort_key: Callable[[RowType], SupportsRichComparison] + attributes: dict[str, str] + format: bool + hrules: HRuleStyle + vrules: VRuleStyle + int_format: str | dict[str, str] | None + float_format: str | dict[str, str] | None + custom_format: ( + Callable[[str, Any], str] | dict[str, Callable[[str, Any], str]] | None + ) + min_table_width: int | None + max_table_width: int | None + padding_width: int + left_padding_width: int | None + right_padding_width: int | None + vertical_char: str + horizontal_char: str + horizontal_align_char: str + junction_char: str + header_style: HeaderStyleType + xhtml: bool + print_empty: bool + oldsortslice: bool + top_junction_char: str + bottom_junction_char: str + right_junction_char: str + left_junction_char: str + top_right_junction_char: str + top_left_junction_char: str + bottom_right_junction_char: str + bottom_left_junction_char: str + align: dict[str, AlignType] + valign: dict[str, VAlignType] + min_width: int | dict[str, int] | None + max_width: int | dict[str, int] | None + none_format: str | dict[str, str | None] | None + escape_header: bool + escape_data: bool -# Table styles -DEFAULT = 10 -MSWORD_FRIENDLY = 11 -PLAIN_COLUMNS = 12 -MARKDOWN = 13 -ORGMODE = 14 -DOUBLE_BORDER = 15 -SINGLE_BORDER = 16 -RANDOM = 20 -BASE_ALIGN_VALUE = "base_align_value" _re = re.compile(r"\033\[[0-9;]*m|\033\(B") -def _get_size(text): +def _get_size(text: str) -> tuple[int, int]: lines = text.split("\n") height = len(lines) width = max(_str_block_width(line) for line in lines) @@ -66,7 +156,56 @@ def _get_size(text): class PrettyTable: - def __init__(self, field_names=None, **kwargs) -> None: + _xhtml: bool + _align: dict[str, AlignType] + _valign: dict[str, VAlignType] + _min_width: dict[str, int] + _max_width: dict[str, int] + _min_table_width: int | None + _max_table_width: int | None + _fields: Sequence[str | None] | None + _title: str | None + _start: int + _end: int | None + _sortby: str | None + _reversesort: bool + _sort_key: Callable[[RowType], SupportsRichComparison] + _header: bool + _header_style: HeaderStyleType + _border: bool + _preserve_internal_border: bool + _hrules: HRuleStyle + _vrules: VRuleStyle + _int_format: dict[str, str] + _float_format: dict[str, str] + _custom_format: dict[str, Callable[[str, Any], str]] + _padding_width: int + _left_padding_width: int | None + _right_padding_width: int | None + _vertical_char: str + _horizontal_char: str + _horizontal_align_char: str | None + _junction_char: str + _top_junction_char: str | None + _bottom_junction_char: str | None + _right_junction_char: str | None + _left_junction_char: str | None + _top_right_junction_char: str | None + _top_left_junction_char: str | None + _bottom_right_junction_char: str | None + _bottom_left_junction_char: str | None + _format: bool + _print_empty: bool + _oldsortslice: bool + _attributes: dict[str, str] + _escape_header: bool + _escape_data: bool + _style: TableStyle | None + orgmode: bool + _widths: list[int] + _hrule: str + + def __init__(self, field_names: Sequence[str] | None = None, **kwargs) -> None: """Return a new PrettyTable instance Arguments: @@ -84,9 +223,9 @@ class PrettyTable: preserve_internal_border - print a border inside the table even if border is disabled (True or False) hrules - controls printing of horizontal rules after rows. - Allowed values: FRAME, HEADER, ALL, NONE + Allowed values: HRuleStyle vrules - controls printing of vertical rules between columns. - Allowed values: FRAME, ALL, NONE + Allowed values: VRuleStyle int_format - controls formatting of integer data float_format - controls formatting of floating point data custom_format - controls formatting of any column using callable @@ -126,7 +265,7 @@ class PrettyTable: # Data self._field_names: list[str] = [] - self._rows: list[list] = [] + self._rows: list[RowType] = [] self._dividers: list[bool] = [] self.align = {} self.valign = {} @@ -171,7 +310,6 @@ class PrettyTable: "horizontal_align_char", "junction_char", "header_style", - "valign", "xhtml", "print_empty", "oldsortslice", @@ -201,7 +339,7 @@ class PrettyTable: self._start = kwargs["start"] or 0 self._end = kwargs["end"] or None self._fields = kwargs["fields"] or None - self._none_format: dict[None, None] = {} + self._none_format: dict[str, str | None] = {} if kwargs["header"] in (True, False): self._header = kwargs["header"] @@ -216,8 +354,8 @@ class PrettyTable: self._preserve_internal_border = kwargs["preserve_internal_border"] else: self._preserve_internal_border = False - self._hrules = kwargs["hrules"] or FRAME - self._vrules = kwargs["vrules"] or ALL + self._hrules = kwargs["hrules"] or HRuleStyle.FRAME + self._vrules = kwargs["vrules"] or VRuleStyle.ALL self._sortby = kwargs["sortby"] or None if kwargs["reversesort"] in (True, False): @@ -278,7 +416,7 @@ class PrettyTable: self._xhtml = kwargs["xhtml"] or False self._attributes = kwargs["attributes"] or {} - def _justify(self, text, width, align): + def _justify(self, text: str, width: int, align: AlignType) -> str: excess = width - _str_block_width(text) if align == "l": return text + excess * " " @@ -312,7 +450,7 @@ class PrettyTable: else: raise AttributeError(name) - def __getitem__(self, index): + def __getitem__(self, index: int | slice) -> PrettyTable: new = PrettyTable() new.field_names = self.field_names for attr in self._options: @@ -334,7 +472,7 @@ class PrettyTable: def __repr__(self) -> str: return self.get_string() - def _repr_html_(self): + def _repr_html_(self) -> str: """ Returns get_html_string value by default as the repr call in Jupyter notebook environment @@ -369,7 +507,6 @@ class PrettyTable: "padding_width", "left_padding_width", "right_padding_width", - "format", ): self._validate_nonnegative_int(option, val) elif option == "sortby": @@ -388,6 +525,7 @@ class PrettyTable: "preserve_internal_border", "reversesort", "xhtml", + "format", "print_empty", "oldsortslice", "escape_header", @@ -472,9 +610,9 @@ class PrettyTable: def _validate_valign(self, val): try: - assert val in ["t", "m", "b", None] + assert val in ["t", "m", "b"] except AssertionError: - msg = f"Alignment {val} is invalid, use t, m, b or None" + msg = f"Alignment {val} is invalid, use t, m, b" raise ValueError(msg) def _validate_nonnegative_int(self, name, val): @@ -528,16 +666,16 @@ class PrettyTable: def _validate_hrules(self, name, val): try: - assert val in (ALL, FRAME, HEADER, NONE) + assert val in list(HRuleStyle) except AssertionError: - msg = f"Invalid value for {name}. Must be ALL, FRAME, HEADER or NONE." + msg = f"Invalid value for {name}. Must be HRuleStyle." raise ValueError(msg) def _validate_vrules(self, name, val): try: - assert val in (ALL, FRAME, NONE) + assert val in list(VRuleStyle) except AssertionError: - msg = f"Invalid value for {name}. Must be ALL, FRAME, or NONE." + msg = f"Invalid value for {name}. Must be VRuleStyle." raise ValueError(msg) def _validate_field_name(self, name, val): @@ -573,7 +711,7 @@ class PrettyTable: # ATTRIBUTE MANAGEMENT # ############################## @property - def rows(self) -> list[Any]: + def rows(self) -> list[RowType]: return self._rows[:] @property @@ -586,7 +724,7 @@ class PrettyTable: return self._xhtml @xhtml.setter - def xhtml(self, val) -> None: + def xhtml(self, val: bool) -> None: self._validate_option("xhtml", val) self._xhtml = val @@ -722,35 +860,35 @@ class PrettyTable: self._min_width[field] = val @property - def min_table_width(self): + def min_table_width(self) -> int | None: return self._min_table_width @min_table_width.setter - def min_table_width(self, val) -> None: + def min_table_width(self, val: int) -> None: self._validate_option("min_table_width", val) self._min_table_width = val @property - def max_table_width(self): + def max_table_width(self) -> int | None: return self._max_table_width @max_table_width.setter - def max_table_width(self, val) -> None: + def max_table_width(self, val: int) -> None: self._validate_option("max_table_width", val) self._max_table_width = val @property - def fields(self): + def fields(self) -> Sequence[str | None] | None: """List or tuple of field names to include in displays""" return self._fields @fields.setter - def fields(self, val) -> None: + def fields(self, val: Sequence[str | None]) -> None: self._validate_option("fields", val) self._fields = val @property - def title(self): + def title(self) -> str | None: """Optional table title Arguments: @@ -759,11 +897,11 @@ class PrettyTable: return self._title @title.setter - def title(self, val) -> None: + def title(self, val: str) -> None: self._title = str(val) @property - def start(self): + def start(self) -> int: """Start index of the range of rows to print Arguments: @@ -772,12 +910,12 @@ class PrettyTable: return self._start @start.setter - def start(self, val) -> None: + def start(self, val: int) -> None: self._validate_option("start", val) self._start = val @property - def end(self): + def end(self) -> int | None: """End index of the range of rows to print Arguments: @@ -786,12 +924,12 @@ class PrettyTable: return self._end @end.setter - def end(self, val) -> None: + def end(self, val: int) -> None: self._validate_option("end", val) self._end = val @property - def sortby(self): + def sortby(self) -> str | None: """Name of field by which to sort rows Arguments: @@ -800,12 +938,12 @@ class PrettyTable: return self._sortby @sortby.setter - def sortby(self, val) -> None: + def sortby(self, val: str | None) -> None: self._validate_option("sortby", val) self._sortby = val @property - def reversesort(self): + def reversesort(self) -> bool: """Controls direction of sorting (ascending vs descending) Arguments: @@ -815,12 +953,12 @@ class PrettyTable: return self._reversesort @reversesort.setter - def reversesort(self, val) -> None: + def reversesort(self, val: bool) -> None: self._validate_option("reversesort", val) self._reversesort = val @property - def sort_key(self): + def sort_key(self) -> Callable[[RowType], SupportsRichComparison]: """Sorting key function, applied to data points before sorting Arguments: @@ -830,12 +968,12 @@ class PrettyTable: return self._sort_key @sort_key.setter - def sort_key(self, val) -> None: + def sort_key(self, val: Callable[[RowType], SupportsRichComparison]) -> None: self._validate_option("sort_key", val) self._sort_key = val @property - def header(self): + def header(self) -> bool: """Controls printing of table header with field names Arguments: @@ -844,12 +982,12 @@ class PrettyTable: return self._header @header.setter - def header(self, val) -> None: + def header(self, val: bool) -> None: self._validate_option("header", val) self._header = val @property - def header_style(self): + def header_style(self) -> HeaderStyleType: """Controls stylisation applied to field names in header Arguments: @@ -859,12 +997,12 @@ class PrettyTable: return self._header_style @header_style.setter - def header_style(self, val) -> None: + def header_style(self, val: HeaderStyleType) -> None: self._validate_header_style(val) self._header_style = val @property - def border(self): + def border(self) -> bool: """Controls printing of border around table Arguments: @@ -873,12 +1011,12 @@ class PrettyTable: return self._border @border.setter - def border(self, val) -> None: + def border(self, val: bool) -> None: self._validate_option("border", val) self._border = val @property - def preserve_internal_border(self): + def preserve_internal_border(self) -> bool: """Controls printing of border inside table Arguments: @@ -888,35 +1026,35 @@ class PrettyTable: return self._preserve_internal_border @preserve_internal_border.setter - def preserve_internal_border(self, val) -> None: + def preserve_internal_border(self, val: bool) -> None: self._validate_option("preserve_internal_border", val) self._preserve_internal_border = val @property - def hrules(self): + def hrules(self) -> HRuleStyle: """Controls printing of horizontal rules after rows Arguments: - hrules - horizontal rules style. Allowed values: FRAME, ALL, HEADER, NONE""" + hrules - horizontal rules style. Allowed values: HRuleStyle""" return self._hrules @hrules.setter - def hrules(self, val) -> None: + def hrules(self, val: HRuleStyle) -> None: self._validate_option("hrules", val) self._hrules = val @property - def vrules(self): + def vrules(self) -> VRuleStyle: """Controls printing of vertical rules between columns Arguments: - vrules - vertical rules style. Allowed values: FRAME, ALL, NONE""" + vrules - vertical rules style. Allowed values: VRuleStyle""" return self._vrules @vrules.setter - def vrules(self, val) -> None: + def vrules(self, val: VRuleStyle) -> None: self._validate_option("vrules", val) self._vrules = val @@ -979,7 +1117,7 @@ class PrettyTable: raise TypeError(msg) @property - def padding_width(self): + def padding_width(self) -> int: """The number of empty spaces between a column's edge and its content Arguments: @@ -988,12 +1126,12 @@ class PrettyTable: return self._padding_width @padding_width.setter - def padding_width(self, val) -> None: + def padding_width(self, val: int) -> None: self._validate_option("padding_width", val) self._padding_width = val @property - def left_padding_width(self): + def left_padding_width(self) -> int | None: """The number of empty spaces between a column's left edge and its content Arguments: @@ -1002,12 +1140,12 @@ class PrettyTable: return self._left_padding_width @left_padding_width.setter - def left_padding_width(self, val) -> None: + def left_padding_width(self, val: int) -> None: self._validate_option("left_padding_width", val) self._left_padding_width = val @property - def right_padding_width(self): + def right_padding_width(self) -> int | None: """The number of empty spaces between a column's right edge and its content Arguments: @@ -1016,12 +1154,12 @@ class PrettyTable: return self._right_padding_width @right_padding_width.setter - def right_padding_width(self, val) -> None: + def right_padding_width(self, val: int) -> None: self._validate_option("right_padding_width", val) self._right_padding_width = val @property - def vertical_char(self): + def vertical_char(self) -> str: """The character used when printing table borders to draw vertical lines Arguments: @@ -1030,13 +1168,13 @@ class PrettyTable: return self._vertical_char @vertical_char.setter - def vertical_char(self, val) -> None: + def vertical_char(self, val: str) -> None: val = str(val) self._validate_option("vertical_char", val) self._vertical_char = val @property - def horizontal_char(self): + def horizontal_char(self) -> str: """The character used when printing table borders to draw horizontal lines Arguments: @@ -1045,13 +1183,13 @@ class PrettyTable: return self._horizontal_char @horizontal_char.setter - def horizontal_char(self, val) -> None: + def horizontal_char(self, val: str) -> None: val = str(val) self._validate_option("horizontal_char", val) self._horizontal_char = val @property - def horizontal_align_char(self): + def horizontal_align_char(self) -> str: """The character used to indicate column alignment in horizontal lines Arguments: @@ -1060,13 +1198,13 @@ class PrettyTable: return self._bottom_left_junction_char or self.junction_char @horizontal_align_char.setter - def horizontal_align_char(self, val) -> None: + def horizontal_align_char(self, val: str) -> None: val = str(val) self._validate_option("horizontal_align_char", val) self._horizontal_align_char = val @property - def junction_char(self): + def junction_char(self) -> str: """The character used when printing table borders to draw line junctions Arguments: @@ -1075,13 +1213,13 @@ class PrettyTable: return self._junction_char @junction_char.setter - def junction_char(self, val) -> None: + def junction_char(self, val: str) -> None: val = str(val) self._validate_option("junction_char", val) self._junction_char = val @property - def top_junction_char(self): + def top_junction_char(self) -> str: """The character used when printing table borders to draw top line junctions Arguments: @@ -1090,13 +1228,13 @@ class PrettyTable: return self._top_junction_char or self.junction_char @top_junction_char.setter - def top_junction_char(self, val) -> None: + def top_junction_char(self, val: str) -> None: val = str(val) self._validate_option("top_junction_char", val) self._top_junction_char = val @property - def bottom_junction_char(self): + def bottom_junction_char(self) -> str: """The character used when printing table borders to draw bottom line junctions Arguments: @@ -1106,13 +1244,13 @@ class PrettyTable: return self._bottom_junction_char or self.junction_char @bottom_junction_char.setter - def bottom_junction_char(self, val) -> None: + def bottom_junction_char(self, val: str) -> None: val = str(val) self._validate_option("bottom_junction_char", val) self._bottom_junction_char = val @property - def right_junction_char(self): + def right_junction_char(self) -> str: """The character used when printing table borders to draw right line junctions Arguments: @@ -1122,13 +1260,13 @@ class PrettyTable: return self._right_junction_char or self.junction_char @right_junction_char.setter - def right_junction_char(self, val) -> None: + def right_junction_char(self, val: str) -> None: val = str(val) self._validate_option("right_junction_char", val) self._right_junction_char = val @property - def left_junction_char(self): + def left_junction_char(self) -> str: """The character used when printing table borders to draw left line junctions Arguments: @@ -1137,13 +1275,13 @@ class PrettyTable: return self._left_junction_char or self.junction_char @left_junction_char.setter - def left_junction_char(self, val) -> None: + def left_junction_char(self, val: str) -> None: val = str(val) self._validate_option("left_junction_char", val) self._left_junction_char = val @property - def top_right_junction_char(self): + def top_right_junction_char(self) -> str: """ The character used when printing table borders to draw top-right line junctions @@ -1154,13 +1292,13 @@ class PrettyTable: return self._top_right_junction_char or self.junction_char @top_right_junction_char.setter - def top_right_junction_char(self, val) -> None: + def top_right_junction_char(self, val: str) -> None: val = str(val) self._validate_option("top_right_junction_char", val) self._top_right_junction_char = val @property - def top_left_junction_char(self): + def top_left_junction_char(self) -> str: """ The character used when printing table borders to draw top-left line junctions @@ -1171,13 +1309,13 @@ class PrettyTable: return self._top_left_junction_char or self.junction_char @top_left_junction_char.setter - def top_left_junction_char(self, val) -> None: + def top_left_junction_char(self, val: str) -> None: val = str(val) self._validate_option("top_left_junction_char", val) self._top_left_junction_char = val @property - def bottom_right_junction_char(self): + def bottom_right_junction_char(self) -> str: """The character used when printing table borders to draw bottom-right line junctions @@ -1188,13 +1326,13 @@ class PrettyTable: return self._bottom_right_junction_char or self.junction_char @bottom_right_junction_char.setter - def bottom_right_junction_char(self, val) -> None: + def bottom_right_junction_char(self, val: str) -> None: val = str(val) self._validate_option("bottom_right_junction_char", val) self._bottom_right_junction_char = val @property - def bottom_left_junction_char(self): + def bottom_left_junction_char(self) -> str: """The character used when printing table borders to draw bottom-left line junctions @@ -1205,13 +1343,13 @@ class PrettyTable: return self._bottom_left_junction_char or self.junction_char @bottom_left_junction_char.setter - def bottom_left_junction_char(self, val) -> None: + def bottom_left_junction_char(self, val: str) -> None: val = str(val) self._validate_option("bottom_left_junction_char", val) self._bottom_left_junction_char = val @property - def format(self): + def format(self) -> bool: """Controls whether or not HTML tables are formatted to match styling options Arguments: @@ -1220,12 +1358,12 @@ class PrettyTable: return self._format @format.setter - def format(self, val) -> None: + def format(self, val: bool) -> None: self._validate_option("format", val) self._format = val @property - def print_empty(self): + def print_empty(self) -> bool: """Controls whether or not empty tables produce a header and frame or just an empty string @@ -1235,12 +1373,12 @@ class PrettyTable: return self._print_empty @print_empty.setter - def print_empty(self, val) -> None: + def print_empty(self, val: bool) -> None: self._validate_option("print_empty", val) self._print_empty = val @property - def attributes(self): + def attributes(self) -> dict[str, str]: """A dictionary of HTML attribute name/value pairs to be included in the <table> tag when printing HTML @@ -1250,37 +1388,37 @@ class PrettyTable: return self._attributes @attributes.setter - def attributes(self, val) -> None: + def attributes(self, val: dict[str, str]) -> None: self._validate_option("attributes", val) self._attributes = val @property - def oldsortslice(self): + def oldsortslice(self) -> bool: """oldsortslice - Slice rows before sorting in the "old style" """ return self._oldsortslice @oldsortslice.setter - def oldsortslice(self, val) -> None: + def oldsortslice(self, val: bool) -> None: self._validate_option("oldsortslice", val) self._oldsortslice = val @property - def escape_header(self): + def escape_header(self) -> bool: """Escapes the text within a header (True or False)""" return self._escape_header @escape_header.setter - def escape_header(self, val): + def escape_header(self, val: bool) -> None: self._validate_option("escape_header", val) self._escape_header = val @property - def escape_data(self): + def escape_data(self) -> bool: """Escapes the text within a data field (True or False)""" return self._escape_data @escape_data.setter - def escape_data(self, val): + def escape_data(self, val: bool) -> None: self._validate_option("escape_data", val) self._escape_data = val @@ -1288,37 +1426,37 @@ class PrettyTable: # OPTION MIXER # ############################## - def _get_options(self, kwargs): - options = {} + def _get_options(self, kwargs: Mapping[str, Any]) -> OptionsType: + options: dict[str, Any] = {} for option in self._options: if option in kwargs: self._validate_option(option, kwargs[option]) options[option] = kwargs[option] else: options[option] = getattr(self, option) - return options + return cast(OptionsType, options) ############################## # PRESET STYLE LOGIC # ############################## - def set_style(self, style) -> None: + def set_style(self, style: TableStyle) -> None: self._style = style - if style == DEFAULT: + if style == TableStyle.DEFAULT: self._set_default_style() - elif style == MSWORD_FRIENDLY: + elif style == TableStyle.MSWORD_FRIENDLY: self._set_msword_style() - elif style == PLAIN_COLUMNS: + elif style == TableStyle.PLAIN_COLUMNS: self._set_columns_style() - elif style == MARKDOWN: + elif style == TableStyle.MARKDOWN: self._set_markdown_style() - elif style == ORGMODE: + elif style == TableStyle.ORGMODE: self._set_orgmode_style() - elif style == DOUBLE_BORDER: + elif style == TableStyle.DOUBLE_BORDER: self._set_double_border_style() - elif style == SINGLE_BORDER: + elif style == TableStyle.SINGLE_BORDER: self._set_single_border_style() - elif style == RANDOM: + elif style == TableStyle.RANDOM: self._set_random_style() else: msg = "Invalid pre-set style" @@ -1331,7 +1469,7 @@ class PrettyTable: def _set_markdown_style(self) -> None: self.header = True self.border = True - self._hrules = None + self._hrules = HRuleStyle.HEADER self.padding_width = 1 self.left_padding_width = 1 self.right_padding_width = 1 @@ -1342,8 +1480,8 @@ class PrettyTable: def _set_default_style(self) -> None: self.header = True self.border = True - self._hrules = FRAME - self._vrules = ALL + self._hrules = HRuleStyle.FRAME + self._vrules = VRuleStyle.ALL self.padding_width = 1 self.left_padding_width = 1 self.right_padding_width = 1 @@ -1363,7 +1501,7 @@ class PrettyTable: def _set_msword_style(self) -> None: self.header = True self.border = True - self._hrules = NONE + self._hrules = HRuleStyle.NONE self.padding_width = 1 self.left_padding_width = 1 self.right_padding_width = 1 @@ -1408,8 +1546,8 @@ class PrettyTable: self.header = random.choice((True, False)) self.border = random.choice((True, False)) - self._hrules = random.choice((ALL, FRAME, HEADER, NONE)) - self._vrules = random.choice((ALL, FRAME, NONE)) + self._hrules = random.choice(list(HRuleStyle)) + self._vrules = random.choice(list(VRuleStyle)) self.left_padding_width = random.randint(0, 5) self.right_padding_width = random.randint(0, 5) self.vertical_char = random.choice(r"~!@#$%^&*()_+|-=\{}[];':\",./;<>?") @@ -1421,7 +1559,7 @@ class PrettyTable: # DATA INPUT METHODS # ############################## - def add_rows(self, rows) -> None: + def add_rows(self, rows: Iterable[RowType]) -> None: """Add rows to the table Arguments: @@ -1431,7 +1569,7 @@ class PrettyTable: for row in rows: self.add_row(row) - def add_row(self, row, *, divider: bool = False) -> None: + def add_row(self, row: RowType, *, divider: bool = False) -> None: """Add a row to the table Arguments: @@ -1450,7 +1588,7 @@ class PrettyTable: self._rows.append(list(row)) self._dividers.append(divider) - def del_row(self, row_index) -> None: + def del_row(self, row_index: int) -> None: """Delete a row from the table Arguments: @@ -1467,7 +1605,11 @@ class PrettyTable: del self._dividers[row_index] def add_column( - self, fieldname, column, align: str = "c", valign: str = "t" + self, + fieldname: str, + column: Sequence[Any], + align: AlignType = "c", + valign: VAlignType = "t", ) -> None: """Add a column to the table. @@ -1509,7 +1651,7 @@ class PrettyTable: for i, row in enumerate(self._rows): row.insert(0, i + 1) - def del_column(self, fieldname) -> None: + def del_column(self, fieldname: str) -> None: """Delete a column from the table Arguments: @@ -1549,7 +1691,7 @@ class PrettyTable: # MISC PUBLIC METHODS # ############################## - def copy(self): + def copy(self) -> Self: import copy return copy.deepcopy(self) @@ -1582,7 +1724,7 @@ class PrettyTable: # MISC PRIVATE METHODS # ############################## - def _format_value(self, field, value): + def _format_value(self, field: str, value: Any) -> str: if isinstance(value, int) and field in self._int_format: return (f"%{self._int_format[field]}d") % value elif isinstance(value, float) and field in self._float_format: @@ -1591,10 +1733,10 @@ class PrettyTable: formatter = self._custom_format.get(field, (lambda f, v: str(v))) return formatter(field, value) - def _compute_table_width(self, options): - if options["vrules"] == FRAME: + def _compute_table_width(self, options) -> int: + if options["vrules"] == VRuleStyle.FRAME: table_width = 2 - if options["vrules"] == ALL: + if options["vrules"] == VRuleStyle.ALL: table_width = 1 else: table_width = 0 @@ -1606,7 +1748,7 @@ class PrettyTable: table_width += self._widths[index] + per_col_padding + 1 return table_width - def _compute_widths(self, rows, options) -> None: + def _compute_widths(self, rows: list[list[str]], options: OptionsType) -> None: if options["header"]: widths = [_get_size(field)[0] for field in self._field_names] else: @@ -1615,9 +1757,11 @@ class PrettyTable: for row in rows: for index, value in enumerate(row): fieldname = self.field_names[index] - if self.none_format.get(fieldname) is not None: - if value == "None" or value is None: - value = self.none_format.get(fieldname) + if ( + value == "None" + and (none_val := self.none_format.get(fieldname)) is not None + ): + value = none_val if fieldname in self.max_width: widths[index] = max( widths[index], @@ -1628,7 +1772,7 @@ class PrettyTable: if fieldname in self.min_width: widths[index] = max(widths[index], self.min_width[fieldname]) - if self._style == MARKDOWN: + if self._style == TableStyle.MARKDOWN: # Markdown needs at least one hyphen in the divider if self._align[fieldname] in ("l", "r"): min_width = 1 @@ -1654,7 +1798,7 @@ class PrettyTable: if self._min_table_width or options["title"]: if options["title"]: title_width = len(options["title"]) + per_col_padding - if options["vrules"] in (FRAME, ALL): + if options["vrules"] in (VRuleStyle.FRAME, VRuleStyle.ALL): title_width += 2 else: title_width = 0 @@ -1680,7 +1824,7 @@ class PrettyTable: widths[-1] += min_width - sum(widths) self._widths = widths - def _get_padding_widths(self, options): + def _get_padding_widths(self, options: OptionsType) -> tuple[int, int]: if options["left_padding_width"] is not None: lpad = options["left_padding_width"] else: @@ -1691,7 +1835,7 @@ class PrettyTable: rpad = options["padding_width"] return lpad, rpad - def _get_rows(self, options): + def _get_rows(self, options: OptionsType) -> list[RowType]: """Return only those data rows that should be printed, based on slicing and sorting. @@ -1721,7 +1865,7 @@ class PrettyTable: return rows - def _get_dividers(self, options): + def _get_dividers(self, options: OptionsType) -> list[bool]: """Return only those dividers that should be printed, based on slicing. Arguments: @@ -1739,13 +1883,13 @@ class PrettyTable: return dividers - def _format_row(self, row): + def _format_row(self, row: RowType) -> list[str]: return [ self._format_value(field, value) for (field, value) in zip(self._field_names, row) ] - def _format_rows(self, rows): + def _format_rows(self, rows: list[RowType]) -> list[list[str]]: return [self._format_row(row) for row in rows] ############################## @@ -1766,9 +1910,9 @@ class PrettyTable: preserve_internal_border - print a border inside the table even if border is disabled (True or False) hrules - controls printing of horizontal rules after rows. - Allowed values: ALL, FRAME, HEADER, NONE + Allowed values: HRuleStyle vrules - controls printing of vertical rules between columns. - Allowed values: FRAME, ALL, NONE + Allowed values: VRuleStyle int_format - controls formatting of integer data float_format - controls formatting of floating point data custom_format - controls formatting of any column using callable @@ -1802,7 +1946,7 @@ class PrettyTable: options = self._get_options(kwargs) - lines = [] + lines: list[str] = [] # Don't think too hard about an empty table # Is this the desired behaviour? Maybe we should still print the header? @@ -1828,11 +1972,18 @@ class PrettyTable: # Add header or top of border if options["header"]: lines.append(self._stringify_header(options)) - elif options["border"] and options["hrules"] in (ALL, FRAME): + elif options["border"] and options["hrules"] in ( + HRuleStyle.ALL, + HRuleStyle.FRAME, + ): lines.append(self._stringify_hrule(options, where="top_")) - if title and options["vrules"] in (ALL, FRAME): + if title and options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): + left_j_len = len(self.left_junction_char) + right_j_len = len(self.right_junction_char) lines[-1] = ( - self.left_junction_char + lines[-1][1:-1] + self.right_junction_char + self.left_junction_char + + lines[-1][left_j_len:-right_j_len] + + self.right_junction_char ) # Add rows @@ -1850,29 +2001,33 @@ class PrettyTable: ) # Add bottom of border - if options["border"] and options["hrules"] == FRAME: + if options["border"] and options["hrules"] == HRuleStyle.FRAME: lines.append(self._stringify_hrule(options, where="bottom_")) if "orgmode" in self.__dict__ and self.orgmode: + left_j_len = len(self.left_junction_char) + right_j_len = len(self.right_junction_char) lines = [ - "|" + new_line[1:-1] + "|" + "|" + new_line[left_j_len:-right_j_len] + "|" for old_line in lines for new_line in old_line.split("\n") ] return "\n".join(lines) - def _stringify_hrule(self, options, where: str = ""): + def _stringify_hrule( + self, options: OptionsType, where: Literal["top_", "bottom_", ""] = "" + ) -> str: if not options["border"] and not options["preserve_internal_border"]: return "" lpad, rpad = self._get_padding_widths(options) - if options["vrules"] in (ALL, FRAME): - bits = [options[where + "left_junction_char"]] + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): + bits = [options[where + "left_junction_char"]] # type: ignore[literal-required] else: bits = [options["horizontal_char"]] # For tables with no data or fieldnames if not self._field_names: - bits.append(options[where + "right_junction_char"]) + bits.append(options[where + "right_junction_char"]) # type: ignore[literal-required] return "".join(bits) for field, width in zip(self._field_names, self._widths): if options["fields"] and field not in options["fields"]: @@ -1888,33 +2043,34 @@ class PrettyTable: line = line[:-2] + self._horizontal_align_char + " " bits.append(line) - if options["vrules"] == ALL: - bits.append(options[where + "junction_char"]) + if options["vrules"] == VRuleStyle.ALL: + bits.append(options[where + "junction_char"]) # type: ignore[literal-required] else: bits.append(options["horizontal_char"]) - if options["vrules"] in (ALL, FRAME): + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): bits.pop() - bits.append(options[where + "right_junction_char"]) + bits.append(options[where + "right_junction_char"]) # type: ignore[literal-required] if options["preserve_internal_border"] and not options["border"]: bits = bits[1:-1] return "".join(bits) - def _stringify_title(self, title, options): - lines = [] + def _stringify_title(self, title: str, options: OptionsType) -> str: + lines: list[str] = [] lpad, rpad = self._get_padding_widths(options) if options["border"]: - if options["vrules"] == ALL: - options["vrules"] = FRAME + if options["vrules"] == VRuleStyle.ALL: + options["vrules"] = VRuleStyle.FRAME lines.append(self._stringify_hrule(options, "top_")) - options["vrules"] = ALL - elif options["vrules"] == FRAME: + options["vrules"] = VRuleStyle.ALL + elif options["vrules"] == VRuleStyle.FRAME: lines.append(self._stringify_hrule(options, "top_")) - bits = [] + bits: list[str] = [] endpoint = ( options["vertical_char"] - if options["vrules"] in (ALL, FRAME) and options["border"] + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME) + and options["border"] else " " ) bits.append(endpoint) @@ -1927,13 +2083,16 @@ class PrettyTable: lines.append("".join(bits)) return "\n".join(lines) - def _stringify_header(self, options): - bits = [] + def _stringify_header(self, options: OptionsType) -> str: + bits: list[str] = [] lpad, rpad = self._get_padding_widths(options) if options["border"]: - if options["hrules"] in (ALL, FRAME): + if options["hrules"] in (HRuleStyle.ALL, HRuleStyle.FRAME): bits.append(self._stringify_hrule(options, "top_")) - if options["title"] and options["vrules"] in (ALL, FRAME): + if options["title"] and options["vrules"] in ( + VRuleStyle.ALL, + VRuleStyle.FRAME, + ): left_j_len = len(self.left_junction_char) right_j_len = len(self.right_junction_char) bits[-1] = ( @@ -1942,13 +2101,13 @@ class PrettyTable: + self.right_junction_char ) bits.append("\n") - if options["vrules"] in (ALL, FRAME): + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): bits.append(options["vertical_char"]) else: bits.append(" ") # For tables with no data or field names if not self._field_names: - if options["vrules"] in (ALL, FRAME): + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): bits.append(options["vertical_char"]) else: bits.append(" ") @@ -1973,7 +2132,7 @@ class PrettyTable: + " " * rpad ) if options["border"] or options["preserve_internal_border"]: - if options["vrules"] == ALL: + if options["vrules"] == VRuleStyle.ALL: bits.append(options["vertical_char"]) else: bits.append(" ") @@ -1985,17 +2144,17 @@ class PrettyTable: bits.append(" ") # If vrules is FRAME, then we just appended a space at the end # of the last field, when we really want a vertical character - if options["border"] and options["vrules"] == FRAME: + if options["border"] and options["vrules"] == VRuleStyle.FRAME: bits.pop() bits.append(options["vertical_char"]) if (options["border"] or options["preserve_internal_border"]) and options[ "hrules" - ] != NONE: + ] != HRuleStyle.NONE: bits.append("\n") bits.append(self._hrule) return "".join(bits) - def _stringify_row(self, row, options, hrule): + def _stringify_row(self, row: list[str], options: OptionsType, hrule: str) -> str: import textwrap for index, field, value, width in zip( @@ -2003,10 +2162,13 @@ class PrettyTable: ): # Enforce max widths lines = value.split("\n") - new_lines = [] + new_lines: list[str] = [] for line in lines: - if line == "None" and self.none_format.get(field) is not None: - line = self.none_format[field] + if ( + line == "None" + and (none_val := self.none_format.get(field)) is not None + ): + line = none_val if _str_block_width(line) > width: line = textwrap.fill(line, width) new_lines.append(line) @@ -2020,12 +2182,12 @@ class PrettyTable: if h > row_height: row_height = h - bits = [] + bits: list[list[str]] = [] lpad, rpad = self._get_padding_widths(options) for y in range(0, row_height): bits.append([]) if options["border"]: - if options["vrules"] in (ALL, FRAME): + if options["vrules"] in (VRuleStyle.ALL, VRuleStyle.FRAME): bits[y].append(self.vertical_char) else: bits[y].append(" ") @@ -2046,8 +2208,7 @@ class PrettyTable: else: lines = lines + [""] * d_height - y = 0 - for line in lines: + for y, line in enumerate(lines): if options["fields"] and field not in options["fields"]: continue @@ -2057,11 +2218,10 @@ class PrettyTable: + " " * rpad ) if options["border"] or options["preserve_internal_border"]: - if options["vrules"] == ALL: + if options["vrules"] == VRuleStyle.ALL: bits[y].append(self.vertical_char) else: bits[y].append(" ") - y += 1 # If only preserve_internal_border is true, then we just appended # a vertical character at the end when we wanted a space @@ -2072,21 +2232,19 @@ class PrettyTable: # If vrules is FRAME, then we just appended a space at the end # of the last field, when we really want a vertical character for y in range(0, row_height): - if options["border"] and options["vrules"] == FRAME: + if options["border"] and options["vrules"] == VRuleStyle.FRAME: bits[y].pop() bits[y].append(options["vertical_char"]) - if options["border"] and options["hrules"] == ALL: + if options["border"] and options["hrules"] == HRuleStyle.ALL: bits[row_height - 1].append("\n") bits[row_height - 1].append(hrule) - for y in range(0, row_height): - bits[y] = "".join(bits[y]) - - return "\n".join(bits) + bits_str = ["".join(bits_y) for bits_y in bits] + return "\n".join(bits_str) - def paginate(self, page_length: int = 58, line_break: str = "\f", **kwargs): - pages = [] + def paginate(self, page_length: int = 58, line_break: str = "\f", **kwargs) -> str: + pages: list[str] = [] kwargs["start"] = kwargs.get("start", 0) true_end = kwargs.get("end", self.rowcount) while True: @@ -2152,7 +2310,11 @@ class PrettyTable: import json options = self._get_options(kwargs) - json_options: Any = {"indent": 4, "separators": (",", ": "), "sort_keys": True} + json_options: dict[str, Any] = { + "indent": 4, + "separators": (",", ": "), + "sort_keys": True, + } json_options.update( {key: value for key, value in kwargs.items() if key not in options} ) @@ -2199,9 +2361,9 @@ class PrettyTable: preserve_internal_border - print a border inside the table even if border is disabled (True or False) hrules - controls printing of horizontal rules after rows. - Allowed values: ALL, FRAME, HEADER, NONE + Allowed values: HRuleStyle vrules - controls printing of vertical rules between columns. - Allowed values: FRAME, ALL, NONE + Allowed values: VRuleStyle int_format - controls formatting of integer data float_format - controls formatting of floating point data custom_format - controls formatting of any column using callable @@ -2227,10 +2389,10 @@ class PrettyTable: return string - def _get_simple_html_string(self, options): + def _get_simple_html_string(self, options: OptionsType) -> str: from html import escape - lines = [] + lines: list[str] = [] if options["xhtml"]: linebreak = "<br/>" else: @@ -2238,10 +2400,8 @@ class PrettyTable: open_tag = ["<table"] if options["attributes"]: - for attr_name in options["attributes"]: - open_tag.append( - f' {escape(attr_name)}="{escape(options["attributes"][attr_name])}"' - ) + for attr_name, attr_value in options["attributes"].items(): + open_tag.append(f' {escape(attr_name)}="{escape(attr_value)}"') open_tag.append(">") lines.append("".join(open_tag)) @@ -2288,10 +2448,10 @@ class PrettyTable: return "\n".join(lines) - def _get_formatted_html_string(self, options): + def _get_formatted_html_string(self, options: OptionsType) -> str: from html import escape - lines = [] + lines: list[str] = [] lpad, rpad = self._get_padding_widths(options) if options["xhtml"]: linebreak = "<br/>" @@ -2300,27 +2460,34 @@ class PrettyTable: open_tag = ["<table"] if options["border"]: - if options["hrules"] == ALL and options["vrules"] == ALL: + if ( + options["hrules"] == HRuleStyle.ALL + and options["vrules"] == VRuleStyle.ALL + ): open_tag.append(' frame="box" rules="all"') - elif options["hrules"] == FRAME and options["vrules"] == FRAME: + elif ( + options["hrules"] == HRuleStyle.FRAME + and options["vrules"] == VRuleStyle.FRAME + ): open_tag.append(' frame="box"') - elif options["hrules"] == FRAME and options["vrules"] == ALL: + elif ( + options["hrules"] == HRuleStyle.FRAME + and options["vrules"] == VRuleStyle.ALL + ): open_tag.append(' frame="box" rules="cols"') - elif options["hrules"] == FRAME: + elif options["hrules"] == HRuleStyle.FRAME: open_tag.append(' frame="hsides"') - elif options["hrules"] == ALL: + elif options["hrules"] == HRuleStyle.ALL: open_tag.append(' frame="hsides" rules="rows"') - elif options["vrules"] == FRAME: + elif options["vrules"] == VRuleStyle.FRAME: open_tag.append(' frame="vsides"') - elif options["vrules"] == ALL: + elif options["vrules"] == VRuleStyle.ALL: open_tag.append(' frame="vsides" rules="cols"') if not options["border"] and options["preserve_internal_border"]: open_tag.append(' rules="cols"') if options["attributes"]: - for attr_name in options["attributes"]: - open_tag.append( - f' {escape(attr_name)}="{escape(options["attributes"][attr_name])}"' - ) + for attr_name, attr_value in options["attributes"].items(): + open_tag.append(f' {escape(attr_name)}="{escape(attr_value)}"') open_tag.append(">") lines.append("".join(open_tag)) @@ -2350,8 +2517,8 @@ class PrettyTable: lines.append(" <tbody>") rows = self._get_rows(options) formatted_rows = self._format_rows(rows) - aligns = [] - valigns = [] + aligns: list[str] = [] + valigns: list[str] = [] for field in self._field_names: aligns.append( {"l": "left", "r": "right", "c": "center"}[self._align[field]] @@ -2403,9 +2570,9 @@ class PrettyTable: preserve_internal_border - print a border inside the table even if border is disabled (True or False) hrules - controls printing of horizontal rules after rows. - Allowed values: ALL, FRAME, HEADER, NONE + Allowed values: HRuleStyle vrules - controls printing of vertical rules between columns. - Allowed values: FRAME, ALL, NONE + Allowed values: VRuleStyle int_format - controls formatting of integer data float_format - controls formatting of floating point data sortby - name of field to sort rows by @@ -2421,8 +2588,8 @@ class PrettyTable: string = self._get_simple_latex_string(options) return string - def _get_simple_latex_string(self, options): - lines = [] + def _get_simple_latex_string(self, options: OptionsType) -> str: + lines: list[str] = [] wanted_fields = [] if options["fields"]: @@ -2454,10 +2621,10 @@ class PrettyTable: return "\r\n".join(lines) - def _get_formatted_latex_string(self, options): - lines = [] + def _get_formatted_latex_string(self, options: OptionsType) -> str: + lines: list[str] = [] - wanted_fields = [] + wanted_fields: list[str] = [] if options["fields"]: wanted_fields = [ field for field in self._field_names if field in options["fields"] @@ -2466,19 +2633,25 @@ class PrettyTable: wanted_fields = self._field_names wanted_alignments = [self._align[field] for field in wanted_fields] - if options["border"] and options["vrules"] == ALL: + if options["border"] and options["vrules"] == VRuleStyle.ALL: alignment_str = "|".join(wanted_alignments) elif not options["border"] and options["preserve_internal_border"]: alignment_str = "|".join(wanted_alignments) else: alignment_str = "".join(wanted_alignments) - if options["border"] and options["vrules"] in [ALL, FRAME]: + if options["border"] and options["vrules"] in [ + VRuleStyle.ALL, + VRuleStyle.FRAME, + ]: alignment_str = "|" + alignment_str + "|" begin_cmd = f"\\begin{{tabular}}{{{alignment_str}}}" lines.append(begin_cmd) - if options["border"] and options["hrules"] in [ALL, FRAME]: + if options["border"] and options["hrules"] in [ + HRuleStyle.ALL, + HRuleStyle.FRAME, + ]: lines.append("\\hline") # Headers @@ -2486,7 +2659,7 @@ class PrettyTable: lines.append(" & ".join(wanted_fields) + " \\\\") if (options["border"] or options["preserve_internal_border"]) and options[ "hrules" - ] in [ALL, HEADER]: + ] in [HRuleStyle.ALL, HRuleStyle.HEADER]: lines.append("\\hline") # Data @@ -2498,10 +2671,10 @@ class PrettyTable: d for f, d in zip(self._field_names, row) if f in wanted_fields ] lines.append(" & ".join(wanted_data) + " \\\\") - if options["border"] and options["hrules"] == ALL: + if options["border"] and options["hrules"] == HRuleStyle.ALL: lines.append("\\hline") - if options["border"] and options["hrules"] == FRAME: + if options["border"] and options["hrules"] == HRuleStyle.FRAME: lines.append("\\hline") lines.append("\\end{tabular}") @@ -2514,8 +2687,8 @@ class PrettyTable: ############################## -def _str_block_width(val): - import wcwidth # type: ignore[import-not-found] +def _str_block_width(val: str) -> int: + import wcwidth # type: ignore[import-untyped] return wcwidth.wcswidth(_re.sub("", val)) @@ -2525,7 +2698,7 @@ def _str_block_width(val): ############################## -def from_csv(fp, field_names: Any | None = None, **kwargs): +def from_csv(fp, field_names: Sequence[str] | None = None, **kwargs) -> PrettyTable: import csv fmtparams = {} @@ -2560,16 +2733,17 @@ def from_csv(fp, field_names: Any | None = None, **kwargs): return table -def from_db_cursor(cursor, **kwargs): +def from_db_cursor(cursor: Cursor, **kwargs) -> PrettyTable | None: if cursor.description: table = PrettyTable(**kwargs) table.field_names = [col[0] for col in cursor.description] for row in cursor.fetchall(): table.add_row(row) return table + return None -def from_json(json_string, **kwargs): +def from_json(json_string: str | bytes, **kwargs) -> PrettyTable: import json table = PrettyTable(**kwargs) @@ -2585,29 +2759,29 @@ class TableHandler(HTMLParser): def __init__(self, **kwargs) -> None: HTMLParser.__init__(self) self.kwargs = kwargs - self.tables: list[list] = [] + self.tables: list[PrettyTable] = [] self.last_row: list[str] = [] - self.rows: list[Any] = [] + self.rows: list[tuple[list[str], bool]] = [] self.max_row_width = 0 - self.active = None + self.active: str | None = None self.last_content = "" self.is_last_row_header = False self.colspan = 0 - def handle_starttag(self, tag, attrs) -> None: + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: self.active = tag if tag == "th": self.is_last_row_header = True for key, value in attrs: if key == "colspan": - self.colspan = int(value) + self.colspan = int(value) # type: ignore[arg-type] - def handle_endtag(self, tag) -> None: + def handle_endtag(self, tag: str) -> None: if tag in ["th", "td"]: stripped_content = self.last_content.strip() self.last_row.append(stripped_content) if self.colspan: - for i in range(1, self.colspan): + for _ in range(1, self.colspan): self.last_row.append("") self.colspan = 0 @@ -2623,10 +2797,10 @@ class TableHandler(HTMLParser): self.last_content = " " self.active = None - def handle_data(self, data) -> None: + def handle_data(self, data: str) -> None: self.last_content += data - def generate_table(self, rows): + def generate_table(self, rows: list[tuple[list[str], bool]]) -> PrettyTable: """ Generates from a list of rows a PrettyTable object. """ @@ -2644,7 +2818,7 @@ class TableHandler(HTMLParser): table.add_row(row[0]) return table - def make_fields_unique(self, fields) -> None: + def make_fields_unique(self, fields: list[str]) -> None: """ iterates over the row and make each field unique """ @@ -2654,7 +2828,7 @@ class TableHandler(HTMLParser): fields[j] += "'" -def from_html(html_code, **kwargs): +def from_html(html_code: str, **kwargs) -> list[PrettyTable]: """ Generates a list of PrettyTables from a string of HTML code. Each <table> in the HTML becomes one PrettyTable object. @@ -2665,9 +2839,9 @@ def from_html(html_code, **kwargs): return parser.tables -def from_html_one(html_code, **kwargs): +def from_html_one(html_code: str, **kwargs) -> PrettyTable: """ - Generates a PrettyTables from a string of HTML code which contains only a + Generates a PrettyTable from a string of HTML code which contains only a single <table> """ @@ -2678,3 +2852,23 @@ def from_html_one(html_code, **kwargs): msg = "More than one <table> in provided HTML code. Use from_html instead." raise ValueError(msg) return tables[0] + + +def _warn_deprecation(name: str, module_globals: dict[str, Any]) -> Any: + if (val := module_globals.get(f"_DEPRECATED_{name}")) is None: + msg = f"module '{__name__}' has no attribute '{name}" + raise AttributeError(msg) + module_globals[name] = val + if name in {"FRAME", "ALL", "NONE", "HEADER"}: + msg = ( + f"the '{name}' constant is deprecated, " + "use the 'HRuleStyle' and 'VRuleStyle' enums instead" + ) + else: + msg = f"the '{name}' constant is deprecated, use the 'TableStyle' enum instead" + warnings.warn(msg, DeprecationWarning, stacklevel=3) + return val + + +def __getattr__(name: str) -> Any: + return _warn_deprecation(name, module_globals=globals()) diff --git a/contrib/python/prettytable/py3/tests/test_colortable.py b/contrib/python/prettytable/py3/tests/test_colortable.py index 4668ed54dd6..50556384383 100644 --- a/contrib/python/prettytable/py3/tests/test_colortable.py +++ b/contrib/python/prettytable/py3/tests/test_colortable.py @@ -105,7 +105,16 @@ class TestColorTableRendering: Tests the color table rendering using the default alignment (`'c'`) """ - def test_color_table_rendering(self) -> None: + @pytest.mark.parametrize( + ["with_title", "with_header"], + [ + (False, True), # the default + (True, True), # titled + (True, False), # titled, no header + (True, True), # both title and header + ], + ) + def test_color_table_rendering(self, with_title: bool, with_header: bool) -> None: """Tests the color table rendering using the default alignment (`'c'`)""" chars = { "+": "\x1b[36m+\x1b[0m\x1b[96m", @@ -140,18 +149,40 @@ class TestColorTableRendering: (plus + minus * 3) * 6 + plus, ) - header_str = str("\n".join(header)) - body_str = str("\n".join(body)) + if with_title: + header_str = str("\n".join(header)) + else: + header_str = str(header[2]) + if with_header: + body_str = str("\n".join(body)) + else: + body_str = str("\n".join(body[2:])) table = ColorTable( ("A", "B", "C", "D", "E", "F"), theme=Themes.OCEAN, ) - table.title = "Efforts" + if with_title: + table.title = "Efforts" + table.header = with_header table.add_row([1, 2, 3, 4, 5, 6]) expected = header_str + "\n" + body_str + "\x1b[0m" result = str(table) assert expected == result + + def test_all_themes(self) -> None: + """Tests rendering with all available themes""" + table = ColorTable( + ("A", "B", "C", "D", "E", "F"), + ) + table.title = "Theme Test" + table.add_row([1, 2, 3, 4, 5, 6]) + + for theme_name, theme in vars(Themes).items(): + if isinstance(theme, Theme): + table.theme = theme + result = str(table) + assert result # Simple check to ensure rendering doesn't fail diff --git a/contrib/python/prettytable/py3/tests/test_prettytable.py b/contrib/python/prettytable/py3/tests/test_prettytable.py index 5cebb89521c..75254ca89a4 100644 --- a/contrib/python/prettytable/py3/tests/test_prettytable.py +++ b/contrib/python/prettytable/py3/tests/test_prettytable.py @@ -12,19 +12,10 @@ from pytest_lazy_fixtures import lf import prettytable from prettytable import ( - ALL, - DEFAULT, - DOUBLE_BORDER, - FRAME, - HEADER, - MARKDOWN, - MSWORD_FRIENDLY, - NONE, - ORGMODE, - PLAIN_COLUMNS, - RANDOM, - SINGLE_BORDER, + HRuleStyle, PrettyTable, + TableStyle, + VRuleStyle, from_csv, from_db_cursor, from_html, @@ -314,22 +305,22 @@ def field_name_less_table() -> PrettyTable: class TestFieldNameLessTable: """Make sure that building and stringing a table with no fieldnames works fine""" - def test_can_string_ascii(self, field_name_less_table: prettytable) -> None: + def test_can_string_ascii(self, field_name_less_table: PrettyTable) -> None: output = field_name_less_table.get_string() assert "| Field 1 | Field 2 | Field 3 | Field 4 |" in output assert "| Adelaide | 1295 | 1158259 | 600.5 |" in output - def test_can_string_html(self, field_name_less_table: prettytable) -> None: + def test_can_string_html(self, field_name_less_table: PrettyTable) -> None: output = field_name_less_table.get_html_string() assert "<th>Field 1</th>" in output assert "<td>Adelaide</td>" in output - def test_can_string_latex(self, field_name_less_table: prettytable) -> None: + def test_can_string_latex(self, field_name_less_table: PrettyTable) -> None: output = field_name_less_table.get_latex_string() assert "Field 1 & Field 2 & Field 3 & Field 4 \\\\" in output assert "Adelaide & 1295 & 1158259 & 600.5 \\\\" in output - def test_add_field_names_later(self, field_name_less_table: prettytable) -> None: + def test_add_field_names_later(self, field_name_less_table: PrettyTable) -> None: field_name_less_table.field_names = [ "City name", "Area", @@ -376,21 +367,21 @@ class TestAlignment: """Make sure alignment works regardless of when it was set""" def test_aligned_ascii( - self, aligned_before_table: prettytable, aligned_after_table: prettytable + self, aligned_before_table: PrettyTable, aligned_after_table: PrettyTable ) -> None: before = aligned_before_table.get_string() after = aligned_after_table.get_string() assert before == after def test_aligned_html( - self, aligned_before_table: prettytable, aligned_after_table: prettytable + self, aligned_before_table: PrettyTable, aligned_after_table: PrettyTable ) -> None: before = aligned_before_table.get_html_string() after = aligned_after_table.get_html_string() assert before == after def test_aligned_latex( - self, aligned_before_table: prettytable, aligned_after_table: prettytable + self, aligned_before_table: PrettyTable, aligned_after_table: PrettyTable ) -> None: before = aligned_before_table.get_latex_string() after = aligned_after_table.get_latex_string() @@ -428,7 +419,7 @@ def city_data_from_csv() -> PrettyTable: class TestOptionOverride: """Make sure all options are properly overwritten by get_string.""" - def test_border(self, city_data_prettytable: prettytable) -> None: + def test_border(self, city_data_prettytable: PrettyTable) -> None: default = city_data_prettytable.get_string() override = city_data_prettytable.get_string(border=False) assert default != override @@ -440,12 +431,12 @@ class TestOptionOverride: def test_hrules_all(self, city_data_prettytable) -> None: default = city_data_prettytable.get_string() - override = city_data_prettytable.get_string(hrules=ALL) + override = city_data_prettytable.get_string(hrules=HRuleStyle.ALL) assert default != override def test_hrules_none(self, city_data_prettytable) -> None: default = city_data_prettytable.get_string() - override = city_data_prettytable.get_string(hrules=NONE) + override = city_data_prettytable.get_string(hrules=HRuleStyle.NONE) assert default != override @@ -535,12 +526,12 @@ class TestBasic: assert len(rows) == 7 assert rows[0] == ["Adelaide", 1295, 1158259, 600.5] - def _test_no_blank_lines(self, table: prettytable) -> None: + def _test_no_blank_lines(self, table: PrettyTable) -> None: string = table.get_string() lines = string.split("\n") assert "" not in lines - def _test_all_length_equal(self, table: prettytable) -> None: + def _test_all_length_equal(self, table: PrettyTable) -> None: string = table.get_string() lines = string.split("\n") lengths = [len(line) for line in lines] @@ -608,42 +599,42 @@ class TestBasic: self, city_data_prettytable: PrettyTable ) -> None: """No table should ever have blank lines in it.""" - city_data_prettytable.hrules = NONE + city_data_prettytable.hrules = HRuleStyle.NONE self._test_no_blank_lines(city_data_prettytable) def test_all_lengths_equal_with_hrules_none( self, city_data_prettytable: PrettyTable ) -> None: """All lines in a table should be of the same length.""" - city_data_prettytable.hrules = NONE + city_data_prettytable.hrules = HRuleStyle.NONE self._test_all_length_equal(city_data_prettytable) def test_no_blank_lines_with_hrules_all( self, city_data_prettytable: PrettyTable ) -> None: """No table should ever have blank lines in it.""" - city_data_prettytable.hrules = ALL + city_data_prettytable.hrules = HRuleStyle.ALL self._test_no_blank_lines(city_data_prettytable) def test_all_lengths_equal_with_hrules_all( self, city_data_prettytable: PrettyTable ) -> None: """All lines in a table should be of the same length.""" - city_data_prettytable.hrules = ALL + city_data_prettytable.hrules = HRuleStyle.ALL self._test_all_length_equal(city_data_prettytable) def test_no_blank_lines_with_style_msword( self, city_data_prettytable: PrettyTable ) -> None: """No table should ever have blank lines in it.""" - city_data_prettytable.set_style(MSWORD_FRIENDLY) + city_data_prettytable.set_style(TableStyle.MSWORD_FRIENDLY) self._test_no_blank_lines(city_data_prettytable) def test_all_lengths_equal_with_style_msword( self, city_data_prettytable: PrettyTable ) -> None: """All lines in a table should be of the same length.""" - city_data_prettytable.set_style(MSWORD_FRIENDLY) + city_data_prettytable.set_style(TableStyle.MSWORD_FRIENDLY) self._test_all_length_equal(city_data_prettytable) def test_no_blank_lines_with_int_format( @@ -850,7 +841,7 @@ class TestBreakLine: [ ( [["value 1", "value2\nsecond line"], ["value 3", "value4"]], - ALL, + HRuleStyle.ALL, """ +---------+-------------+ | Field 1 | Field 2 | @@ -867,7 +858,7 @@ class TestBreakLine: ["value 1", "value2\nsecond line"], ["value 3\n\nother line", "value4\n\n\nvalue5"], ], - ALL, + HRuleStyle.ALL, """ +------------+-------------+ | Field 1 | Field 2 | @@ -887,7 +878,7 @@ class TestBreakLine: ["value 1", "value2\nsecond line"], ["value 3\n\nother line", "value4\n\n\nvalue5"], ], - FRAME, + HRuleStyle.FRAME, """ +------------+-------------+ | Field 1 | Field 2 | @@ -916,7 +907,7 @@ class TestBreakLine: table = PrettyTable(["Field 1", "Field 2"]) table.add_row(["value 1", "value2\nsecond line"]) table.add_row(["value 3", "value4"]) - result = table.get_html_string(hrules=ALL) + result = table.get_html_string(hrules=HRuleStyle.ALL) assert ( result.strip() == """ @@ -1455,7 +1446,7 @@ class TestPositionalJunctions: """Verify different cases for positional-junction characters""" def test_default(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) assert ( city_data_prettytable.get_string().strip() @@ -1474,7 +1465,7 @@ class TestPositionalJunctions: ) def test_no_header(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) city_data_prettytable.header = False assert ( @@ -1492,7 +1483,7 @@ class TestPositionalJunctions: ) def test_with_title(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) city_data_prettytable.title = "Title" assert ( @@ -1514,7 +1505,7 @@ class TestPositionalJunctions: ) def test_with_title_no_header(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) city_data_prettytable.title = "Title" city_data_prettytable.header = False assert ( @@ -1534,9 +1525,9 @@ class TestPositionalJunctions: ) def test_hrule_all(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) city_data_prettytable.title = "Title" - city_data_prettytable.hrules = ALL + city_data_prettytable.hrules = HRuleStyle.ALL assert ( city_data_prettytable.get_string().strip() == """ @@ -1562,8 +1553,8 @@ class TestPositionalJunctions: ) def test_vrules_none(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) - city_data_prettytable.vrules = NONE + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) + city_data_prettytable.vrules = VRuleStyle.NONE assert ( city_data_prettytable.get_string().strip() == "═══════════════════════════════════════════════════\n" @@ -1580,8 +1571,8 @@ class TestPositionalJunctions: ) def test_vrules_frame_with_title(self, city_data_prettytable: PrettyTable) -> None: - city_data_prettytable.set_style(DOUBLE_BORDER) - city_data_prettytable.vrules = FRAME + city_data_prettytable.set_style(TableStyle.DOUBLE_BORDER) + city_data_prettytable.vrules = VRuleStyle.FRAME city_data_prettytable.title = "Title" assert ( city_data_prettytable.get_string().strip() @@ -1607,7 +1598,7 @@ class TestStyle: "style, expected", [ pytest.param( - DEFAULT, + TableStyle.DEFAULT, """ +---+---------+---------+---------+ | | Field 1 | Field 2 | Field 3 | @@ -1620,7 +1611,7 @@ class TestStyle: id="DEFAULT", ), pytest.param( - MARKDOWN, # TODO fix + TableStyle.MARKDOWN, # TODO fix """ | | Field 1 | Field 2 | Field 3 | | :-: | :-----: | :-----: | :-----: | @@ -1631,7 +1622,7 @@ class TestStyle: id="MARKDOWN", ), pytest.param( - MSWORD_FRIENDLY, + TableStyle.MSWORD_FRIENDLY, """ | | Field 1 | Field 2 | Field 3 | | 1 | value 1 | value2 | value3 | @@ -1641,7 +1632,7 @@ class TestStyle: id="MSWORD_FRIENDLY", ), pytest.param( - ORGMODE, + TableStyle.ORGMODE, """ |---+---------+---------+---------| | | Field 1 | Field 2 | Field 3 | @@ -1654,7 +1645,7 @@ class TestStyle: id="ORGMODE", ), pytest.param( - PLAIN_COLUMNS, + TableStyle.PLAIN_COLUMNS, """ Field 1 Field 2 Field 3 1 value 1 value2 value3 @@ -1664,20 +1655,18 @@ class TestStyle: id="PLAIN_COLUMNS", ), pytest.param( - RANDOM, + TableStyle.RANDOM, """ -'^^^^^'^^^^^^^^^^^'^^^^^^^^^^'^^^^^^^^^^' -% 1% value 1% value2% value3% -'^^^^^'^^^^^^^^^^^'^^^^^^^^^^'^^^^^^^^^^' -% 4% value 4% value5% value6% -'^^^^^'^^^^^^^^^^^'^^^^^^^^^^'^^^^^^^^^^' -% 7% value 7% value8% value9% -'^^^^^'^^^^^^^^^^^'^^^^^^^^^^'^^^^^^^^^^' +'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^' +% 1 value 1 value2 value3% +% 4 value 4 value5 value6% +% 7 value 7 value8 value9% +'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^' """, id="RANDOM", ), pytest.param( - DOUBLE_BORDER, + TableStyle.DOUBLE_BORDER, """ ╔═══╦═════════╦═════════╦═════════╗ ║ ║ Field 1 ║ Field 2 ║ Field 3 ║ @@ -1689,7 +1678,7 @@ class TestStyle: """, ), pytest.param( - SINGLE_BORDER, + TableStyle.SINGLE_BORDER, """ ┌───┬─────────┬─────────┬─────────┐ │ │ Field 1 │ Field 2 │ Field 3 │ @@ -1721,13 +1710,13 @@ class TestStyle: # Act / Assert # This is an hrule style, not a table style with pytest.raises(ValueError): - t.set_style(ALL) + t.set_style(HRuleStyle.ALL) # type: ignore[arg-type] @pytest.mark.parametrize( "style, expected", [ pytest.param( - MARKDOWN, + TableStyle.MARKDOWN, """ | l | c | r | Align left | Align centre | Align right | | :-| :-: |-: | :----------| :----------: |-----------: | @@ -1829,7 +1818,7 @@ class TestLatexOutput: "\\end{tabular}" ) - options = {"vrules": FRAME} + options = {"vrules": VRuleStyle.FRAME} assert t.get_latex_string(format=True, **options) == ( "\\begin{tabular}{|cccc|}\r\n" "\\hline\r\n" @@ -1841,7 +1830,7 @@ class TestLatexOutput: "\\end{tabular}" ) - options = {"hrules": ALL} + options = {"hrules": HRuleStyle.ALL} assert t.get_latex_string(format=True, **options) == ( "\\begin{tabular}{|c|c|c|c|}\r\n" "\\hline\r\n" @@ -1858,7 +1847,7 @@ class TestLatexOutput: def test_latex_output_header(self) -> None: t = helper_table() - assert t.get_latex_string(format=True, hrules=HEADER) == ( + assert t.get_latex_string(format=True, hrules=HRuleStyle.HEADER) == ( "\\begin{tabular}{|c|c|c|c|}\r\n" " & Field 1 & Field 2 & Field 3 \\\\\r\n" "\\hline\r\n" @@ -2368,7 +2357,7 @@ class TestMaxTableWidth: def test_max_table_width_wide_vrules_frame(self) -> None: table = PrettyTable() table.max_table_width = 52 - table.vrules = FRAME + table.vrules = VRuleStyle.FRAME table.add_row( [ 0, @@ -2400,7 +2389,7 @@ class TestMaxTableWidth: def test_max_table_width_wide_vrules_none(self) -> None: table = PrettyTable() table.max_table_width = 52 - table.vrules = NONE + table.vrules = VRuleStyle.NONE table.add_row( [ 0, @@ -2608,3 +2597,51 @@ class TestGeneralOutput: t = helper_table() with pytest.raises(ValueError): t.get_formatted_string("pdf") + + +class TestDeprecations: + @pytest.mark.parametrize( + "module_name", + [ + "prettytable", + "prettytable.prettytable", + ], + ) + @pytest.mark.parametrize( + "name", + [ + "FRAME", + "ALL", + "NONE", + "HEADER", + ], + ) + def test_hrule_constant_deprecations(self, module_name: str, name: str) -> None: + with pytest.deprecated_call(match=f"the '{name}' constant is deprecated"): + exec(f"from {module_name} import {name}") + + @pytest.mark.parametrize( + "module_name", + [ + "prettytable", + "prettytable.prettytable", + ], + ) + @pytest.mark.parametrize( + "name", + [ + "DEFAULT", + "MSWORD_FRIENDLY", + "PLAIN_COLUMNS", + "MARKDOWN", + "ORGMODE", + "DOUBLE_BORDER", + "SINGLE_BORDER", + "RANDOM", + ], + ) + def test_table_style_constant_deprecations( + self, module_name: str, name: str + ) -> None: + with pytest.deprecated_call(match=f"the '{name}' constant is deprecated"): + exec(f"from {module_name} import {name}") diff --git a/contrib/python/prettytable/py3/ya.make b/contrib/python/prettytable/py3/ya.make index 108de3e08a7..6f495203a6b 100644 --- a/contrib/python/prettytable/py3/ya.make +++ b/contrib/python/prettytable/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.11.0) +VERSION(3.12.0) LICENSE(BSD-3-Clause) @@ -15,6 +15,7 @@ NO_LINT() PY_SRCS( TOP_LEVEL prettytable/__init__.py + prettytable/_version.py prettytable/colortable.py prettytable/prettytable.py ) diff --git a/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpack-f2pywrappers.f b/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpack-f2pywrappers.f index be8b409ec3e..655e88ff3f3 100644 --- a/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpack-f2pywrappers.f +++ b/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpack-f2pywrappers.f @@ -36,7 +36,7 @@ C It contains Fortran 77 wrappers to fortran functions. &,mneupd,mcaupd,mcaup2,mcaitr,mceigh,mcapps,mcgets,mceupd) end - subroutine f2pyinittiming(setupfunc) + subroutine f2pyinittiming_uniq(setupfunc) external setupfunc integer nopx integer nbx diff --git a/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpackmodule.c b/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpackmodule.c index 86f7ad8e04a..c11a3ac645f 100644 --- a/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpackmodule.c +++ b/contrib/python/scipy/py3/scipy/sparse/linalg/_eigen/arpack/_arpackmodule.c @@ -5280,7 +5280,7 @@ static void f2py_setup_timing(char *nopx,char *nbx,char *nrorth,char *nitref,cha f2py_timing_def[i_f2py++].data = titref; f2py_timing_def[i_f2py++].data = trvec; } -extern void F_FUNC(f2pyinittiming,F2PYINITTIMING)(void(*)(char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*)); +extern void F_FUNC(f2pyinittiming_uniq,F2PYINITTIMING)(void(*)(char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*,char*)); static void f2py_init_timing(void) { F_FUNC(f2pyinittiming,F2PYINITTIMING)(f2py_setup_timing); } |