diff options
Diffstat (limited to 'contrib/python')
48 files changed, 1555 insertions, 1374 deletions
diff --git a/contrib/python/aiohttp/.dist-info/METADATA b/contrib/python/aiohttp/.dist-info/METADATA index 7ad8cef0fd3..72c03e1c7f4 100644 --- a/contrib/python/aiohttp/.dist-info/METADATA +++ b/contrib/python/aiohttp/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: aiohttp -Version: 3.10.6 +Version: 3.10.11 Summary: Async http client/server framework (asyncio) Home-page: https://github.com/aio-libs/aiohttp Maintainer: aiohttp team <[email protected]> @@ -39,7 +39,7 @@ Requires-Dist: attrs >=17.3.0 Requires-Dist: frozenlist >=1.1.1 Requires-Dist: multidict <7.0,>=4.5 Requires-Dist: yarl <2.0,>=1.12.0 -Requires-Dist: async-timeout <5.0,>=4.0 ; python_version < "3.11" +Requires-Dist: async-timeout <6.0,>=4.0 ; python_version < "3.11" Provides-Extra: speedups Requires-Dist: brotlicffi ; (platform_python_implementation != "CPython") and extra == 'speedups' Requires-Dist: Brotli ; (platform_python_implementation == "CPython") and extra == 'speedups' @@ -64,6 +64,10 @@ Async http client/server framework :target: https://codecov.io/gh/aio-libs/aiohttp :alt: codecov.io status for master branch +.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json + :target: https://codspeed.io/aio-libs/aiohttp + :alt: Codspeed.io status for aiohttp + .. image:: https://badge.fury.io/py/aiohttp.svg :target: https://pypi.org/project/aiohttp :alt: Latest PyPI package version diff --git a/contrib/python/aiohttp/README.rst b/contrib/python/aiohttp/README.rst index 470ced9b29c..554627a42e7 100644 --- a/contrib/python/aiohttp/README.rst +++ b/contrib/python/aiohttp/README.rst @@ -17,6 +17,10 @@ Async http client/server framework :target: https://codecov.io/gh/aio-libs/aiohttp :alt: codecov.io status for master branch +.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json + :target: https://codspeed.io/aio-libs/aiohttp + :alt: Codspeed.io status for aiohttp + .. image:: https://badge.fury.io/py/aiohttp.svg :target: https://pypi.org/project/aiohttp :alt: Latest PyPI package version diff --git a/contrib/python/aiohttp/aiohttp/__init__.py b/contrib/python/aiohttp/aiohttp/__init__.py index 8830d340940..4fc7908843a 100644 --- a/contrib/python/aiohttp/aiohttp/__init__.py +++ b/contrib/python/aiohttp/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.10.6" +__version__ = "3.10.11" from typing import TYPE_CHECKING, Tuple @@ -8,6 +8,7 @@ from .client import ( ClientConnectionError, ClientConnectionResetError, ClientConnectorCertificateError, + ClientConnectorDNSError, ClientConnectorError, ClientConnectorSSLError, ClientError, @@ -127,6 +128,7 @@ __all__: Tuple[str, ...] = ( "ClientConnectionError", "ClientConnectionResetError", "ClientConnectorCertificateError", + "ClientConnectorDNSError", "ClientConnectorError", "ClientConnectorSSLError", "ClientError", diff --git a/contrib/python/aiohttp/aiohttp/_http_writer.pyx b/contrib/python/aiohttp/aiohttp/_http_writer.pyx index eff85219586..d19c20d76cc 100644 --- a/contrib/python/aiohttp/aiohttp/_http_writer.pyx +++ b/contrib/python/aiohttp/aiohttp/_http_writer.pyx @@ -127,10 +127,6 @@ def _serialize_headers(str status_line, headers): _init_writer(&writer) - for key, val in headers.items(): - _safe_header(to_str(key)) - _safe_header(to_str(val)) - try: if _write_str(&writer, status_line) < 0: raise @@ -140,6 +136,9 @@ def _serialize_headers(str status_line, headers): raise for key, val in headers.items(): + _safe_header(to_str(key)) + _safe_header(to_str(val)) + if _write_str(&writer, to_str(key)) < 0: raise if _write_byte(&writer, b':') < 0: diff --git a/contrib/python/aiohttp/aiohttp/client.py b/contrib/python/aiohttp/aiohttp/client.py index 19d27eca31a..9c7ca13fb7d 100644 --- a/contrib/python/aiohttp/aiohttp/client.py +++ b/contrib/python/aiohttp/aiohttp/client.py @@ -42,6 +42,7 @@ from .client_exceptions import ( ClientConnectionError, ClientConnectionResetError, ClientConnectorCertificateError, + ClientConnectorDNSError, ClientConnectorError, ClientConnectorSSLError, ClientError, @@ -88,7 +89,6 @@ from .helpers import ( DEBUG, BasicAuth, TimeoutHandle, - ceil_timeout, get_env_proxy_for_url, method_must_be_empty_body, sentinel, @@ -105,6 +105,7 @@ __all__ = ( "ClientConnectionError", "ClientConnectionResetError", "ClientConnectorCertificateError", + "ClientConnectorDNSError", "ClientConnectorError", "ClientConnectorSSLError", "ClientError", @@ -208,7 +209,7 @@ class ClientTimeout: # 5 Minute default read timeout -DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60) +DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60, sock_connect=30) # https://www.rfc-editor.org/rfc/rfc9110#section-9.2.2 IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE"}) @@ -354,7 +355,7 @@ class ClientSession: cookie_jar = CookieJar(loop=loop) self._cookie_jar = cookie_jar - if cookies is not None: + if cookies: self._cookie_jar.update_cookies(cookies) self._connector = connector @@ -444,7 +445,7 @@ class ClientSession: if self._base_url is None: return url else: - assert not url.is_absolute() and url.path.startswith("/") + assert not url.absolute and url.path.startswith("/") return self._base_url.join(url) async def _request( @@ -477,7 +478,7 @@ class ClientSession: ssl: Union[SSLContext, bool, Fingerprint] = True, server_hostname: Optional[str] = None, proxy_headers: Optional[LooseHeaders] = None, - trace_request_ctx: Optional[Mapping[str, str]] = None, + trace_request_ctx: Optional[Mapping[str, Any]] = None, read_bufsize: Optional[int] = None, auto_decompress: Optional[bool] = None, max_line_size: Optional[int] = None, @@ -504,13 +505,12 @@ class ClientSession: warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) redirects = 0 - history = [] + history: List[ClientResponse] = [] version = self._version params = params or {} # Merge with default headers and transform to CIMultiDict headers = self._prepare_headers(headers) - proxy_headers = self._prepare_headers(proxy_headers) try: url = self._build_url(str_or_url) @@ -526,7 +526,10 @@ class ClientSession: for i in skip_auto_headers: skip_headers.add(istr(i)) - if proxy is not None: + if proxy is None: + proxy_headers = None + else: + proxy_headers = self._prepare_headers(proxy_headers) try: proxy = URL(proxy) except ValueError as e: @@ -586,13 +589,18 @@ class ClientSession: else InvalidUrlClientError ) raise err_exc_cls(url) - if auth and auth_from_url: + # If `auth` was passed for an already authenticated URL, + # disallow only if this is the initial URL; this is to avoid issues + # with sketchy redirects that are not the caller's responsibility + if not history and (auth and auth_from_url): raise ValueError( "Cannot combine AUTH argument with " "credentials encoded in URL" ) - if auth is None: + # Override the auth with the one from the URL only if we + # have no auth, or if we got an auth from a redirect URL + if auth is None or (history and auth_from_url is not None): auth = auth_from_url if auth is None: auth = self._default_auth @@ -652,13 +660,9 @@ class ClientSession: # connection timeout try: - async with ceil_timeout( - real_timeout.connect, - ceil_threshold=real_timeout.ceil_threshold, - ): - conn = await self._connector.connect( - req, traces=traces, timeout=real_timeout - ) + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) except asyncio.TimeoutError as exc: raise ConnectionTimeoutError( f"Connection timeout to host {url}" @@ -702,7 +706,8 @@ class ClientSession: raise raise ClientOSError(*exc.args) from exc - self._cookie_jar.update_cookies(resp.cookies, resp.url) + if cookies := resp.cookies: + self._cookie_jar.update_cookies(cookies, resp.url) # redirects if resp.status in (301, 302, 303, 307, 308) and allow_redirects: diff --git a/contrib/python/aiohttp/aiohttp/client_exceptions.py b/contrib/python/aiohttp/aiohttp/client_exceptions.py index 94991c42477..2cf6cf88328 100644 --- a/contrib/python/aiohttp/aiohttp/client_exceptions.py +++ b/contrib/python/aiohttp/aiohttp/client_exceptions.py @@ -30,6 +30,7 @@ __all__ = ( "ClientConnectorError", "ClientProxyConnectionError", "ClientSSLError", + "ClientConnectorDNSError", "ClientConnectorSSLError", "ClientConnectorCertificateError", "ConnectionTimeoutError", @@ -206,6 +207,14 @@ class ClientConnectorError(ClientOSError): __reduce__ = BaseException.__reduce__ +class ClientConnectorDNSError(ClientConnectorError): + """DNS resolution failed during client connection. + + Raised in :class:`aiohttp.connector.TCPConnector` if + DNS resolution fails. + """ + + class ClientProxyConnectionError(ClientConnectorError): """Proxy connection error. diff --git a/contrib/python/aiohttp/aiohttp/client_reqrep.py b/contrib/python/aiohttp/aiohttp/client_reqrep.py index aa8f54e67b8..91605f0e83d 100644 --- a/contrib/python/aiohttp/aiohttp/client_reqrep.py +++ b/contrib/python/aiohttp/aiohttp/client_reqrep.py @@ -3,6 +3,7 @@ import codecs import contextlib import functools import io +import itertools import re import sys import traceback @@ -89,8 +90,11 @@ if TYPE_CHECKING: _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") -_YARL_SUPPORTS_EXTEND_QUERY = tuple(map(int, yarl_version.split(".")[:2])) >= (1, 11) json_re = re.compile(r"^application/(?:[\w.+-]+?\+)?json") +_YARL_SUPPORTS_HOST_SUBCOMPONENT = tuple(map(int, yarl_version.split(".")[:2])) >= ( + 1, + 13, +) def _gen_default_accept_encoding() -> str: @@ -241,6 +245,9 @@ class ClientRequest: } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) + _HOST_STRINGS = frozenset( + map("".join, itertools.product(*zip("host".upper(), "host".lower()))) + ) DEFAULT_HEADERS = { hdrs.ACCEPT: "*/*", @@ -289,29 +296,24 @@ class ClientRequest: ): if loop is None: loop = asyncio.get_event_loop() - - match = _CONTAINS_CONTROL_CHAR_RE.search(method) - if match: + if match := _CONTAINS_CONTROL_CHAR_RE.search(method): raise ValueError( f"Method cannot contain non-token characters {method!r} " - "(found at least {match.group()!r})" + f"(found at least {match.group()!r})" ) - - assert isinstance(url, URL), url - assert isinstance(proxy, (URL, type(None))), proxy + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL, url + if proxy is not None: + assert type(proxy) is URL, proxy # FIXME: session is None in tests only, need to fix tests # assert session is not None - self._session = cast("ClientSession", session) + if TYPE_CHECKING: + assert session is not None + self._session = session if params: - if _YARL_SUPPORTS_EXTEND_QUERY: - url = url.extend_query(params) - else: - q = MultiDict(url.query) - url2 = url.with_query(params) - q.extend(url2.query) - url = url.with_query(q) + url = url.extend_query(params) self.original_url = url - self.url = url.with_fragment(None) + self.url = url.with_fragment(None) if url.raw_fragment else url self.method = method.upper() self.chunked = chunked self.compress = compress @@ -342,9 +344,7 @@ class ClientRequest: if data is not None or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) - if traces is None: - traces = [] - self._traces = traces + self._traces = [] if traces is None else traces def __reset_writer(self, _: object = None) -> None: self.__writer = None @@ -354,17 +354,11 @@ class ClientRequest: return self.__writer @_writer.setter - def _writer(self, writer: Optional["asyncio.Task[None]"]) -> None: + def _writer(self, writer: "asyncio.Task[None]") -> None: if self.__writer is not None: self.__writer.remove_done_callback(self.__reset_writer) self.__writer = writer - if writer is None: - return - if writer.done(): - # The writer is already done, so we can reset it immediately. - self.__reset_writer() - else: - writer.add_done_callback(self.__reset_writer) + writer.add_done_callback(self.__reset_writer) def is_ssl(self) -> bool: return self.url.scheme in ("https", "wss") @@ -377,7 +371,7 @@ class ClientRequest: def connection_key(self) -> ConnectionKey: proxy_headers = self.proxy_headers if proxy_headers: - h: Optional[int] = hash(tuple((k, v) for k, v in proxy_headers.items())) + h: Optional[int] = hash(tuple(proxy_headers.items())) else: h = None return ConnectionKey( @@ -412,9 +406,8 @@ class ClientRequest: raise InvalidURL(url) # basic auth info - username, password = url.user, url.password - if username or password: - self.auth = helpers.BasicAuth(username or "", password or "") + if url.raw_user or url.raw_password: + self.auth = helpers.BasicAuth(url.user or "", url.password or "") def update_version(self, version: Union[http.HttpVersion, str]) -> None: """Convert request version to two elements tuple. @@ -435,26 +428,49 @@ class ClientRequest: """Update request headers.""" self.headers: CIMultiDict[str] = CIMultiDict() - # add host - netloc = cast(str, self.url.raw_host) - if helpers.is_ipv6_address(netloc): - netloc = f"[{netloc}]" - # See https://github.com/aio-libs/aiohttp/issues/3636. - netloc = netloc.rstrip(".") - if self.url.port is not None and not self.url.is_default_port(): - netloc += ":" + str(self.url.port) - self.headers[hdrs.HOST] = netloc + # Build the host header + if _YARL_SUPPORTS_HOST_SUBCOMPONENT: + host = self.url.host_subcomponent + # host_subcomponent is None when the URL is a relative URL. + # but we know we do not have a relative URL here. + assert host is not None + else: + host = cast(str, self.url.raw_host) + if helpers.is_ipv6_address(host): + host = f"[{host}]" - if headers: - if isinstance(headers, (dict, MultiDictProxy, MultiDict)): - headers = headers.items() + if host[-1] == ".": + # Remove all trailing dots from the netloc as while + # they are valid FQDNs in DNS, TLS validation fails. + # See https://github.com/aio-libs/aiohttp/issues/3636. + # To avoid string manipulation we only call rstrip if + # the last character is a dot. + host = host.rstrip(".") - for key, value in headers: # type: ignore[misc] - # A special case for Host header - if key.lower() == "host": - self.headers[key] = value - else: - self.headers.add(key, value) + # If explicit port is not None, it means that the port was + # explicitly specified in the URL. In this case we check + # if its not the default port for the scheme and add it to + # the host header. We check explicit_port first because + # yarl caches explicit_port and its likely to already be + # in the cache and non-default port URLs are far less common. + explicit_port = self.url.explicit_port + if explicit_port is not None and not self.url.is_default_port(): + host = f"{host}:{explicit_port}" + + self.headers[hdrs.HOST] = host + + if not headers: + return + + if isinstance(headers, (dict, MultiDictProxy, MultiDict)): + headers = headers.items() + + for key, value in headers: # type: ignore[misc] + # A special case for Host header + if key in self._HOST_STRINGS: + self.headers[key] = value + else: + self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers: Optional[Iterable[str]]) -> None: if skip_auto_headers is not None: @@ -592,7 +608,10 @@ class ClientRequest: def update_expect_continue(self, expect: bool = False) -> None: if expect: self.headers[hdrs.EXPECT] = "100-continue" - elif self.headers.get(hdrs.EXPECT, "").lower() == "100-continue": + elif ( + hdrs.EXPECT in self.headers + and self.headers[hdrs.EXPECT].lower() == "100-continue" + ): expect = True if expect: @@ -604,10 +623,16 @@ class ClientRequest: proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders], ) -> None: + self.proxy = proxy + if proxy is None: + self.proxy_auth = None + self.proxy_headers = None + return + if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") - self.proxy = proxy self.proxy_auth = proxy_auth + if proxy_headers is not None and not isinstance( proxy_headers, (MultiDict, MultiDictProxy) ): @@ -619,10 +644,8 @@ class ClientRequest: # keep alive not supported at all return False if self.version == HttpVersion10: - if self.headers.get(hdrs.CONNECTION) == "keep-alive": - return True - else: # no headers means we close for Http 1.0 - return False + # no headers means we close for Http 1.0 + return self.headers.get(hdrs.CONNECTION) == "keep-alive" elif self.headers.get(hdrs.CONNECTION) == "close": return False @@ -683,17 +706,19 @@ class ClientRequest: # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: - connect_host = self.url.raw_host - assert connect_host is not None - if helpers.is_ipv6_address(connect_host): - connect_host = f"[{connect_host}]" + if _YARL_SUPPORTS_HOST_SUBCOMPONENT: + connect_host = self.url.host_subcomponent + assert connect_host is not None + else: + connect_host = self.url.raw_host + assert connect_host is not None + if helpers.is_ipv6_address(connect_host): + connect_host = f"[{connect_host}]" path = f"{connect_host}:{self.url.port}" elif self.proxy and not self.is_ssl(): path = str(self.url) else: - path = self.url.raw_path - if self.url.raw_query_string: - path += "?" + self.url.raw_query_string + path = self.url.raw_path_qs protocol = conn.protocol assert protocol is not None @@ -745,6 +770,7 @@ class ClientRequest: await writer.write_headers(status_line, self.headers) coro = self.write_bytes(writer, conn) + task: Optional["asyncio.Task[None]"] if sys.version_info >= (3, 12): # Optimization for Python 3.12, try to write # bytes immediately to avoid having to schedule @@ -753,13 +779,17 @@ class ClientRequest: else: task = self.loop.create_task(coro) - self._writer = task + if task.done(): + task = None + else: + self._writer = task + response_class = self.response_class assert response_class is not None self.response = response_class( self.method, self.original_url, - writer=self._writer, + writer=task, continue100=self._continue, timer=self._timer, request_info=self.request_info, @@ -770,9 +800,9 @@ class ClientRequest: return self.response async def close(self) -> None: - if self._writer is not None: + if self.__writer is not None: try: - await self._writer + await self.__writer except asyncio.CancelledError: if ( sys.version_info >= (3, 11) @@ -782,11 +812,11 @@ class ClientRequest: raise def terminate(self) -> None: - if self._writer is not None: + if self.__writer is not None: if not self.loop.is_closed(): - self._writer.cancel() - self._writer.remove_done_callback(self.__reset_writer) - self._writer = None + self.__writer.cancel() + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = None async def _on_chunk_request_sent(self, method: str, url: URL, chunk: bytes) -> None: for trace in self._traces: @@ -799,6 +829,9 @@ class ClientRequest: await trace.send_request_headers(method, url, headers) +_CONNECTION_CLOSED_EXCEPTION = ClientConnectionError("Connection closed") + + class ClientResponse(HeadersMixin): # Some of these attributes are None when created, @@ -827,7 +860,7 @@ class ClientResponse(HeadersMixin): method: str, url: URL, *, - writer: "asyncio.Task[None]", + writer: "Optional[asyncio.Task[None]]", continue100: Optional["asyncio.Future[bool]"], timer: BaseTimerContext, request_info: RequestInfo, @@ -841,9 +874,10 @@ class ClientResponse(HeadersMixin): self.cookies = SimpleCookie() self._real_url = url - self._url = url.with_fragment(None) - self._body: Any = None - self._writer: Optional[asyncio.Task[None]] = writer + self._url = url.with_fragment(None) if url.raw_fragment else url + self._body: Optional[bytes] = None + if writer is not None: + self._writer = writer self._continue = continue100 # None by default self._closed = True self._history: Tuple[ClientResponse, ...] = () @@ -871,18 +905,24 @@ class ClientResponse(HeadersMixin): @property def _writer(self) -> Optional["asyncio.Task[None]"]: + """The writer task for streaming data. + + _writer is only provided for backwards compatibility + for subclasses that may need to access it. + """ return self.__writer @_writer.setter def _writer(self, writer: Optional["asyncio.Task[None]"]) -> None: + """Set the writer task for streaming data.""" if self.__writer is not None: self.__writer.remove_done_callback(self.__reset_writer) self.__writer = writer if writer is None: return if writer.done(): - # The writer is already done, so we can reset it immediately. - self.__reset_writer() + # The writer is already done, so we can clear it immediately. + self.__writer = None else: writer.add_done_callback(self.__reset_writer) @@ -1125,16 +1165,16 @@ class ClientResponse(HeadersMixin): def _release_connection(self) -> None: if self._connection is not None: - if self._writer is None: + if self.__writer is None: self._connection.release() self._connection = None else: - self._writer.add_done_callback(lambda f: self._release_connection()) + self.__writer.add_done_callback(lambda f: self._release_connection()) async def _wait_released(self) -> None: - if self._writer is not None: + if self.__writer is not None: try: - await self._writer + await self.__writer except asyncio.CancelledError: if ( sys.version_info >= (3, 11) @@ -1145,20 +1185,20 @@ class ClientResponse(HeadersMixin): self._release_connection() def _cleanup_writer(self) -> None: - if self._writer is not None: - self._writer.cancel() + if self.__writer is not None: + self.__writer.cancel() self._session = None def _notify_content(self) -> None: content = self.content if content and content.exception() is None: - set_exception(content, ClientConnectionError("Connection closed")) + set_exception(content, _CONNECTION_CLOSED_EXCEPTION) self._released = True async def wait_for_close(self) -> None: - if self._writer is not None: + if self.__writer is not None: try: - await self._writer + await self.__writer except asyncio.CancelledError: if ( sys.version_info >= (3, 11) @@ -1186,7 +1226,7 @@ class ClientResponse(HeadersMixin): protocol = self._connection and self._connection.protocol if protocol is None or not protocol.upgraded: await self._wait_released() # Underlying connection released - return self._body # type: ignore[no-any-return] + return self._body def get_encoding(self) -> str: ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() @@ -1219,9 +1259,7 @@ class ClientResponse(HeadersMixin): if encoding is None: encoding = self.get_encoding() - return self._body.decode( # type: ignore[no-any-return,union-attr] - encoding, errors=errors - ) + return self._body.decode(encoding, errors=errors) # type: ignore[union-attr] async def json( self, diff --git a/contrib/python/aiohttp/aiohttp/client_ws.py b/contrib/python/aiohttp/aiohttp/client_ws.py index c6b5da5103b..96e2c59715c 100644 --- a/contrib/python/aiohttp/aiohttp/client_ws.py +++ b/contrib/python/aiohttp/aiohttp/client_ws.py @@ -249,10 +249,30 @@ class ClientWebSocketResponse: self._reader.feed_data(WS_CLOSING_MESSAGE, 0) await self._close_wait - if not self._closed: - self._set_closed() + if self._closed: + return False + + self._set_closed() + try: + await self._writer.close(code, message) + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if self._close_code: + self._response.close() + return True + + while True: try: - await self._writer.close(code, message) + async with async_timeout.timeout(self._timeout): + msg = await self._reader.read() except asyncio.CancelledError: self._close_code = WSCloseCode.ABNORMAL_CLOSURE self._response.close() @@ -263,31 +283,11 @@ class ClientWebSocketResponse: self._response.close() return True - if self._close_code: + if msg.type is WSMsgType.CLOSE: + self._close_code = msg.data self._response.close() return True - while True: - try: - async with async_timeout.timeout(self._timeout): - msg = await self._reader.read() - except asyncio.CancelledError: - self._close_code = WSCloseCode.ABNORMAL_CLOSURE - self._response.close() - raise - except Exception as exc: - self._close_code = WSCloseCode.ABNORMAL_CLOSURE - self._exception = exc - self._response.close() - return True - - if msg.type is WSMsgType.CLOSE: - self._close_code = msg.data - self._response.close() - return True - else: - return False - async def receive(self, timeout: Optional[float] = None) -> WSMessage: receive_timeout = timeout or self._receive_timeout diff --git a/contrib/python/aiohttp/aiohttp/compression_utils.py b/contrib/python/aiohttp/aiohttp/compression_utils.py index ab4a2f1cc84..ebe8857f487 100644 --- a/contrib/python/aiohttp/aiohttp/compression_utils.py +++ b/contrib/python/aiohttp/aiohttp/compression_utils.py @@ -70,6 +70,14 @@ class ZLibCompressor(ZlibBaseHandler): return self._compressor.compress(data) async def compress(self, data: bytes) -> bytes: + """Compress the data and returned the compressed bytes. + + Note that flush() must be called after the last call to compress() + + If the data size is large than the max_sync_chunk_size, the compression + will be done in the executor. Otherwise, the compression will be done + in the event loop. + """ async with self._compress_lock: # To ensure the stream is consistent in the event # there are multiple writers, we need to lock @@ -79,8 +87,8 @@ class ZLibCompressor(ZlibBaseHandler): self._max_sync_chunk_size is not None and len(data) > self._max_sync_chunk_size ): - return await asyncio.get_event_loop().run_in_executor( - self._executor, self.compress_sync, data + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._compressor.compress, data ) return self.compress_sync(data) @@ -107,12 +115,18 @@ class ZLibDecompressor(ZlibBaseHandler): return self._decompressor.decompress(data, max_length) async def decompress(self, data: bytes, max_length: int = 0) -> bytes: + """Decompress the data and return the decompressed bytes. + + If the data size is large than the max_sync_chunk_size, the decompression + will be done in the executor. Otherwise, the decompression will be done + in the event loop. + """ if ( self._max_sync_chunk_size is not None and len(data) > self._max_sync_chunk_size ): - return await asyncio.get_event_loop().run_in_executor( - self._executor, self.decompress_sync, data, max_length + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._decompressor.decompress, data, max_length ) return self.decompress_sync(data, max_length) diff --git a/contrib/python/aiohttp/aiohttp/connector.py b/contrib/python/aiohttp/aiohttp/connector.py index 8288a0115b7..7d0160fa038 100644 --- a/contrib/python/aiohttp/aiohttp/connector.py +++ b/contrib/python/aiohttp/aiohttp/connector.py @@ -5,11 +5,10 @@ import socket import sys import traceback import warnings -from collections import defaultdict, deque +from collections import OrderedDict, defaultdict from contextlib import suppress from http import HTTPStatus -from http.cookies import SimpleCookie -from itertools import cycle, islice +from itertools import chain, cycle, islice from time import monotonic from types import TracebackType from typing import ( @@ -39,6 +38,7 @@ from .abc import AbstractResolver, ResolveResult from .client_exceptions import ( ClientConnectionError, ClientConnectorCertificateError, + ClientConnectorDNSError, ClientConnectorError, ClientConnectorSSLError, ClientHttpProxyError, @@ -50,8 +50,14 @@ from .client_exceptions import ( ) from .client_proto import ResponseHandler from .client_reqrep import ClientRequest, Fingerprint, _merge_ssl_params -from .helpers import ceil_timeout, is_ip_address, noop, sentinel -from .locks import EventResultOrError +from .helpers import ( + ceil_timeout, + is_ip_address, + noop, + sentinel, + set_exception, + set_result, +) from .resolver import DefaultResolver try: @@ -70,6 +76,14 @@ WS_SCHEMA_SET = frozenset({"ws", "wss"}) HTTP_AND_EMPTY_SCHEMA_SET = HTTP_SCHEMA_SET | EMPTY_SCHEMA_SET HIGH_LEVEL_SCHEMA_SET = HTTP_AND_EMPTY_SCHEMA_SET | WS_SCHEMA_SET +NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < ( + 3, + 13, + 1, +) or sys.version_info < (3, 12, 7) +# Cleanup closed is no longer needed after https://github.com/python/cpython/pull/118960 +# which first appeared in Python 3.12.7 and 3.13.1 + __all__ = ("BaseConnector", "TCPConnector", "UnixConnector", "NamedPipeConnector") @@ -103,7 +117,6 @@ class _DeprecationWaiter: class Connection: _source_traceback = None - _transport = None def __init__( self, @@ -181,9 +194,7 @@ class Connection: self._notify_release() if self._protocol is not None: - self._connector._release( - self._key, self._protocol, should_close=self._protocol.should_close - ) + self._connector._release(self._key, self._protocol) self._protocol = None @property @@ -194,8 +205,10 @@ class Connection: class _TransportPlaceholder: """placeholder for BaseConnector.connect function""" + __slots__ = () + def close(self) -> None: - pass + """Close the placeholder transport.""" class BaseConnector: @@ -260,18 +273,31 @@ class BaseConnector: self._force_close = force_close # {host_key: FIFO list of waiters} - self._waiters = defaultdict(deque) # type: ignore[var-annotated] + # The FIFO is implemented with an OrderedDict with None keys because + # python does not have an ordered set. + self._waiters: DefaultDict[ + ConnectionKey, OrderedDict[asyncio.Future[None], None] + ] = defaultdict(OrderedDict) self._loop = loop self._factory = functools.partial(ResponseHandler, loop=loop) - self.cookies = SimpleCookie() - # start keep-alive connection cleanup task self._cleanup_handle: Optional[asyncio.TimerHandle] = None # start cleanup closed transports task self._cleanup_closed_handle: Optional[asyncio.TimerHandle] = None + + if enable_cleanup_closed and not NEEDS_CLEANUP_CLOSED: + warnings.warn( + "enable_cleanup_closed ignored because " + "https://github.com/python/cpython/pull/118960 is fixed " + f"in Python version {sys.version_info}", + DeprecationWarning, + stacklevel=2, + ) + enable_cleanup_closed = False + self._cleanup_closed_disabled = not enable_cleanup_closed self._cleanup_closed_transports: List[Optional[asyncio.Transport]] = [] self._cleanup_closed() @@ -350,28 +376,22 @@ class BaseConnector: # recreate it ever! self._cleanup_handle = None - now = self._loop.time() + now = monotonic() timeout = self._keepalive_timeout if self._conns: connections = {} deadline = now - timeout for key, conns in self._conns.items(): - alive = [] + alive: List[Tuple[ResponseHandler, float]] = [] for proto, use_time in conns: - if proto.is_connected(): - if use_time - deadline < 0: - transport = proto.transport - proto.close() - if key.is_ssl and not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transport) - else: - alive.append((proto, use_time)) - else: - transport = proto.transport - proto.close() - if key.is_ssl and not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transport) + if proto.is_connected() and use_time - deadline >= 0: + alive.append((proto, use_time)) + continue + transport = proto.transport + proto.close() + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) if alive: connections[key] = alive @@ -387,17 +407,6 @@ class BaseConnector: timeout_ceil_threshold=self._timeout_ceil_threshold, ) - def _drop_acquired_per_host( - self, key: "ConnectionKey", val: ResponseHandler - ) -> None: - acquired_per_host = self._acquired_per_host - if key not in acquired_per_host: - return - conns = acquired_per_host[key] - conns.remove(val) - if not conns: - del self._acquired_per_host[key] - def _cleanup_closed(self) -> None: """Double confirmation for transport close. @@ -458,6 +467,9 @@ class BaseConnector: finally: self._conns.clear() self._acquired.clear() + for keyed_waiters in self._waiters.values(): + for keyed_waiter in keyed_waiters: + keyed_waiter.cancel() self._waiters.clear() self._cleanup_handle = None self._cleanup_closed_transports.clear() @@ -480,139 +492,153 @@ class BaseConnector: If it returns less than 1 means that there are no connections available. """ - if self._limit: - # total calc available connections - available = self._limit - len(self._acquired) + # check total available connections + # If there are no limits, this will always return 1 + total_remain = 1 - # check limit per host - if ( - self._limit_per_host - and available > 0 - and key in self._acquired_per_host - ): - acquired = self._acquired_per_host.get(key) - assert acquired is not None - available = self._limit_per_host - len(acquired) + if self._limit and (total_remain := self._limit - len(self._acquired)) <= 0: + return total_remain - elif self._limit_per_host and key in self._acquired_per_host: - # check limit per host - acquired = self._acquired_per_host.get(key) - assert acquired is not None - available = self._limit_per_host - len(acquired) - else: - available = 1 + # check limit per host + if host_remain := self._limit_per_host: + if acquired := self._acquired_per_host.get(key): + host_remain -= len(acquired) + if total_remain > host_remain: + return host_remain - return available + return total_remain async def connect( self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" ) -> Connection: """Get from pool or create new connection.""" key = req.connection_key - available = self._available_connections(key) - - # Wait if there are no available connections or if there are/were - # waiters (i.e. don't steal connection from a waiter about to wake up) - if available <= 0 or key in self._waiters: - fut = self._loop.create_future() - - # This connection will now count towards the limit. - self._waiters[key].append(fut) - - if traces: - for trace in traces: - await trace.send_connection_queued_start() - - try: - await fut - except BaseException as e: - if key in self._waiters: - # remove a waiter even if it was cancelled, normally it's - # removed when it's notified - try: - self._waiters[key].remove(fut) - except ValueError: # fut may no longer be in list - pass - - raise e - finally: - if key in self._waiters and not self._waiters[key]: - del self._waiters[key] + if (conn := await self._get(key, traces)) is not None: + # If we do not have to wait and we can get a connection from the pool + # we can avoid the timeout ceil logic and directly return the connection + return conn - if traces: - for trace in traces: - await trace.send_connection_queued_end() + async with ceil_timeout(timeout.connect, timeout.ceil_threshold): + if self._available_connections(key) <= 0: + await self._wait_for_available_connection(key, traces) + if (conn := await self._get(key, traces)) is not None: + return conn - proto = self._get(key) - if proto is None: placeholder = cast(ResponseHandler, _TransportPlaceholder()) self._acquired.add(placeholder) self._acquired_per_host[key].add(placeholder) - if traces: - for trace in traces: - await trace.send_connection_create_start() - try: + # Traces are done inside the try block to ensure that the + # that the placeholder is still cleaned up if an exception + # is raised. + if traces: + for trace in traces: + await trace.send_connection_create_start() proto = await self._create_connection(req, traces, timeout) - if self._closed: - proto.close() - raise ClientConnectionError("Connector is closed.") + if traces: + for trace in traces: + await trace.send_connection_create_end() except BaseException: - if not self._closed: - self._acquired.remove(placeholder) - self._drop_acquired_per_host(key, placeholder) - self._release_waiter() + self._release_acquired(key, placeholder) raise else: - if not self._closed: - self._acquired.remove(placeholder) - self._drop_acquired_per_host(key, placeholder) - - if traces: - for trace in traces: - await trace.send_connection_create_end() - else: - if traces: - # Acquire the connection to prevent race conditions with limits - placeholder = cast(ResponseHandler, _TransportPlaceholder()) - self._acquired.add(placeholder) - self._acquired_per_host[key].add(placeholder) - for trace in traces: - await trace.send_connection_reuseconn() - self._acquired.remove(placeholder) - self._drop_acquired_per_host(key, placeholder) + if self._closed: + proto.close() + raise ClientConnectionError("Connector is closed.") + # The connection was successfully created, drop the placeholder + # and add the real connection to the acquired set. There should + # be no awaits after the proto is added to the acquired set + # to ensure that the connection is not left in the acquired set + # on cancellation. + acquired_per_host = self._acquired_per_host[key] + self._acquired.remove(placeholder) + acquired_per_host.remove(placeholder) self._acquired.add(proto) - self._acquired_per_host[key].add(proto) + acquired_per_host.add(proto) return Connection(self, key, proto, self._loop) - def _get(self, key: "ConnectionKey") -> Optional[ResponseHandler]: + async def _wait_for_available_connection( + self, key: "ConnectionKey", traces: List["Trace"] + ) -> None: + """Wait for an available connection slot.""" + # We loop here because there is a race between + # the connection limit check and the connection + # being acquired. If the connection is acquired + # between the check and the await statement, we + # need to loop again to check if the connection + # slot is still available. + attempts = 0 + while True: + fut: asyncio.Future[None] = self._loop.create_future() + keyed_waiters = self._waiters[key] + keyed_waiters[fut] = None + if attempts: + # If we have waited before, we need to move the waiter + # to the front of the queue as otherwise we might get + # starved and hit the timeout. + keyed_waiters.move_to_end(fut, last=False) + + try: + # Traces happen in the try block to ensure that the + # the waiter is still cleaned up if an exception is raised. + if traces: + for trace in traces: + await trace.send_connection_queued_start() + await fut + if traces: + for trace in traces: + await trace.send_connection_queued_end() + finally: + # pop the waiter from the queue if its still + # there and not already removed by _release_waiter + keyed_waiters.pop(fut, None) + if not self._waiters.get(key, True): + del self._waiters[key] + + if self._available_connections(key) > 0: + break + attempts += 1 + + async def _get( + self, key: "ConnectionKey", traces: List["Trace"] + ) -> Optional[Connection]: + """Get next reusable connection for the key or None. + + The connection will be marked as acquired. + """ try: conns = self._conns[key] except KeyError: return None - t1 = self._loop.time() + t1 = monotonic() while conns: proto, t0 = conns.pop() - if proto.is_connected(): - if t1 - t0 > self._keepalive_timeout: - transport = proto.transport - proto.close() - # only for SSL transports - if key.is_ssl and not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transport) - else: - if not conns: - # The very last connection was reclaimed: drop the key - del self._conns[key] - return proto - else: - transport = proto.transport - proto.close() - if key.is_ssl and not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transport) + # We will we reuse the connection if its connected and + # the keepalive timeout has not been exceeded + if proto.is_connected() and t1 - t0 <= self._keepalive_timeout: + if not conns: + # The very last connection was reclaimed: drop the key + del self._conns[key] + self._acquired.add(proto) + self._acquired_per_host[key].add(proto) + if traces: + for trace in traces: + try: + await trace.send_connection_reuseconn() + except BaseException: + self._release_acquired(key, proto) + raise + return Connection(self, key, proto, self._loop) + + # Connection cannot be reused, close it + transport = proto.transport + proto.close() + # only for SSL transports + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) # No more connections: drop the key del self._conns[key] @@ -630,7 +656,7 @@ class BaseConnector: # Having the dict keys ordered this avoids to iterate # at the same order at each call. - queues = list(self._waiters.keys()) + queues = list(self._waiters) random.shuffle(queues) for key in queues: @@ -639,25 +665,23 @@ class BaseConnector: waiters = self._waiters[key] while waiters: - waiter = waiters.popleft() + waiter, _ = waiters.popitem(last=False) if not waiter.done(): waiter.set_result(None) return def _release_acquired(self, key: "ConnectionKey", proto: ResponseHandler) -> None: + """Release acquired connection.""" if self._closed: # acquired connection is already released on connector closing return - try: - self._acquired.remove(proto) - self._drop_acquired_per_host(key, proto) - except KeyError: # pragma: no cover - # this may be result of undetermenistic order of objects - # finalization due garbage collection. - pass - else: - self._release_waiter() + self._acquired.discard(proto) + if conns := self._acquired_per_host.get(key): + conns.discard(proto) + if not conns: + del self._acquired_per_host[key] + self._release_waiter() def _release( self, @@ -681,20 +705,21 @@ class BaseConnector: if key.is_ssl and not self._cleanup_closed_disabled: self._cleanup_closed_transports.append(transport) - else: - conns = self._conns.get(key) - if conns is None: - conns = self._conns[key] = [] - conns.append((protocol, self._loop.time())) + return - if self._cleanup_handle is None: - self._cleanup_handle = helpers.weakref_handle( - self, - "_cleanup", - self._keepalive_timeout, - self._loop, - timeout_ceil_threshold=self._timeout_ceil_threshold, - ) + conns = self._conns.get(key) + if conns is None: + conns = self._conns[key] = [] + conns.append((protocol, monotonic())) + + if self._cleanup_handle is None: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + self._keepalive_timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) async def _create_connection( self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" @@ -840,7 +865,9 @@ class TCPConnector(BaseConnector): self._use_dns_cache = use_dns_cache self._cached_hosts = _DNSCacheTable(ttl=ttl_dns_cache) - self._throttle_dns_events: Dict[Tuple[str, int], EventResultOrError] = {} + self._throttle_dns_futures: Dict[ + Tuple[str, int], Set["asyncio.Future[None]"] + ] = {} self._family = family self._local_addr_infos = aiohappyeyeballs.addr_to_addr_infos(local_addr) self._happy_eyeballs_delay = happy_eyeballs_delay @@ -849,8 +876,8 @@ class TCPConnector(BaseConnector): def close(self) -> Awaitable[None]: """Close all ongoing DNS calls.""" - for ev in self._throttle_dns_events.values(): - ev.cancel() + for fut in chain.from_iterable(self._throttle_dns_futures.values()): + fut.cancel() for t in self._resolve_host_tasks: t.cancel() @@ -918,22 +945,46 @@ class TCPConnector(BaseConnector): await trace.send_dns_cache_hit(host) return result + futures: Set["asyncio.Future[None]"] # # If multiple connectors are resolving the same host, we wait # for the first one to resolve and then use the result for all of them. - # We use a throttle event to ensure that we only resolve the host once + # We use a throttle to ensure that we only resolve the host once # and then use the result for all the waiters. # + if key in self._throttle_dns_futures: + # get futures early, before any await (#4014) + futures = self._throttle_dns_futures[key] + future: asyncio.Future[None] = self._loop.create_future() + futures.add(future) + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + try: + await future + finally: + futures.discard(future) + return self._cached_hosts.next_addrs(key) + + # update dict early, before any await (#4014) + self._throttle_dns_futures[key] = futures = set() # In this case we need to create a task to ensure that we can shield # the task from cancellation as cancelling this lookup should not cancel # the underlying lookup or else the cancel event will get broadcast to # all the waiters across all connections. # - resolved_host_task = asyncio.create_task( - self._resolve_host_with_throttle(key, host, port, traces) - ) - self._resolve_host_tasks.add(resolved_host_task) - resolved_host_task.add_done_callback(self._resolve_host_tasks.discard) + coro = self._resolve_host_with_throttle(key, host, port, futures, traces) + loop = asyncio.get_running_loop() + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send immediately + resolved_host_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + resolved_host_task = loop.create_task(coro) + + if not resolved_host_task.done(): + self._resolve_host_tasks.add(resolved_host_task) + resolved_host_task.add_done_callback(self._resolve_host_tasks.discard) + try: return await asyncio.shield(resolved_host_task) except asyncio.CancelledError: @@ -950,42 +1001,39 @@ class TCPConnector(BaseConnector): key: Tuple[str, int], host: str, port: int, + futures: Set["asyncio.Future[None]"], traces: Optional[Sequence["Trace"]], ) -> List[ResolveResult]: - """Resolve host with a dns events throttle.""" - if key in self._throttle_dns_events: - # get event early, before any await (#4014) - event = self._throttle_dns_events[key] + """Resolve host and set result for all waiters. + + This method must be run in a task and shielded from cancellation + to avoid cancelling the underlying lookup. + """ + if traces: + for trace in traces: + await trace.send_dns_cache_miss(host) + try: if traces: for trace in traces: - await trace.send_dns_cache_hit(host) - await event.wait() - else: - # update dict early, before any await (#4014) - self._throttle_dns_events[key] = EventResultOrError(self._loop) + await trace.send_dns_resolvehost_start(host) + + addrs = await self._resolver.resolve(host, port, family=self._family) if traces: for trace in traces: - await trace.send_dns_cache_miss(host) - try: - - if traces: - for trace in traces: - await trace.send_dns_resolvehost_start(host) - - addrs = await self._resolver.resolve(host, port, family=self._family) - if traces: - for trace in traces: - await trace.send_dns_resolvehost_end(host) + await trace.send_dns_resolvehost_end(host) - self._cached_hosts.add(key, addrs) - self._throttle_dns_events[key].set() - except BaseException as e: - # any DNS exception, independently of the implementation - # is set for the waiters to raise the same exception. - self._throttle_dns_events[key].set(exc=e) - raise - finally: - self._throttle_dns_events.pop(key) + self._cached_hosts.add(key, addrs) + for fut in futures: + set_result(fut, None) + except BaseException as e: + # any DNS exception is set for the waiters to raise the same exception. + # This coro is always run in task that is shielded from cancellation so + # we should never be propagating cancellation here. + for fut in futures: + set_exception(fut, e) + raise + finally: + self._throttle_dns_futures.pop(key) return self._cached_hosts.next_addrs(key) @@ -1290,7 +1338,7 @@ class TCPConnector(BaseConnector): raise # in case of proxy it is not ClientProxyConnectionError # it is problem of resolving proxy ip itself - raise ClientConnectorError(req.connection_key, exc) from exc + raise ClientConnectorDNSError(req.connection_key, exc) from exc last_exc: Optional[Exception] = None addr_infos = self._convert_hosts_to_addr_infos(hosts) @@ -1311,7 +1359,7 @@ class TCPConnector(BaseConnector): req=req, client_error=client_error, ) - except ClientConnectorError as exc: + except (ClientConnectorError, asyncio.TimeoutError) as exc: last_exc = exc aiohappyeyeballs.pop_addr_infos_interleave(addr_infos, self._interleave) continue @@ -1412,7 +1460,6 @@ class TCPConnector(BaseConnector): raise else: conn._protocol = None - conn._transport = None try: if resp.status != 200: message = resp.reason diff --git a/contrib/python/aiohttp/aiohttp/helpers.py b/contrib/python/aiohttp/aiohttp/helpers.py index 40705b16d71..6ee70786cfb 100644 --- a/contrib/python/aiohttp/aiohttp/helpers.py +++ b/contrib/python/aiohttp/aiohttp/helpers.py @@ -163,7 +163,9 @@ class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])): """Create BasicAuth from url.""" if not isinstance(url, URL): raise TypeError("url should be yarl.URL instance") - if url.user is None and url.password is None: + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: return None return cls(url.user or "", url.password or "", encoding=encoding) @@ -174,11 +176,12 @@ class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])): def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]: - auth = BasicAuth.from_url(url) - if auth is None: + """Remove user and password from URL if present and return BasicAuth object.""" + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: return url, None - else: - return url.with_user(None), auth + return url.with_user(None), BasicAuth(url.user or "", url.password or "") def netrc_from_env() -> Optional[netrc.netrc]: @@ -614,6 +617,8 @@ def calculate_timeout_when( class TimeoutHandle: """Timeout handle""" + __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks") + def __init__( self, loop: asyncio.AbstractEventLoop, @@ -662,11 +667,17 @@ class TimeoutHandle: class BaseTimerContext(ContextManager["BaseTimerContext"]): + + __slots__ = () + def assert_timeout(self) -> None: """Raise TimeoutError if timeout has been exceeded.""" class TimerNoop(BaseTimerContext): + + __slots__ = () + def __enter__(self) -> BaseTimerContext: return self @@ -682,10 +693,13 @@ class TimerNoop(BaseTimerContext): class TimerContext(BaseTimerContext): """Low resolution timeout context manager""" + __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling") + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._tasks: List[asyncio.Task[Any]] = [] self._cancelled = False + self._cancelling = 0 def assert_timeout(self) -> None: """Raise TimeoutError if timer has already been cancelled.""" @@ -694,12 +708,17 @@ class TimerContext(BaseTimerContext): def __enter__(self) -> BaseTimerContext: task = asyncio.current_task(loop=self._loop) - if task is None: raise RuntimeError( "Timeout context manager should be used " "inside a task" ) + if sys.version_info >= (3, 11): + # Remember if the task was already cancelling + # so when we __exit__ we can decide if we should + # raise asyncio.TimeoutError or let the cancellation propagate + self._cancelling = task.cancelling() + if self._cancelled: raise asyncio.TimeoutError from None @@ -712,11 +731,22 @@ class TimerContext(BaseTimerContext): exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> Optional[bool]: + enter_task: Optional[asyncio.Task[Any]] = None if self._tasks: - self._tasks.pop() + enter_task = self._tasks.pop() if exc_type is asyncio.CancelledError and self._cancelled: - raise asyncio.TimeoutError from None + assert enter_task is not None + # The timeout was hit, and the task was cancelled + # so we need to uncancel the last task that entered the context manager + # since the cancellation should not leak out of the context manager + if sys.version_info >= (3, 11): + # If the task was already cancelling don't raise + # asyncio.TimeoutError and instead return None + # to allow the cancellation to propagate + if enter_task.uncancel() > self._cancelling: + return None + raise asyncio.TimeoutError from exc_val return None def timeout(self) -> None: @@ -784,11 +814,7 @@ class HeadersMixin: def content_length(self) -> Optional[int]: """The value of Content-Length HTTP header.""" content_length = self._headers.get(hdrs.CONTENT_LENGTH) - - if content_length is not None: - return int(content_length) - else: - return None + return None if content_length is None else int(content_length) def set_result(fut: "asyncio.Future[_T]", result: _T) -> None: diff --git a/contrib/python/aiohttp/aiohttp/http_parser.py b/contrib/python/aiohttp/aiohttp/http_parser.py index 686a2d02e28..9fc7e8a2c8b 100644 --- a/contrib/python/aiohttp/aiohttp/http_parser.py +++ b/contrib/python/aiohttp/aiohttp/http_parser.py @@ -845,6 +845,13 @@ class HttpPayloadParser: i = chunk.find(CHUNK_EXT, 0, pos) if i >= 0: size_b = chunk[:i] # strip chunk-extensions + # Verify no LF in the chunk-extension + if b"\n" in (ext := chunk[i:pos]): + exc = BadHttpMessage( + f"Unexpected LF in chunk-extension: {ext!r}" + ) + set_exception(self.payload, exc) + raise exc else: size_b = chunk[:pos] diff --git a/contrib/python/aiohttp/aiohttp/http_websocket.py b/contrib/python/aiohttp/aiohttp/http_websocket.py index fb00ebc7d35..0916530dbe8 100644 --- a/contrib/python/aiohttp/aiohttp/http_websocket.py +++ b/contrib/python/aiohttp/aiohttp/http_websocket.py @@ -133,8 +133,12 @@ class WSMessage(NamedTuple): return loads(self.data) -WS_CLOSED_MESSAGE = WSMessage(WSMsgType.CLOSED, None, None) -WS_CLOSING_MESSAGE = WSMessage(WSMsgType.CLOSING, None, None) +# Constructing the tuple directly to avoid the overhead of +# the lambda and arg processing since NamedTuples are constructed +# with a run time built lambda +# https://github.com/python/cpython/blob/d83fcf8371f2f33c7797bc8f5423a8bca8c46e5c/Lib/collections/__init__.py#L441 +WS_CLOSED_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSED, None, None)) +WS_CLOSING_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSING, None, None)) class WebSocketError(Exception): @@ -411,12 +415,14 @@ class WebSocketReader: WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" ) from exc - self.queue.feed_data(WSMessage(WSMsgType.TEXT, text, ""), len(text)) + # tuple.__new__ is used to avoid the overhead of the lambda + msg = tuple.__new__(WSMessage, (WSMsgType.TEXT, text, "")) + self.queue.feed_data(msg, len(payload_merged)) continue - self.queue.feed_data( - WSMessage(WSMsgType.BINARY, payload_merged, ""), len(payload_merged) - ) + # tuple.__new__ is used to avoid the overhead of the lambda + msg = tuple.__new__(WSMessage, (WSMsgType.BINARY, payload_merged, "")) + self.queue.feed_data(msg, len(payload_merged)) elif opcode == WSMsgType.CLOSE: if len(payload) >= 2: close_code = UNPACK_CLOSE_CODE(payload[:2])[0] @@ -431,26 +437,26 @@ class WebSocketReader: raise WebSocketError( WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" ) from exc - msg = WSMessage(WSMsgType.CLOSE, close_code, close_message) + msg = tuple.__new__( + WSMessage, (WSMsgType.CLOSE, close_code, close_message) + ) elif payload: raise WebSocketError( WSCloseCode.PROTOCOL_ERROR, f"Invalid close frame: {fin} {opcode} {payload!r}", ) else: - msg = WSMessage(WSMsgType.CLOSE, 0, "") + msg = tuple.__new__(WSMessage, (WSMsgType.CLOSE, 0, "")) self.queue.feed_data(msg, 0) elif opcode == WSMsgType.PING: - self.queue.feed_data( - WSMessage(WSMsgType.PING, payload, ""), len(payload) - ) + msg = tuple.__new__(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) elif opcode == WSMsgType.PONG: - self.queue.feed_data( - WSMessage(WSMsgType.PONG, payload, ""), len(payload) - ) + msg = tuple.__new__(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) else: raise WebSocketError( @@ -669,13 +675,16 @@ class WebSocketWriter: if msg_length < 126: header = PACK_LEN1(first_byte, msg_length | mask_bit) header_len = 2 - elif msg_length < (1 << 16): + elif msg_length < 65536: header = PACK_LEN2(first_byte, 126 | mask_bit, msg_length) header_len = 4 else: header = PACK_LEN3(first_byte, 127 | mask_bit, msg_length) header_len = 10 + if self.transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.3 # If we are using a mask, we need to generate it randomly # and apply it to the message before sending it. A mask is @@ -687,17 +696,15 @@ class WebSocketWriter: mask = PACK_RANDBITS(self.get_random_bits()) message = bytearray(message) _websocket_mask(mask, message) - self._write(header + mask + message) - self._output_size += header_len + MASK_LEN + msg_length - + self.transport.write(header + mask + message) + self._output_size += MASK_LEN + elif msg_length > MSG_SIZE: + self.transport.write(header) + self.transport.write(message) else: - if msg_length > MSG_SIZE: - self._write(header) - self._write(message) - else: - self._write(header + message) + self.transport.write(header + message) - self._output_size += header_len + msg_length + self._output_size += header_len + msg_length # It is safe to return control to the event loop when using compression # after this point as we have already sent or buffered all the data. @@ -718,11 +725,6 @@ class WebSocketWriter: max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, ) - def _write(self, data: bytes) -> None: - if self.transport is None or self.transport.is_closing(): - raise ClientConnectionResetError("Cannot write to closing transport") - self.transport.write(data) - async def pong(self, message: Union[bytes, str] = b"") -> None: """Send pong message.""" if isinstance(message, str): diff --git a/contrib/python/aiohttp/aiohttp/locks.py b/contrib/python/aiohttp/aiohttp/locks.py deleted file mode 100644 index de2dc83d09d..00000000000 --- a/contrib/python/aiohttp/aiohttp/locks.py +++ /dev/null @@ -1,41 +0,0 @@ -import asyncio -import collections -from typing import Any, Deque, Optional - - -class EventResultOrError: - """Event asyncio lock helper class. - - Wraps the Event asyncio lock allowing either to awake the - locked Tasks without any error or raising an exception. - - thanks to @vorpalsmith for the simple design. - """ - - def __init__(self, loop: asyncio.AbstractEventLoop) -> None: - self._loop = loop - self._exc: Optional[BaseException] = None - self._event = asyncio.Event() - self._waiters: Deque[asyncio.Future[Any]] = collections.deque() - - def set(self, exc: Optional[BaseException] = None) -> None: - self._exc = exc - self._event.set() - - async def wait(self) -> Any: - waiter = self._loop.create_task(self._event.wait()) - self._waiters.append(waiter) - try: - val = await waiter - finally: - self._waiters.remove(waiter) - - if self._exc is not None: - raise self._exc - - return val - - def cancel(self) -> None: - """Cancel all waiters""" - for waiter in self._waiters: - waiter.cancel() diff --git a/contrib/python/aiohttp/aiohttp/payload.py b/contrib/python/aiohttp/aiohttp/payload.py index 27636977774..151f9dd497b 100644 --- a/contrib/python/aiohttp/aiohttp/payload.py +++ b/contrib/python/aiohttp/aiohttp/payload.py @@ -101,6 +101,7 @@ class PayloadRegistry: self._first: List[_PayloadRegistryItem] = [] self._normal: List[_PayloadRegistryItem] = [] self._last: List[_PayloadRegistryItem] = [] + self._normal_lookup: Dict[Any, PayloadType] = {} def get( self, @@ -109,12 +110,20 @@ class PayloadRegistry: _CHAIN: "Type[chain[_PayloadRegistryItem]]" = chain, **kwargs: Any, ) -> "Payload": + if self._first: + for factory, type_ in self._first: + if isinstance(data, type_): + return factory(data, *args, **kwargs) + # Try the fast lookup first + if lookup_factory := self._normal_lookup.get(type(data)): + return lookup_factory(data, *args, **kwargs) + # Bail early if its already a Payload if isinstance(data, Payload): return data - for factory, type in _CHAIN(self._first, self._normal, self._last): - if isinstance(data, type): + # Fallback to the slower linear search + for factory, type_ in _CHAIN(self._normal, self._last): + if isinstance(data, type_): return factory(data, *args, **kwargs) - raise LookupError() def register( @@ -124,6 +133,11 @@ class PayloadRegistry: self._first.append((factory, type)) elif order is Order.normal: self._normal.append((factory, type)) + if isinstance(type, Iterable): + for t in type: + self._normal_lookup[t] = factory + else: + self._normal_lookup[type] = factory elif order is Order.try_last: self._last.append((factory, type)) else: @@ -159,7 +173,8 @@ class Payload(ABC): self._headers[hdrs.CONTENT_TYPE] = content_type else: self._headers[hdrs.CONTENT_TYPE] = self._default_content_type - self._headers.update(headers or {}) + if headers: + self._headers.update(headers) @property def size(self) -> Optional[int]: @@ -228,9 +243,6 @@ class BytesPayload(Payload): def __init__( self, value: Union[bytes, bytearray, memoryview], *args: Any, **kwargs: Any ) -> None: - if not isinstance(value, (bytes, bytearray, memoryview)): - raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") - if "content_type" not in kwargs: kwargs["content_type"] = "application/octet-stream" @@ -238,8 +250,10 @@ class BytesPayload(Payload): if isinstance(value, memoryview): self._size = value.nbytes - else: + elif isinstance(value, (bytes, bytearray)): self._size = len(value) + else: + raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") if self._size > TOO_LARGE_BYTES_BODY: kwargs = {"source": self} diff --git a/contrib/python/aiohttp/aiohttp/resolver.py b/contrib/python/aiohttp/aiohttp/resolver.py index 06855fa13fd..5bcdb8ada43 100644 --- a/contrib/python/aiohttp/aiohttp/resolver.py +++ b/contrib/python/aiohttp/aiohttp/resolver.py @@ -111,7 +111,7 @@ class AsyncResolver(AbstractResolver): ) except aiodns.error.DNSError as exc: msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" - raise OSError(msg) from exc + raise OSError(None, msg) from exc hosts: List[ResolveResult] = [] for node in resp.nodes: address: Union[Tuple[bytes, int], Tuple[bytes, int, int, int]] = node.addr @@ -145,7 +145,7 @@ class AsyncResolver(AbstractResolver): ) if not hosts: - raise OSError("DNS lookup failed") + raise OSError(None, "DNS lookup failed") return hosts @@ -161,7 +161,7 @@ class AsyncResolver(AbstractResolver): resp = await self._resolver.query(host, qtype) except aiodns.error.DNSError as exc: msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" - raise OSError(msg) from exc + raise OSError(None, msg) from exc hosts = [] for rr in resp: @@ -177,7 +177,7 @@ class AsyncResolver(AbstractResolver): ) if not hosts: - raise OSError("DNS lookup failed") + raise OSError(None, "DNS lookup failed") return hosts diff --git a/contrib/python/aiohttp/aiohttp/streams.py b/contrib/python/aiohttp/aiohttp/streams.py index c927cfbb1b3..030c6735947 100644 --- a/contrib/python/aiohttp/aiohttp/streams.py +++ b/contrib/python/aiohttp/aiohttp/streams.py @@ -245,9 +245,10 @@ class StreamReader(AsyncStreamReaderMixin): if not data: return - self._size += len(data) + data_len = len(data) + self._size += data_len self._buffer.append(data) - self.total_bytes += len(data) + self.total_bytes += data_len waiter = self._waiter if waiter is not None: diff --git a/contrib/python/aiohttp/aiohttp/test_utils.py b/contrib/python/aiohttp/aiohttp/test_utils.py index 01496b6711a..850efcb8b65 100644 --- a/contrib/python/aiohttp/aiohttp/test_utils.py +++ b/contrib/python/aiohttp/aiohttp/test_utils.py @@ -148,7 +148,7 @@ class BaseTestServer(ABC): assert self._root is not None url = URL(path) if not self.skip_url_asserts: - assert not url.is_absolute() + assert not url.absolute return self._root.join(url) else: return URL(str(self._root) + str(path)) diff --git a/contrib/python/aiohttp/aiohttp/tracing.py b/contrib/python/aiohttp/aiohttp/tracing.py index a42fc0d2a44..067a132464e 100644 --- a/contrib/python/aiohttp/aiohttp/tracing.py +++ b/contrib/python/aiohttp/aiohttp/tracing.py @@ -1,5 +1,5 @@ from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Awaitable, Mapping, Optional, Protocol, Type, TypeVar +from typing import TYPE_CHECKING, Awaitable, Mapping, Optional, Protocol, Type, TypeVar import attr from aiosignal import Signal @@ -64,7 +64,7 @@ class TraceConfig: self._trace_config_ctx_factory = trace_config_ctx_factory def trace_config_ctx( - self, trace_request_ctx: Optional[Mapping[str, Any]] = None + self, trace_request_ctx: Optional[Mapping[str, str]] = None ) -> SimpleNamespace: """Return a new trace_config_ctx instance""" return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) diff --git a/contrib/python/aiohttp/aiohttp/typedefs.py b/contrib/python/aiohttp/aiohttp/typedefs.py index 2e285fa2561..cc8c0825b4e 100644 --- a/contrib/python/aiohttp/aiohttp/typedefs.py +++ b/contrib/python/aiohttp/aiohttp/typedefs.py @@ -8,23 +8,12 @@ from typing import ( Iterable, Mapping, Protocol, - Sequence, Tuple, Union, ) from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr -from yarl import URL - -try: - # Available in yarl>=1.10.0 - from yarl import Query as _Query -except ImportError: # pragma: no cover - SimpleQuery = Union[str, int, float] # pragma: no cover - QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] # pragma: no cover - _Query = Union[ # type: ignore[misc] # pragma: no cover - None, str, "Mapping[str, QueryVariable]", "Sequence[Tuple[str, QueryVariable]]" - ] +from yarl import URL, Query as _Query Query = _Query diff --git a/contrib/python/aiohttp/aiohttp/web_app.py b/contrib/python/aiohttp/aiohttp/web_app.py index ab11981a8a5..628e098d2ea 100644 --- a/contrib/python/aiohttp/aiohttp/web_app.py +++ b/contrib/python/aiohttp/aiohttp/web_app.py @@ -54,6 +54,7 @@ from .web_urldispatcher import ( MaskDomain, MatchedSubAppResource, PrefixedSubAppResource, + SystemRoute, UrlDispatcher, ) @@ -79,7 +80,6 @@ _U = TypeVar("_U") _Resource = TypeVar("_Resource", bound=AbstractResource) -@lru_cache(None) def _build_middlewares( handler: Handler, apps: Tuple["Application", ...] ) -> Callable[[Request], Awaitable[StreamResponse]]: @@ -90,6 +90,9 @@ def _build_middlewares( return handler +_cached_build_middleware = lru_cache(maxsize=1024)(_build_middlewares) + + class Application(MutableMapping[Union[str, AppKey[Any]], Any]): ATTRS = frozenset( [ @@ -544,8 +547,13 @@ class Application(MutableMapping[Union[str, AppKey[Any]], Any]): handler = match_info.handler if self._run_middlewares: - if not self._has_legacy_middlewares: - handler = _build_middlewares(handler, match_info.apps) + # If its a SystemRoute, don't cache building the middlewares since + # they are constructed for every MatchInfoError as a new handler + # is made each time. + if not self._has_legacy_middlewares and not isinstance( + match_info.route, SystemRoute + ): + handler = _cached_build_middleware(handler, match_info.apps) else: for app in match_info.apps[::-1]: for m, new_style in app._middlewares_handlers: # type: ignore[union-attr] diff --git a/contrib/python/aiohttp/aiohttp/web_request.py b/contrib/python/aiohttp/aiohttp/web_request.py index eca71e4413a..62a08ea248b 100644 --- a/contrib/python/aiohttp/aiohttp/web_request.py +++ b/contrib/python/aiohttp/aiohttp/web_request.py @@ -174,7 +174,7 @@ class BaseRequest(MutableMapping[str, Any], HeadersMixin): self._version = message.version self._cache: Dict[str, Any] = {} url = message.url - if url.is_absolute(): + if url.absolute: if scheme is not None: url = url.with_scheme(scheme) if host is not None: @@ -431,6 +431,10 @@ class BaseRequest(MutableMapping[str, Any], HeadersMixin): - overridden value by .clone(host=new_host) call. - HOST HTTP header - socket.getfqdn() value + + For example, 'example.com' or 'localhost:8080'. + + For historical reasons, the port number may be included. """ host = self._message.headers.get(hdrs.HOST) if host is not None: @@ -454,8 +458,10 @@ class BaseRequest(MutableMapping[str, Any], HeadersMixin): @reify def url(self) -> URL: - url = URL.build(scheme=self.scheme, host=self.host) - return url.join(self._rel_url) + """The full URL of the request.""" + # authority is used here because it may include the port number + # and we want yarl to parse it correctly + return URL.build(scheme=self.scheme, authority=self.host).join(self._rel_url) @reify def path(self) -> str: diff --git a/contrib/python/aiohttp/aiohttp/web_response.py b/contrib/python/aiohttp/aiohttp/web_response.py index 956359718da..2db96222709 100644 --- a/contrib/python/aiohttp/aiohttp/web_response.py +++ b/contrib/python/aiohttp/aiohttp/web_response.py @@ -42,6 +42,7 @@ from .payload import Payload from .typedefs import JSONEncoder, LooseHeaders REASON_PHRASES = {http_status.value: http_status.phrase for http_status in HTTPStatus} +LARGE_BODY_SIZE = 1024**2 __all__ = ("ContentCoding", "StreamResponse", "Response", "json_response") @@ -395,25 +396,26 @@ class StreamResponse(BaseClass, HeadersMixin): self._headers[CONTENT_TYPE] = ctype async def _do_start_compression(self, coding: ContentCoding) -> None: - if coding != ContentCoding.identity: - assert self._payload_writer is not None - self._headers[hdrs.CONTENT_ENCODING] = coding.value - self._payload_writer.enable_compression(coding.value) - # Compressed payload may have different content length, - # remove the header - self._headers.popall(hdrs.CONTENT_LENGTH, None) + if coding is ContentCoding.identity: + return + assert self._payload_writer is not None + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._payload_writer.enable_compression(coding.value) + # Compressed payload may have different content length, + # remove the header + self._headers.popall(hdrs.CONTENT_LENGTH, None) async def _start_compression(self, request: "BaseRequest") -> None: if self._compression_force: await self._do_start_compression(self._compression_force) - else: - # Encoding comparisons should be case-insensitive - # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 - accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() - for value, coding in CONTENT_CODINGS.items(): - if value in accept_encoding: - await self._do_start_compression(coding) - return + return + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + for value, coding in CONTENT_CODINGS.items(): + if value in accept_encoding: + await self._do_start_compression(coding) + return async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]: if self._eof_sent: @@ -446,9 +448,10 @@ class StreamResponse(BaseClass, HeadersMixin): version = request.version headers = self._headers - for cookie in self._cookies.values(): - value = cookie.output(header="")[1:] - headers.add(hdrs.SET_COOKIE, value) + if self._cookies: + for cookie in self._cookies.values(): + value = cookie.output(header="")[1:] + headers.add(hdrs.SET_COOKIE, value) if self._compression: await self._start_compression(request) @@ -464,7 +467,7 @@ class StreamResponse(BaseClass, HeadersMixin): headers[hdrs.TRANSFER_ENCODING] = "chunked" if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] - elif self._length_check: + elif self._length_check: # Disabled for WebSockets writer.length = self.content_length if writer.length is None: if version >= HttpVersion11: @@ -485,7 +488,7 @@ class StreamResponse(BaseClass, HeadersMixin): # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-13 if hdrs.TRANSFER_ENCODING in headers: del headers[hdrs.TRANSFER_ENCODING] - elif self.content_length != 0: + elif (writer.length if self._length_check else self.content_length) != 0: # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") headers.setdefault(hdrs.DATE, rfc822_formatted_time()) @@ -496,9 +499,8 @@ class StreamResponse(BaseClass, HeadersMixin): if keep_alive: if version == HttpVersion10: headers[hdrs.CONNECTION] = "keep-alive" - else: - if version == HttpVersion11: - headers[hdrs.CONNECTION] = "close" + elif version == HttpVersion11: + headers[hdrs.CONNECTION] = "close" async def _write_headers(self) -> None: request = self._req @@ -626,19 +628,17 @@ class Response(StreamResponse): real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset body = text.encode(charset) text = None - else: - if hdrs.CONTENT_TYPE in real_headers: - if content_type is not None or charset is not None: - raise ValueError( - "passing both Content-Type header and " - "content_type or charset params " - "is forbidden" - ) - else: - if content_type is not None: - if charset is not None: - content_type += "; charset=" + charset - real_headers[hdrs.CONTENT_TYPE] = content_type + elif hdrs.CONTENT_TYPE in real_headers: + if content_type is not None or charset is not None: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + elif content_type is not None: + if charset is not None: + content_type += "; charset=" + charset + real_headers[hdrs.CONTENT_TYPE] = content_type super().__init__(status=status, reason=reason, headers=real_headers) @@ -707,7 +707,7 @@ class Response(StreamResponse): return None if hdrs.CONTENT_LENGTH in self._headers: - return super().content_length + return int(self._headers[hdrs.CONTENT_LENGTH]) if self._compressed_body is not None: # Return length of the compressed body @@ -734,16 +734,13 @@ class Response(StreamResponse): assert not data, f"data arg is not supported, got {data!r}" assert self._req is not None assert self._payload_writer is not None - if body is not None: - if self._must_be_empty_body: - await super().write_eof() - elif isinstance(self._body, Payload): - await self._body.write(self._payload_writer) - await super().write_eof() - else: - await super().write_eof(cast(bytes, body)) - else: + if body is None or self._must_be_empty_body: + await super().write_eof() + elif isinstance(self._body, Payload): + await self._body.write(self._payload_writer) await super().write_eof() + else: + await super().write_eof(cast(bytes, body)) async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: if hdrs.CONTENT_LENGTH in self._headers: @@ -766,30 +763,28 @@ class Response(StreamResponse): async def _do_start_compression(self, coding: ContentCoding) -> None: if self._chunked or isinstance(self._body, Payload): return await super()._do_start_compression(coding) - - if coding != ContentCoding.identity: - # Instead of using _payload_writer.enable_compression, - # compress the whole body - compressor = ZLibCompressor( - encoding=str(coding.value), - max_sync_chunk_size=self._zlib_executor_size, - executor=self._zlib_executor, - ) - assert self._body is not None - if self._zlib_executor_size is None and len(self._body) > 1024 * 1024: - warnings.warn( - "Synchronous compression of large response bodies " - f"({len(self._body)} bytes) might block the async event loop. " - "Consider providing a custom value to zlib_executor_size/" - "zlib_executor response properties or disabling compression on it." - ) - self._compressed_body = ( - await compressor.compress(self._body) + compressor.flush() + if coding is ContentCoding.identity: + return + # Instead of using _payload_writer.enable_compression, + # compress the whole body + compressor = ZLibCompressor( + encoding=coding.value, + max_sync_chunk_size=self._zlib_executor_size, + executor=self._zlib_executor, + ) + assert self._body is not None + if self._zlib_executor_size is None and len(self._body) > LARGE_BODY_SIZE: + warnings.warn( + "Synchronous compression of large response bodies " + f"({len(self._body)} bytes) might block the async event loop. " + "Consider providing a custom value to zlib_executor_size/" + "zlib_executor response properties or disabling compression on it." ) - assert self._compressed_body is not None - - self._headers[hdrs.CONTENT_ENCODING] = coding.value - self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body)) + self._compressed_body = ( + await compressor.compress(self._body) + compressor.flush() + ) + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body)) def json_response( diff --git a/contrib/python/aiohttp/aiohttp/web_ws.py b/contrib/python/aiohttp/aiohttp/web_ws.py index 382223097ea..0e8adef6f8d 100644 --- a/contrib/python/aiohttp/aiohttp/web_ws.py +++ b/contrib/python/aiohttp/aiohttp/web_ws.py @@ -379,14 +379,14 @@ class WebSocketResponse(StreamResponse): raise RuntimeError("Call .prepare() first") await self._writer.pong(message) - async def send_str(self, data: str, compress: Optional[bool] = None) -> None: + async def send_str(self, data: str, compress: Optional[int] = None) -> None: if self._writer is None: raise RuntimeError("Call .prepare() first") if not isinstance(data, str): raise TypeError("data argument must be str (%r)" % type(data)) await self._writer.send(data, binary=False, compress=compress) - async def send_bytes(self, data: bytes, compress: Optional[bool] = None) -> None: + async def send_bytes(self, data: bytes, compress: Optional[int] = None) -> None: if self._writer is None: raise RuntimeError("Call .prepare() first") if not isinstance(data, (bytes, bytearray, memoryview)): @@ -396,7 +396,7 @@ class WebSocketResponse(StreamResponse): async def send_json( self, data: Any, - compress: Optional[bool] = None, + compress: Optional[int] = None, *, dumps: JSONEncoder = json.dumps, ) -> None: @@ -453,7 +453,11 @@ class WebSocketResponse(StreamResponse): try: async with async_timeout.timeout(self._timeout): - msg = await reader.read() + while True: + msg = await reader.read() + if msg.type is WSMsgType.CLOSE: + self._set_code_close_transport(msg.data) + return True except asyncio.CancelledError: self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) raise @@ -462,14 +466,6 @@ class WebSocketResponse(StreamResponse): self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) return True - if msg.type is WSMsgType.CLOSE: - self._set_code_close_transport(msg.data) - return True - - self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) - self._exception = asyncio.TimeoutError() - return True - def _set_closing(self, code: WSCloseCode) -> None: """Set the close code and mark the connection as closing.""" self._closing = True @@ -490,8 +486,6 @@ class WebSocketResponse(StreamResponse): if self._reader is None: raise RuntimeError("Call .prepare() first") - loop = self._loop - assert loop is not None receive_timeout = timeout or self._receive_timeout while True: if self._waiting: diff --git a/contrib/python/aiohttp/patches/06-pr9398-fix-trace-request-ctx-typing.patch b/contrib/python/aiohttp/patches/06-pr9398-fix-trace-request-ctx-typing.patch deleted file mode 100644 index 55b5b76f4d6..00000000000 --- a/contrib/python/aiohttp/patches/06-pr9398-fix-trace-request-ctx-typing.patch +++ /dev/null @@ -1,32 +0,0 @@ -# -# back-port pull-req #9398 -# ---- contrib/python/aiohttp/aiohttp/client.py -+++ contrib/python/aiohttp/aiohttp/client.py -@@ -178,7 +178,7 @@ class _RequestOptions(TypedDict, total=False): - ssl: Union[SSLContext, bool, Fingerprint] - server_hostname: Union[str, None] - proxy_headers: Union[LooseHeaders, None] -- trace_request_ctx: Union[Mapping[str, str], None] -+ trace_request_ctx: Union[Mapping[str, Any], None] - read_bufsize: Union[int, None] - auto_decompress: Union[bool, None] - max_line_size: Union[int, None] ---- contrib/python/aiohttp/aiohttp/tracing.py -+++ contrib/python/aiohttp/aiohttp/tracing.py -@@ -1,5 +1,5 @@ - from types import SimpleNamespace --from typing import TYPE_CHECKING, Awaitable, Mapping, Optional, Protocol, Type, TypeVar -+from typing import TYPE_CHECKING, Any, Awaitable, Mapping, Optional, Protocol, Type, TypeVar - - import attr - from aiosignal import Signal -@@ -64,7 +64,7 @@ class TraceConfig: - self._trace_config_ctx_factory = trace_config_ctx_factory - - def trace_config_ctx( -- self, trace_request_ctx: Optional[Mapping[str, str]] = None -+ self, trace_request_ctx: Optional[Mapping[str, Any]] = None - ) -> SimpleNamespace: - """Return a new trace_config_ctx instance""" - return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) diff --git a/contrib/python/aiohttp/patches/99-rep-get-running-loop.sh b/contrib/python/aiohttp/patches/99-rep-get-running-loop.sh index b97011e3c67..8565b28f2cf 100644 --- a/contrib/python/aiohttp/patches/99-rep-get-running-loop.sh +++ b/contrib/python/aiohttp/patches/99-rep-get-running-loop.sh @@ -1,4 +1,10 @@ # This patch may be dropped after python 3.13 upver -find aiohttp -type f -exec sed --in-place 's|loop or asyncio.get_running_loop|loop or asyncio.get_event_loop|g' '{}' ';' +if [ "$(uname)" = "Darwin" ]; then + SED=gsed +else + SED=sed +fi + +find aiohttp -type f -exec ${SED} --in-place 's|loop or asyncio.get_running_loop|loop or asyncio.get_event_loop|g' '{}' ';' diff --git a/contrib/python/aiohttp/ya.make b/contrib/python/aiohttp/ya.make index e714d0fd423..3767e000884 100644 --- a/contrib/python/aiohttp/ya.make +++ b/contrib/python/aiohttp/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.10.6) +VERSION(3.10.11) LICENSE(Apache-2.0) @@ -57,7 +57,6 @@ PY_SRCS( aiohttp/http_parser.py aiohttp/http_websocket.py aiohttp/http_writer.py - aiohttp/locks.py aiohttp/log.py aiohttp/multipart.py aiohttp/payload.py diff --git a/contrib/python/docker/.dist-info/METADATA b/contrib/python/docker/.dist-info/METADATA index 90e41721a6f..d4e1c5135c2 100644 --- a/contrib/python/docker/.dist-info/METADATA +++ b/contrib/python/docker/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.3 +Metadata-Version: 2.4 Name: docker -Version: 7.1.0 +Version: 7.2.0 Summary: A Python library for the Docker Engine API. Project-URL: Changelog, https://docker-py.readthedocs.io/en/stable/change-log.html Project-URL: Documentation, https://docker-py.readthedocs.io diff --git a/contrib/python/docker/docker/__init__.py b/contrib/python/docker/docker/__init__.py index fb7a5e921ad..4e06ac2ac69 100644 --- a/contrib/python/docker/docker/__init__.py +++ b/contrib/python/docker/docker/__init__.py @@ -1,5 +1,5 @@ from .api import APIClient -from .client import DockerClient, from_env +from .client import DockerClient, from_context, from_env from .context import Context, ContextAPI from .tls import TLSConfig from .version import __version__ diff --git a/contrib/python/docker/docker/_version.py b/contrib/python/docker/docker/_version.py index 32913e53d55..a8e096249dd 100644 --- a/contrib/python/docker/docker/_version.py +++ b/contrib/python/docker/docker/_version.py @@ -1,16 +1,24 @@ -# file generated by setuptools_scm +# file generated by vcs-versioning # don't change, don't track in version control -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple, Union - VERSION_TUPLE = Tuple[Union[int, str], ...] -else: - VERSION_TUPLE = object +from __future__ import annotations + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] version: str __version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None + +__version__ = version = '7.2.0' +__version_tuple__ = version_tuple = (7, 2, 0) -__version__ = version = '7.1.0' -__version_tuple__ = version_tuple = (7, 1, 0) +__commit_id__ = commit_id = None diff --git a/contrib/python/docker/docker/client.py b/contrib/python/docker/docker/client.py index 9012d24c9c1..821706387c1 100644 --- a/contrib/python/docker/docker/client.py +++ b/contrib/python/docker/docker/client.py @@ -1,5 +1,8 @@ +import os + from .api.client import APIClient from .constants import DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS +from .context import ContextAPI from .models.configs import ConfigCollection from .models.containers import ContainerCollection from .models.images import ImageCollection @@ -78,6 +81,10 @@ class DockerClient: use_ssh_client (bool): If set to `True`, an ssh connection is made via shelling out to the ssh client. Ensure the ssh client is installed and configured on the host. + use_context (bool): If ``True`` (the default), fall back to the + current Docker CLI context (``~/.docker/config.json`` / + ``DOCKER_CONTEXT``) when ``DOCKER_HOST`` is not set. This + allows the client to talk to Docker Desktop out of the box. Example: @@ -91,12 +98,66 @@ class DockerClient: max_pool_size = kwargs.pop('max_pool_size', DEFAULT_MAX_POOL_SIZE) version = kwargs.pop('version', None) use_ssh_client = kwargs.pop('use_ssh_client', False) + use_context = kwargs.pop('use_context', True) + environment = kwargs.get('environment') or os.environ + + params = kwargs_from_env(**kwargs) + if use_context and 'base_url' not in params: + for k, v in ContextAPI.kwargs_from_context( + environment=environment).items(): + params.setdefault(k, v) + + return cls( + timeout=timeout, + max_pool_size=max_pool_size, + version=version, + use_ssh_client=use_ssh_client, + **params, + ) + + @classmethod + def from_context(cls, name=None, **kwargs): + """ + Return a client configured from a Docker CLI context. + + With no ``name``, resolves the current context the same way the + Docker CLI does: ``DOCKER_CONTEXT`` env var, then the + ``currentContext`` field in ``~/.docker/config.json``, falling back + to the built-in ``default`` context. On a machine with Docker + Desktop installed this typically resolves to ``desktop-linux``. + + Args: + name (str): Name of the context to load. ``None`` (the default) + means "use the current context". + version (str): The version of the API to use. Set to ``auto`` to + automatically detect the server's version. + timeout (int): Default timeout for API calls, in seconds. + max_pool_size (int): The maximum number of connections to save in + the pool. + use_ssh_client (bool): If ``True``, shell out to the ssh client + for ssh:// contexts. + + Example: + + >>> import docker + >>> client = docker.DockerClient.from_context() + >>> # or, pick a specific context: + >>> client = docker.DockerClient.from_context('desktop-linux') + """ + timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS) + max_pool_size = kwargs.pop('max_pool_size', DEFAULT_MAX_POOL_SIZE) + version = kwargs.pop('version', None) + use_ssh_client = kwargs.pop('use_ssh_client', False) + + params = ContextAPI.kwargs_from_context(name=name) + params.update(kwargs) + return cls( timeout=timeout, max_pool_size=max_pool_size, version=version, use_ssh_client=use_ssh_client, - **kwargs_from_env(**kwargs) + **params, ) # Resources @@ -220,3 +281,4 @@ class DockerClient: from_env = DockerClient.from_env +from_context = DockerClient.from_context diff --git a/contrib/python/docker/docker/constants.py b/contrib/python/docker/docker/constants.py index 3c527b47e3a..0e39dc2917c 100644 --- a/contrib/python/docker/docker/constants.py +++ b/contrib/python/docker/docker/constants.py @@ -2,7 +2,7 @@ import sys from .version import __version__ -DEFAULT_DOCKER_API_VERSION = '1.44' +DEFAULT_DOCKER_API_VERSION = '1.45' MINIMUM_DOCKER_API_VERSION = '1.24' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/contrib/python/docker/docker/context/api.py b/contrib/python/docker/docker/context/api.py index 9ac4ff470af..c86f1c74d17 100644 --- a/contrib/python/docker/docker/context/api.py +++ b/contrib/python/docker/docker/context/api.py @@ -133,6 +133,33 @@ class ContextAPI: return cls.get_context() @classmethod + def kwargs_from_context(cls, name=None, environment=None): + """Build ``base_url`` / ``tls`` kwargs from a Docker CLI context. + + Mirrors the Docker CLI: if ``name`` is not given, honours the + ``DOCKER_CONTEXT`` env var, then the ``currentContext`` field in + ``~/.docker/config.json``, defaulting to the built-in ``default`` + context (local socket / named pipe). On a host with Docker Desktop + this resolves to the ``desktop-linux`` (or equivalent) context, so + client construction targets Docker Desktop out of the box. + """ + if environment is None: + environment = os.environ + if name is None: + name = environment.get("DOCKER_CONTEXT") + ctx = cls.get_context(name) + if ctx is None: + return {} + host = ctx.Host + if not host: + return {} + params = {"base_url": host} + tls_cfg = ctx.TLSConfig + if tls_cfg is not None: + params["tls"] = tls_cfg + return params + + @classmethod def set_current_context(cls, name="default"): ctx = cls.get_context(name) if not ctx: diff --git a/contrib/python/docker/docker/models/containers.py b/contrib/python/docker/docker/models/containers.py index 4795523a158..9c9e92c90fd 100644 --- a/contrib/python/docker/docker/models/containers.py +++ b/contrib/python/docker/docker/models/containers.py @@ -181,7 +181,8 @@ class Container(Model): user (str): User to execute command as. Default: root detach (bool): If true, detach from the exec command. Default: False - stream (bool): Stream response data. Default: False + stream (bool): Stream response data. Ignored if ``detach`` is true. + Default: False socket (bool): Return the connection socket to allow custom read/write operations. Default: False environment (dict or list): A dictionary or a list of strings in diff --git a/contrib/python/docker/docker/models/images.py b/contrib/python/docker/docker/models/images.py index 4f058d24d93..0e8cce3f822 100644 --- a/contrib/python/docker/docker/models/images.py +++ b/contrib/python/docker/docker/models/images.py @@ -407,8 +407,8 @@ class ImageCollection(Collection): if match: image_id = match.group(2) images.append(image_id) - if 'error' in chunk: - raise ImageLoadError(chunk['error']) + if 'errorDetail' in chunk: + raise ImageLoadError(chunk['errorDetail']['message']) return [self.get(i) for i in images] diff --git a/contrib/python/docker/docker/types/services.py b/contrib/python/docker/docker/types/services.py index 821115411c6..69c0c498ea4 100644 --- a/contrib/python/docker/docker/types/services.py +++ b/contrib/python/docker/docker/types/services.py @@ -242,6 +242,7 @@ class Mount(dict): for the ``volume`` type. driver_config (DriverConfig): Volume driver configuration. Only valid for the ``volume`` type. + subpath (str): Path inside a volume to mount instead of the volume root. tmpfs_size (int or string): The size for the tmpfs mount in bytes. tmpfs_mode (int): The permission mode for the tmpfs mount. """ @@ -249,7 +250,7 @@ class Mount(dict): def __init__(self, target, source, type='volume', read_only=False, consistency=None, propagation=None, no_copy=False, labels=None, driver_config=None, tmpfs_size=None, - tmpfs_mode=None): + tmpfs_mode=None, subpath=None): self['Target'] = target self['Source'] = source if type not in ('bind', 'volume', 'tmpfs', 'npipe'): @@ -267,7 +268,7 @@ class Mount(dict): self['BindOptions'] = { 'Propagation': propagation } - if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode]): + if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode, subpath]): raise errors.InvalidArgument( 'Incompatible options have been provided for the bind ' 'type mount.' @@ -280,6 +281,8 @@ class Mount(dict): volume_opts['Labels'] = labels if driver_config: volume_opts['DriverConfig'] = driver_config + if subpath: + volume_opts['Subpath'] = subpath if volume_opts: self['VolumeOptions'] = volume_opts if any([propagation, tmpfs_size, tmpfs_mode]): diff --git a/contrib/python/docker/ya.make b/contrib/python/docker/ya.make index 5dbd794e305..bf30b2c36b0 100644 --- a/contrib/python/docker/ya.make +++ b/contrib/python/docker/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.1.0) +VERSION(7.2.0) LICENSE(Apache-2.0) diff --git a/contrib/python/pyOpenSSL/py3/.dist-info/METADATA b/contrib/python/pyOpenSSL/py3/.dist-info/METADATA index e4139673e38..5691fc9e4ee 100644 --- a/contrib/python/pyOpenSSL/py3/.dist-info/METADATA +++ b/contrib/python/pyOpenSSL/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pyOpenSSL -Version: 23.0.0 +Version: 23.3.0 Summary: Python wrapper module around the OpenSSL library Home-page: https://pyopenssl.org/ Author: The pyOpenSSL developers @@ -14,27 +14,27 @@ Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Security :: Cryptography Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: System :: Networking -Requires-Python: >=3.6 +Requires-Python: >=3.7 License-File: LICENSE -Requires-Dist: cryptography (<40,>=38.0.0) +Requires-Dist: cryptography <42,>=41.0.5 Provides-Extra: docs -Requires-Dist: sphinx (!=5.2.0,!=5.2.0.post0) ; extra == 'docs' +Requires-Dist: sphinx !=5.2.0,!=5.2.0.post0,!=7.2.5 ; extra == 'docs' Requires-Dist: sphinx-rtd-theme ; extra == 'docs' Provides-Extra: test Requires-Dist: flaky ; extra == 'test' Requires-Dist: pretend ; extra == 'test' -Requires-Dist: pytest (>=3.0.1) ; extra == 'test' +Requires-Dist: pytest >=3.0.1 ; extra == 'test' ======================================================== pyOpenSSL -- A Python wrapper around the OpenSSL library @@ -87,6 +87,89 @@ You can also join ``#pyca`` on ``irc.libera.chat`` to ask questions or get invol Release Information =================== +23.3.0 (2023-10-25) +------------------- + +Backward-incompatible changes: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Dropped support for Python 3.6. +- The minimum ``cryptography`` version is now 41.0.5. +- Removed ``OpenSSL.crypto.loads_pkcs7`` and ``OpenSSL.crypto.loads_pkcs12`` which had been deprecated for 3 years. +- Added ``OpenSSL.SSL.OP_LEGACY_SERVER_CONNECT`` to allow legacy insecure renegotiation between OpenSSL and unpatched servers. + `#1234 <https://github.com/pyca/pyopenssl/pull/1234>`_. + +Deprecations: +^^^^^^^^^^^^^ + +- Deprecated ``OpenSSL.crypto.PKCS12`` (which was intended to have been deprecated at the same time as ``OpenSSL.crypto.load_pkcs12``). +- Deprecated ``OpenSSL.crypto.NetscapeSPKI``. +- Deprecated ``OpenSSL.crypto.CRL`` +- Deprecated ``OpenSSL.crypto.Revoked`` +- Deprecated ``OpenSSL.crypto.load_crl`` and ``OpenSSL.crypto.dump_crl`` +- Deprecated ``OpenSSL.crypto.sign`` and ``OpenSSL.crypto.verify`` +- Deprecated ``OpenSSL.crypto.X509Extension`` + +Changes: +^^^^^^^^ + +- Changed ``OpenSSL.crypto.X509Store.add_crl`` to also accept + ``cryptography``'s ``x509.CertificateRevocationList`` arguments in addition + to the now deprecated ``OpenSSL.crypto.CRL`` arguments. +- Fixed ``test_set_default_verify_paths`` test so that it is skipped if no + network connection is available. + +23.2.0 (2023-05-30) +------------------- + +Backward-incompatible changes: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Removed ``X509StoreFlags.NOTIFY_POLICY``. + `#1213 <https://github.com/pyca/pyopenssl/pull/1213>`_. + +Deprecations: +^^^^^^^^^^^^^ + +Changes: +^^^^^^^^ + +- ``cryptography`` maximum version has been increased to 41.0.x. +- Invalid versions are now rejected in ``OpenSSL.crypto.X509Req.set_version``. +- Added ``X509VerificationCodes`` to ``OpenSSL.SSL``. + `#1202 <https://github.com/pyca/pyopenssl/pull/1202>`_. + +23.1.1 (2023-03-28) +------------------- + +Backward-incompatible changes: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Changes: +^^^^^^^^ + +- Worked around an issue in OpenSSL 3.1.0 which caused `X509Extension.get_short_name` to raise an exception when no short name was known to OpenSSL. + `#1204 <https://github.com/pyca/pyopenssl/pull/1204>`_. + +23.1.0 (2023-03-24) +------------------- + +Backward-incompatible changes: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Changes: +^^^^^^^^ + +- ``cryptography`` maximum version has been increased to 40.0.x. +- Add ``OpenSSL.SSL.Connection.DTLSv1_get_timeout`` and ``OpenSSL.SSL.Connection.DTLSv1_handle_timeout`` + to support DTLS timeouts `#1180 <https://github.com/pyca/pyopenssl/pull/1180>`_. + 23.0.0 (2023-01-01) ------------------- @@ -111,7 +194,7 @@ Backward-incompatible changes: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Remove support for SSLv2 and SSLv3. -- The minimum ``cryptography`` version is now 38.0.x (and we now pin releases +- The minimum ``cryptography`` version is now 38.0.x (and we now pin releases against ``cryptography`` major versions to prevent future breakage) - The ``OpenSSL.crypto.X509StoreContextError`` exception has been refactored, changing its internal attributes. diff --git a/contrib/python/pyOpenSSL/py3/OpenSSL/SSL.py b/contrib/python/pyOpenSSL/py3/OpenSSL/SSL.py index dfbd1094e9d..ae07442f5d1 100644 --- a/contrib/python/pyOpenSSL/py3/OpenSSL/SSL.py +++ b/contrib/python/pyOpenSSL/py3/OpenSSL/SSL.py @@ -1,5 +1,6 @@ import os import socket +import typing from errno import errorcode from functools import partial, wraps from itertools import chain, count @@ -8,18 +9,32 @@ from weakref import WeakValueDictionary from OpenSSL._util import ( UNSPECIFIED as _UNSPECIFIED, +) +from OpenSSL._util import ( exception_from_error_queue as _exception_from_error_queue, +) +from OpenSSL._util import ( ffi as _ffi, +) +from OpenSSL._util import ( lib as _lib, +) +from OpenSSL._util import ( make_assert as _make_assert, +) +from OpenSSL._util import ( no_zero_allocator as _no_zero_allocator, +) +from OpenSSL._util import ( path_bytes as _path_bytes, +) +from OpenSSL._util import ( text_to_bytes_and_warn as _text_to_bytes_and_warn, ) from OpenSSL.crypto import ( FILETYPE_PEM, - PKey, X509, + PKey, X509Name, X509Store, _PassphraseHelper, @@ -123,6 +138,7 @@ __all__ = [ "Session", "Context", "Connection", + "X509VerificationCodes", ] @@ -216,6 +232,12 @@ try: except AttributeError: pass +try: + OP_LEGACY_SERVER_CONNECT = _lib.SSL_OP_LEGACY_SERVER_CONNECT + __all__.append("OP_LEGACY_SERVER_CONNECT") +except AttributeError: + pass + OP_ALL = _lib.SSL_OP_ALL VERIFY_PEER = _lib.SSL_VERIFY_PEER @@ -250,6 +272,113 @@ SSL_CB_CONNECT_EXIT = _lib.SSL_CB_CONNECT_EXIT SSL_CB_HANDSHAKE_START = _lib.SSL_CB_HANDSHAKE_START SSL_CB_HANDSHAKE_DONE = _lib.SSL_CB_HANDSHAKE_DONE + +class X509VerificationCodes: + """ + Success and error codes for X509 verification, as returned by the + underlying ``X509_STORE_CTX_get_error()`` function and passed by pyOpenSSL + to verification callback functions. + + See `OpenSSL Verification Errors + <https://www.openssl.org/docs/manmaster/man3/X509_verify_cert_error_string.html#ERROR-CODES>`_ + for details. + """ + + OK = _lib.X509_V_OK + ERR_UNABLE_TO_GET_ISSUER_CERT = _lib.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT + ERR_UNABLE_TO_GET_CRL = _lib.X509_V_ERR_UNABLE_TO_GET_CRL + ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE = ( + _lib.X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE + ) + ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE = ( + _lib.X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE + ) + ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY = ( + _lib.X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY + ) + ERR_CERT_SIGNATURE_FAILURE = _lib.X509_V_ERR_CERT_SIGNATURE_FAILURE + ERR_CRL_SIGNATURE_FAILURE = _lib.X509_V_ERR_CRL_SIGNATURE_FAILURE + ERR_CERT_NOT_YET_VALID = _lib.X509_V_ERR_CERT_NOT_YET_VALID + ERR_CERT_HAS_EXPIRED = _lib.X509_V_ERR_CERT_HAS_EXPIRED + ERR_CRL_NOT_YET_VALID = _lib.X509_V_ERR_CRL_NOT_YET_VALID + ERR_CRL_HAS_EXPIRED = _lib.X509_V_ERR_CRL_HAS_EXPIRED + ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD = ( + _lib.X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD + ) + ERR_ERROR_IN_CERT_NOT_AFTER_FIELD = ( + _lib.X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD + ) + ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD = ( + _lib.X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD + ) + ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD = ( + _lib.X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD + ) + ERR_OUT_OF_MEM = _lib.X509_V_ERR_OUT_OF_MEM + ERR_DEPTH_ZERO_SELF_SIGNED_CERT = ( + _lib.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT + ) + ERR_SELF_SIGNED_CERT_IN_CHAIN = _lib.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN + ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY = ( + _lib.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY + ) + ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE = ( + _lib.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE + ) + ERR_CERT_CHAIN_TOO_LONG = _lib.X509_V_ERR_CERT_CHAIN_TOO_LONG + ERR_CERT_REVOKED = _lib.X509_V_ERR_CERT_REVOKED + ERR_INVALID_CA = _lib.X509_V_ERR_INVALID_CA + ERR_PATH_LENGTH_EXCEEDED = _lib.X509_V_ERR_PATH_LENGTH_EXCEEDED + ERR_INVALID_PURPOSE = _lib.X509_V_ERR_INVALID_PURPOSE + ERR_CERT_UNTRUSTED = _lib.X509_V_ERR_CERT_UNTRUSTED + ERR_CERT_REJECTED = _lib.X509_V_ERR_CERT_REJECTED + ERR_SUBJECT_ISSUER_MISMATCH = _lib.X509_V_ERR_SUBJECT_ISSUER_MISMATCH + ERR_AKID_SKID_MISMATCH = _lib.X509_V_ERR_AKID_SKID_MISMATCH + ERR_AKID_ISSUER_SERIAL_MISMATCH = ( + _lib.X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH + ) + ERR_KEYUSAGE_NO_CERTSIGN = _lib.X509_V_ERR_KEYUSAGE_NO_CERTSIGN + ERR_UNABLE_TO_GET_CRL_ISSUER = _lib.X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER + ERR_UNHANDLED_CRITICAL_EXTENSION = ( + _lib.X509_V_ERR_UNHANDLED_CRITICAL_EXTENSION + ) + ERR_KEYUSAGE_NO_CRL_SIGN = _lib.X509_V_ERR_KEYUSAGE_NO_CRL_SIGN + ERR_UNHANDLED_CRITICAL_CRL_EXTENSION = ( + _lib.X509_V_ERR_UNHANDLED_CRITICAL_CRL_EXTENSION + ) + ERR_INVALID_NON_CA = _lib.X509_V_ERR_INVALID_NON_CA + ERR_PROXY_PATH_LENGTH_EXCEEDED = _lib.X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED + ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE = ( + _lib.X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE + ) + ERR_PROXY_CERTIFICATES_NOT_ALLOWED = ( + _lib.X509_V_ERR_PROXY_CERTIFICATES_NOT_ALLOWED + ) + ERR_INVALID_EXTENSION = _lib.X509_V_ERR_INVALID_EXTENSION + ERR_INVALID_POLICY_EXTENSION = _lib.X509_V_ERR_INVALID_POLICY_EXTENSION + ERR_NO_EXPLICIT_POLICY = _lib.X509_V_ERR_NO_EXPLICIT_POLICY + ERR_DIFFERENT_CRL_SCOPE = _lib.X509_V_ERR_DIFFERENT_CRL_SCOPE + ERR_UNSUPPORTED_EXTENSION_FEATURE = ( + _lib.X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE + ) + ERR_UNNESTED_RESOURCE = _lib.X509_V_ERR_UNNESTED_RESOURCE + ERR_PERMITTED_VIOLATION = _lib.X509_V_ERR_PERMITTED_VIOLATION + ERR_EXCLUDED_VIOLATION = _lib.X509_V_ERR_EXCLUDED_VIOLATION + ERR_SUBTREE_MINMAX = _lib.X509_V_ERR_SUBTREE_MINMAX + ERR_UNSUPPORTED_CONSTRAINT_TYPE = ( + _lib.X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE + ) + ERR_UNSUPPORTED_CONSTRAINT_SYNTAX = ( + _lib.X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX + ) + ERR_UNSUPPORTED_NAME_SYNTAX = _lib.X509_V_ERR_UNSUPPORTED_NAME_SYNTAX + ERR_CRL_PATH_VALIDATION_ERROR = _lib.X509_V_ERR_CRL_PATH_VALIDATION_ERROR + ERR_HOSTNAME_MISMATCH = _lib.X509_V_ERR_HOSTNAME_MISMATCH + ERR_EMAIL_MISMATCH = _lib.X509_V_ERR_EMAIL_MISMATCH + ERR_IP_ADDRESS_MISMATCH = _lib.X509_V_ERR_IP_ADDRESS_MISMATCH + ERR_APPLICATION_VERIFICATION = _lib.X509_V_ERR_APPLICATION_VERIFICATION + + # Taken from https://golang.org/src/crypto/x509/root_linux.go _CERTIFICATE_FILE_LOCATIONS = [ "/etc/ssl/certs/ca-certificates.crt", # Debian/Ubuntu/Gentoo etc. @@ -689,7 +818,7 @@ class Context: not be used. """ - _methods = { + _methods: typing.ClassVar[typing.Dict] = { SSLv23_METHOD: (_lib.TLS_method, None), TLSv1_METHOD: (_lib.TLS_method, TLS1_VERSION), TLSv1_1_METHOD: (_lib.TLS_method, TLS1_1_VERSION), @@ -847,9 +976,9 @@ class Context: the ``[email protected]`` `Homebrew <https://brew.sh>`_ formula installed in the default location. * Windows will not work. - * manylinux1 cryptography wheels will work on most common Linux + * manylinux cryptography wheels will work on most common Linux distributions in pyOpenSSL 17.1.0 and above. pyOpenSSL detects the - manylinux1 wheel and attempts to load roots via a fallback path. + manylinux wheel and attempts to load roots via a fallback path. :return: None """ @@ -874,13 +1003,13 @@ class Context: default_dir = _ffi.string(_lib.X509_get_default_cert_dir()) default_file = _ffi.string(_lib.X509_get_default_cert_file()) # Now we check to see if the default_dir and default_file are set - # to the exact values we use in our manylinux1 builds. If they are + # to the exact values we use in our manylinux builds. If they are # then we know to load the fallbacks if ( default_dir == _CRYPTOGRAPHY_MANYLINUX_CA_DIR and default_file == _CRYPTOGRAPHY_MANYLINUX_CA_FILE ): - # This is manylinux1, let's load our fallback paths + # This is manylinux, let's load our fallback paths self._fallback_default_verify_paths( _CERTIFICATE_FILE_LOCATIONS, _CERTIFICATE_PATH_LOCATIONS ) @@ -899,7 +1028,7 @@ class Context: def _fallback_default_verify_paths(self, file_path, dir_path): """ Default verify paths are based on the compiled version of OpenSSL. - However, when pyca/cryptography is compiled as a manylinux1 wheel + However, when pyca/cryptography is compiled as a manylinux wheel that compiled location can potentially be wrong. So, like Go, we will try a predefined set of paths and attempt to load roots from there. @@ -1259,8 +1388,8 @@ class Context: for ca_name in certificate_authorities: if not isinstance(ca_name, X509Name): raise TypeError( - "client CAs must be X509Name objects, not %s " - "objects" % (type(ca_name).__name__,) + "client CAs must be X509Name objects, not {} " + "objects".format(type(ca_name).__name__) ) copy = _lib.X509_NAME_dup(ca_name._name) _openssl_assert(copy != _ffi.NULL) @@ -1663,8 +1792,7 @@ class Connection: """ if self._socket is None: raise AttributeError( - "'%s' object has no attribute '%s'" - % (self.__class__.__name__, name) + f"'{self.__class__.__name__}' object has no attribute '{name}'" ) else: return getattr(self._socket, name) @@ -1916,7 +2044,6 @@ class Connection: buf = _text_to_bytes_and_warn("buf", buf) with _ffi.from_buffer(buf) as data: - left_to_send = len(buf) total_sent = 0 @@ -2160,6 +2287,37 @@ class Connection: if result < 0: self._raise_ssl_error(self._ssl, result) + def DTLSv1_get_timeout(self): + """ + Determine when the DTLS SSL object next needs to perform internal + processing due to the passage of time. + + When the returned number of seconds have passed, the + :meth:`DTLSv1_handle_timeout` method needs to be called. + + :return: The time left in seconds before the next timeout or `None` + if no timeout is currently active. + """ + ptv_sec = _ffi.new("time_t *") + ptv_usec = _ffi.new("long *") + if _lib.Cryptography_DTLSv1_get_timeout(self._ssl, ptv_sec, ptv_usec): + return ptv_sec[0] + (ptv_usec[0] / 1000000) + else: + return None + + def DTLSv1_handle_timeout(self): + """ + Handles any timeout events which have become pending on a DTLS SSL + object. + + :return: `True` if there was a pending timeout, `False` otherwise. + """ + result = _lib.DTLSv1_handle_timeout(self._ssl) + if result < 0: + self._raise_ssl_error(self._ssl, result) + else: + return bool(result) + def bio_shutdown(self): """ If the Connection was created with a memory BIO, this method can be diff --git a/contrib/python/pyOpenSSL/py3/OpenSSL/__init__.py b/contrib/python/pyOpenSSL/py3/OpenSSL/__init__.py index 0af3acdb8fd..2442ee6f054 100644 --- a/contrib/python/pyOpenSSL/py3/OpenSSL/__init__.py +++ b/contrib/python/pyOpenSSL/py3/OpenSSL/__init__.py @@ -17,7 +17,6 @@ from OpenSSL.version import ( __version__, ) - __all__ = [ "SSL", "crypto", diff --git a/contrib/python/pyOpenSSL/py3/OpenSSL/crypto.py b/contrib/python/pyOpenSSL/py3/OpenSSL/crypto.py index a9b673c53f5..cccc216bbda 100644 --- a/contrib/python/pyOpenSSL/py3/OpenSSL/crypto.py +++ b/contrib/python/pyOpenSSL/py3/OpenSSL/crypto.py @@ -1,6 +1,7 @@ import calendar import datetime import functools +import typing from base64 import b16encode from functools import partial from os import PathLike @@ -22,23 +23,36 @@ from cryptography import utils, x509 from cryptography.hazmat.primitives.asymmetric import ( dsa, ec, - ed25519, ed448, + ed25519, rsa, ) from OpenSSL._util import ( UNSPECIFIED as _UNSPECIFIED, +) +from OpenSSL._util import ( byte_string as _byte_string, +) +from OpenSSL._util import ( exception_from_error_queue as _exception_from_error_queue, +) +from OpenSSL._util import ( ffi as _ffi, +) +from OpenSSL._util import ( lib as _lib, +) +from OpenSSL._util import ( make_assert as _make_assert, +) +from OpenSSL._util import ( path_bytes as _path_bytes, +) +from OpenSSL._util import ( text_to_bytes_and_warn as _text_to_bytes_and_warn, ) - __all__ = [ "FILETYPE_PEM", "FILETYPE_ASN1", @@ -63,7 +77,6 @@ __all__ = [ "dump_privatekey", "Revoked", "CRL", - "PKCS7", "PKCS12", "NetscapeSPKI", "load_publickey", @@ -74,8 +87,6 @@ __all__ = [ "verify", "dump_crl", "load_crl", - "load_pkcs7_data", - "load_pkcs12", ] @@ -114,7 +125,7 @@ def _untested_error(where: str) -> NoReturn: encountered isn't one that's exercised by the test suite so future behavior of pyOpenSSL is now somewhat less predictable. """ - raise RuntimeError("Unknown %s failure" % (where,)) + raise RuntimeError(f"Unknown {where} failure") def _new_mem_buf(buffer: Optional[bytes] = None) -> Any: @@ -451,7 +462,7 @@ class _EllipticCurve: circumstance. """ if isinstance(other, _EllipticCurve): - return super(_EllipticCurve, self).__ne__(other) + return super().__ne__(other) return NotImplemented @classmethod @@ -521,7 +532,7 @@ class _EllipticCurve: self.name = name def __repr__(self) -> str: - return "<Curve %r>" % (self.name,) + return f"<Curve {self.name!r}>" def _to_EC_KEY(self) -> Any: """ @@ -605,14 +616,15 @@ class X509Name: def __setattr__(self, name: str, value: Any) -> None: if name.startswith("_"): - return super(X509Name, self).__setattr__(name, value) + return super().__setattr__(name, value) # Note: we really do not want str subclasses here, so we do not use # isinstance. - if type(name) is not str: + if type(name) is not str: # noqa: E721 raise TypeError( - "attribute name must be string, not '%.200s'" - % (type(value).__name__,) + "attribute name must be string, not '{:.200}'".format( + type(value).__name__ + ) ) nid = _lib.OBJ_txt2nid(_byte_string(name)) @@ -704,7 +716,7 @@ class X509Name: ) _openssl_assert(format_result != _ffi.NULL) - return "<X509Name object '%s'>" % ( + return "<X509Name object '{}'>".format( _ffi.string(result_buffer).decode("utf-8"), ) @@ -842,7 +854,7 @@ class X509Extension: _lib.X509_EXTENSION_get_object(self._extension) ) - _prefixes = { + _prefixes: typing.ClassVar[typing.Dict[int, str]] = { _lib.GEN_EMAIL: "email", _lib.GEN_DNS: "DNS", _lib.GEN_URI: "URI", @@ -904,7 +916,14 @@ class X509Extension: """ obj = _lib.X509_EXTENSION_get_object(self._extension) nid = _lib.OBJ_obj2nid(obj) - return _ffi.string(_lib.OBJ_nid2sn(nid)) + # OpenSSL 3.1.0 has a bug where nid2sn returns NULL for NIDs that + # previously returned UNDEF. This is a workaround for that issue. + # https://github.com/openssl/openssl/commit/908ba3ed9adbb3df90f76 + buf = _lib.OBJ_nid2sn(nid) + if buf != _ffi.NULL: + return _ffi.string(buf) + else: + return b"UNDEF" def get_data(self) -> bytes: """ @@ -922,6 +941,19 @@ class X509Extension: return _ffi.buffer(char_result, result_length)[:] +_X509ExtensionInternal = X509Extension +utils.deprecated( + X509Extension, + __name__, + ( + "X509Extension support in pyOpenSSL is deprecated. You should use the " + "APIs in cryptography." + ), + DeprecationWarning, + name="X509Extension", +) + + class X509Req: """ An X.509 certificate signing requests. @@ -1003,6 +1035,12 @@ class X509Req: :param int version: The version number. :return: ``None`` """ + if not isinstance(version, int): + raise TypeError("version must be an int") + if version != 0: + raise ValueError( + "Invalid version. The only valid version for X509Req is 0." + ) set_result = _lib.X509_REQ_set_version(self._req, version) _openssl_assert(set_result == 1) @@ -1038,7 +1076,9 @@ class X509Req: return name - def add_extensions(self, extensions: Iterable[X509Extension]) -> None: + def add_extensions( + self, extensions: Iterable[_X509ExtensionInternal] + ) -> None: """ Add extensions to the certificate signing request. @@ -1052,7 +1092,7 @@ class X509Req: stack = _ffi.gc(stack, _lib.sk_X509_EXTENSION_free) for ext in extensions: - if not isinstance(ext, X509Extension): + if not isinstance(ext, _X509ExtensionInternal): raise ValueError("One of the elements is not an X509Extension") # TODO push can fail (here and elsewhere) @@ -1061,7 +1101,7 @@ class X509Req: add_result = _lib.X509_REQ_add_extensions(self._req, stack) _openssl_assert(add_result == 1) - def get_extensions(self) -> List[X509Extension]: + def get_extensions(self) -> List[_X509ExtensionInternal]: """ Get X.509 extensions in the certificate signing request. @@ -1081,7 +1121,7 @@ class X509Req: ) for i in range(_lib.sk_X509_EXTENSION_num(native_exts_obj)): - ext = X509Extension.__new__(X509Extension) + ext = _X509ExtensionInternal.__new__(_X509ExtensionInternal) extension = _lib.X509_EXTENSION_dup( _lib.sk_X509_EXTENSION_value(native_exts_obj, i) ) @@ -1282,8 +1322,10 @@ class X509: .. versionadded:: 0.13 """ - algor = _lib.X509_get0_tbs_sigalg(self._x509) - nid = _lib.OBJ_obj2nid(algor.algorithm) + sig_alg = _lib.X509_get0_tbs_sigalg(self._x509) + alg = _ffi.new("ASN1_OBJECT **") + _lib.X509_ALGOR_get0(alg, _ffi.NULL, _ffi.NULL, sig_alg) + nid = _lib.OBJ_obj2nid(alg[0]) if nid == _lib.NID_undef: raise ValueError("Undefined signature algorithm") return _ffi.string(_lib.OBJ_nid2ln(nid)) @@ -1427,7 +1469,9 @@ class X509: time_string = time_bytes.decode("utf-8") not_after = datetime.datetime.strptime(time_string, "%Y%m%d%H%M%SZ") - return not_after < datetime.datetime.utcnow() + UTC = datetime.timezone.utc + utcnow = datetime.datetime.now(UTC).replace(tzinfo=None) + return not_after < utcnow def _get_boundary_time(self, which: Any) -> Optional[bytes]: return _get_asn1_time(which(self._x509)) @@ -1573,7 +1617,9 @@ class X509: """ return _lib.X509_get_ext_count(self._x509) - def add_extensions(self, extensions: Iterable[X509Extension]) -> None: + def add_extensions( + self, extensions: Iterable[_X509ExtensionInternal] + ) -> None: """ Add extensions to the certificate. @@ -1582,14 +1628,14 @@ class X509: :return: ``None`` """ for ext in extensions: - if not isinstance(ext, X509Extension): + if not isinstance(ext, _X509ExtensionInternal): raise ValueError("One of the elements is not an X509Extension") add_result = _lib.X509_add_ext(self._x509, ext._extension, -1) if not add_result: _raise_current_error() - def get_extension(self, index: int) -> X509Extension: + def get_extension(self, index: int) -> _X509ExtensionInternal: """ Get a specific extension of the certificate by index. @@ -1603,7 +1649,7 @@ class X509: .. versionadded:: 0.12 """ - ext = X509Extension.__new__(X509Extension) + ext = _X509ExtensionInternal.__new__(_X509ExtensionInternal) ext._extension = _lib.X509_get_ext(self._x509, index) if ext._extension == _ffi.NULL: raise IndexError("extension index out of bounds") @@ -1632,7 +1678,6 @@ class X509StoreFlags: POLICY_CHECK: int = _lib.X509_V_FLAG_POLICY_CHECK EXPLICIT_POLICY: int = _lib.X509_V_FLAG_EXPLICIT_POLICY INHIBIT_MAP: int = _lib.X509_V_FLAG_INHIBIT_MAP - NOTIFY_POLICY: int = _lib.X509_V_FLAG_NOTIFY_POLICY CHECK_SS_SIGNATURE: int = _lib.X509_V_FLAG_CHECK_SS_SIGNATURE PARTIAL_CHAIN: int = _lib.X509_V_FLAG_PARTIAL_CHAIN @@ -1677,7 +1722,9 @@ class X509Store: res = _lib.X509_STORE_add_cert(self._store, cert._x509) _openssl_assert(res == 1) - def add_crl(self, crl: "CRL") -> None: + def add_crl( + self, crl: Union["_CRLInternal", x509.CertificateRevocationList] + ) -> None: """ Add a certificate revocation list to this store. @@ -1687,11 +1734,29 @@ class X509Store: .. versionadded:: 16.1.0 - :param CRL crl: The certificate revocation list to add to this store. + :param crl: The certificate revocation list to add to this store. + :type crl: ``Union[CRL, cryptography.x509.CertificateRevocationList]`` :return: ``None`` if the certificate revocation list was added successfully. """ - _openssl_assert(_lib.X509_STORE_add_crl(self._store, crl._crl) != 0) + if isinstance(crl, x509.CertificateRevocationList): + from cryptography.hazmat.primitives.serialization import Encoding + + bio = _new_mem_buf(crl.public_bytes(Encoding.DER)) + openssl_crl = _lib.d2i_X509_CRL_bio(bio, _ffi.NULL) + if openssl_crl == _ffi.NULL: + _raise_current_error() + + crl = _ffi.gc(openssl_crl, _lib.X509_CRL_free) + elif isinstance(crl, _CRLInternal): + crl = crl._crl + else: + raise TypeError( + "CRL must be of type OpenSSL.crypto.CRL or " + "cryptography.x509.CertificateRevocationList" + ) + + _openssl_assert(_lib.X509_STORE_add_crl(self._store, crl) != 0) def set_flags(self, flags: int) -> None: """ @@ -1803,7 +1868,7 @@ class X509StoreContextError(Exception): def __init__( self, message: str, errors: List[Any], certificate: X509 ) -> None: - super(X509StoreContextError, self).__init__(message) + super().__init__(message) self.errors = errors self.certificate = certificate @@ -2155,7 +2220,7 @@ class Revoked: # which differs from crl_reasons of crypto/x509v3/v3_enum.c that matches # OCSP_crl_reason_str. We use the latter, just like the command line # program. - _crl_reasons = [ + _crl_reasons: typing.ClassVar[typing.List[bytes]] = [ b"unspecified", b"keyCompromise", b"CACompromise", @@ -2323,6 +2388,19 @@ class Revoked: return _get_asn1_time(dt) +_RevokedInternal = Revoked +utils.deprecated( + Revoked, + __name__, + ( + "CRL support in pyOpenSSL is deprecated. You should use the APIs " + "in cryptography." + ), + DeprecationWarning, + name="Revoked", +) + + class CRL: """ A certificate revocation list. @@ -2342,13 +2420,13 @@ class CRL: """ from cryptography.x509 import load_der_x509_crl - der = dump_crl(FILETYPE_ASN1, self) + der = _dump_crl_internal(FILETYPE_ASN1, self) return load_der_x509_crl(der) @classmethod def from_cryptography( cls, crypto_crl: x509.CertificateRevocationList - ) -> "CRL": + ) -> "_CRLInternal": """ Construct based on a ``cryptography`` *crypto_crl*. @@ -2365,9 +2443,9 @@ class CRL: from cryptography.hazmat.primitives.serialization import Encoding der = crypto_crl.public_bytes(Encoding.DER) - return load_crl(FILETYPE_ASN1, der) + return _load_crl_internal(FILETYPE_ASN1, der) - def get_revoked(self) -> Optional[Tuple[Revoked, ...]]: + def get_revoked(self) -> Optional[Tuple[_RevokedInternal, ...]]: """ Return the revocations in this certificate revocation list. @@ -2382,14 +2460,14 @@ class CRL: for i in range(_lib.sk_X509_REVOKED_num(revoked_stack)): revoked = _lib.sk_X509_REVOKED_value(revoked_stack, i) revoked_copy = _lib.X509_REVOKED_dup(revoked) - pyrev = Revoked.__new__(Revoked) + pyrev = _RevokedInternal.__new__(_RevokedInternal) pyrev._revoked = _ffi.gc(revoked_copy, _lib.X509_REVOKED_free) results.append(pyrev) if results: return tuple(results) return None - def add_revoked(self, revoked: Revoked) -> None: + def add_revoked(self, revoked: _RevokedInternal) -> None: """ Add a revoked (by value not reference) to the CRL structure @@ -2552,54 +2630,20 @@ class CRL: if not sign_result: _raise_current_error() - return dump_crl(type, self) - - -class PKCS7: - - _pkcs7: Any - - def type_is_signed(self) -> bool: - """ - Check if this NID_pkcs7_signed object - - :return: True if the PKCS7 is of type signed - """ - return bool(_lib.PKCS7_type_is_signed(self._pkcs7)) - - def type_is_enveloped(self) -> bool: - """ - Check if this NID_pkcs7_enveloped object - - :returns: True if the PKCS7 is of type enveloped - """ - return bool(_lib.PKCS7_type_is_enveloped(self._pkcs7)) - - def type_is_signedAndEnveloped(self) -> bool: - """ - Check if this NID_pkcs7_signedAndEnveloped object - - :returns: True if the PKCS7 is of type signedAndEnveloped - """ - return bool(_lib.PKCS7_type_is_signedAndEnveloped(self._pkcs7)) - - def type_is_data(self) -> bool: - """ - Check if this NID_pkcs7_data object - - :return: True if the PKCS7 is of type data - """ - return bool(_lib.PKCS7_type_is_data(self._pkcs7)) + return _dump_crl_internal(type, self) - def get_type_name(self) -> str: - """ - Returns the type name of the PKCS7 structure - :return: A string with the typename - """ - nid = _lib.OBJ_obj2nid(self._pkcs7.type) - string_type = _lib.OBJ_nid2sn(nid) - return _ffi.string(string_type) +_CRLInternal = CRL +utils.deprecated( + CRL, + __name__, + ( + "CRL support in pyOpenSSL is deprecated. You should use the APIs " + "in cryptography." + ), + DeprecationWarning, + name="CRL", +) class PKCS12: @@ -2703,7 +2747,7 @@ class PKCS12: self._friendlyname = None elif not isinstance(name, bytes): raise TypeError( - "name must be a byte string or None (not %r)" % (name,) + f"name must be a byte string or None (not {name!r})" ) self._friendlyname = name @@ -2789,6 +2833,18 @@ class PKCS12: return _bio_to_string(bio) +utils.deprecated( + PKCS12, + __name__, + ( + "PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs " + "in cryptography." + ), + DeprecationWarning, + name="PKCS12", +) + + class NetscapeSPKI: """ A Netscape SPKI object. @@ -2879,6 +2935,15 @@ class NetscapeSPKI: _openssl_assert(set_result == 1) +utils.deprecated( + NetscapeSPKI, + __name__, + "NetscapeSPKI support in pyOpenSSL is deprecated.", + DeprecationWarning, + name="NetscapeSPKI", +) + + class _PassphraseHelper: def __init__( self, @@ -2920,7 +2985,6 @@ class _PassphraseHelper: def raise_if_problem(self, exceptionType: Type[Exception] = Error) -> None: if self._problems: - # Flush the OpenSSL error queue try: _exception_from_error_queue(exceptionType) @@ -3124,6 +3188,15 @@ def sign(pkey: PKey, data: Union[str, bytes], digest: str) -> bytes: return _ffi.buffer(signature_buffer, signature_length[0])[:] +utils.deprecated( + sign, + __name__, + "sign() is deprecated. Use the equivilant APIs in cryptography.", + DeprecationWarning, + name="sign", +) + + def verify( cert: X509, signature: bytes, data: Union[str, bytes], digest: str ) -> None: @@ -3162,7 +3235,16 @@ def verify( _raise_current_error() -def dump_crl(type: int, crl: CRL) -> bytes: +utils.deprecated( + verify, + __name__, + "verify() is deprecated. Use the equivilant APIs in cryptography.", + DeprecationWarning, + name="verify", +) + + +def dump_crl(type: int, crl: _CRLInternal) -> bytes: """ Dump a certificate revocation list to a buffer. @@ -3191,7 +3273,20 @@ def dump_crl(type: int, crl: CRL) -> bytes: return _bio_to_string(bio) -def load_crl(type: int, buffer: Union[str, bytes]) -> CRL: +_dump_crl_internal = dump_crl +utils.deprecated( + dump_crl, + __name__, + ( + "CRL support in pyOpenSSL is deprecated. You should use the APIs " + "in cryptography." + ), + DeprecationWarning, + name="dump_crl", +) + + +def load_crl(type: int, buffer: Union[str, bytes]) -> _CRLInternal: """ Load Certificate Revocation List (CRL) data from a string *buffer*. *buffer* encoded with the type *type*. @@ -3216,146 +3311,19 @@ def load_crl(type: int, buffer: Union[str, bytes]) -> CRL: if crl == _ffi.NULL: _raise_current_error() - result = CRL.__new__(CRL) + result = _CRLInternal.__new__(_CRLInternal) result._crl = _ffi.gc(crl, _lib.X509_CRL_free) return result -def load_pkcs7_data(type: int, buffer: Union[str, bytes]) -> PKCS7: - """ - Load pkcs7 data from the string *buffer* encoded with the type - *type*. - - :param type: The file type (one of FILETYPE_PEM or FILETYPE_ASN1) - :param buffer: The buffer with the pkcs7 data. - :return: The PKCS7 object - """ - if isinstance(buffer, str): - buffer = buffer.encode("ascii") - - bio = _new_mem_buf(buffer) - - if type == FILETYPE_PEM: - pkcs7 = _lib.PEM_read_bio_PKCS7(bio, _ffi.NULL, _ffi.NULL, _ffi.NULL) - elif type == FILETYPE_ASN1: - pkcs7 = _lib.d2i_PKCS7_bio(bio, _ffi.NULL) - else: - raise ValueError("type argument must be FILETYPE_PEM or FILETYPE_ASN1") - - if pkcs7 == _ffi.NULL: - _raise_current_error() - - pypkcs7 = PKCS7.__new__(PKCS7) - pypkcs7._pkcs7 = _ffi.gc(pkcs7, _lib.PKCS7_free) - return pypkcs7 - - -utils.deprecated( - load_pkcs7_data, - __name__, - ( - "PKCS#7 support in pyOpenSSL is deprecated. You should use the APIs " - "in cryptography." - ), - DeprecationWarning, - name="load_pkcs7_data", -) - - -def load_pkcs12( - buffer: Union[str, bytes], passphrase: Optional[bytes] = None -) -> PKCS12: - """ - Load pkcs12 data from the string *buffer*. If the pkcs12 structure is - encrypted, a *passphrase* must be included. The MAC is always - checked and thus required. - - See also the man page for the C function :py:func:`PKCS12_parse`. - - :param buffer: The buffer the certificate is stored in - :param passphrase: (Optional) The password to decrypt the PKCS12 lump - :returns: The PKCS12 object - """ - passphrase = _text_to_bytes_and_warn("passphrase", passphrase) - - if isinstance(buffer, str): - buffer = buffer.encode("ascii") - - bio = _new_mem_buf(buffer) - - # Use null passphrase if passphrase is None or empty string. With PKCS#12 - # password based encryption no password and a zero length password are two - # different things, but OpenSSL implementation will try both to figure out - # which one works. - if not passphrase: - passphrase = _ffi.NULL - - p12 = _lib.d2i_PKCS12_bio(bio, _ffi.NULL) - if p12 == _ffi.NULL: - _raise_current_error() - p12 = _ffi.gc(p12, _lib.PKCS12_free) - - pkey = _ffi.new("EVP_PKEY**") - cert = _ffi.new("X509**") - cacerts = _ffi.new("Cryptography_STACK_OF_X509**") - - parse_result = _lib.PKCS12_parse(p12, passphrase, pkey, cert, cacerts) - if not parse_result: - _raise_current_error() - - cacerts = _ffi.gc(cacerts[0], _lib.sk_X509_free) - - # openssl 1.0.0 sometimes leaves an X509_check_private_key error in the - # queue for no particular reason. This error isn't interesting to anyone - # outside this function. It's not even interesting to us. Get rid of it. - try: - _raise_current_error() - except Error: - pass - - if pkey[0] == _ffi.NULL: - pykey = None - else: - pykey = PKey.__new__(PKey) - pykey._pkey = _ffi.gc(pkey[0], _lib.EVP_PKEY_free) - - if cert[0] == _ffi.NULL: - pycert = None - friendlyname = None - else: - pycert = X509._from_raw_x509_ptr(cert[0]) - - friendlyname_length = _ffi.new("int*") - friendlyname_buffer = _lib.X509_alias_get0( - cert[0], friendlyname_length - ) - friendlyname = _ffi.buffer( - friendlyname_buffer, friendlyname_length[0] - )[:] - if friendlyname_buffer == _ffi.NULL: - friendlyname = None - - pycacerts = [] - for i in range(_lib.sk_X509_num(cacerts)): - x509 = _lib.sk_X509_value(cacerts, i) - pycacert = X509._from_raw_x509_ptr(x509) - pycacerts.append(pycacert) - - pkcs12 = PKCS12.__new__(PKCS12) - pkcs12._pkey = pykey - pkcs12._cert = pycert - pkcs12._cacerts = pycacerts if pycacerts else None - pkcs12._friendlyname = friendlyname - return pkcs12 - - +_load_crl_internal = load_crl utils.deprecated( - load_pkcs12, + load_crl, __name__, ( - "PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs " + "CRL support in pyOpenSSL is deprecated. You should use the APIs " "in cryptography." ), DeprecationWarning, - name="load_pkcs12", + name="load_crl", ) diff --git a/contrib/python/pyOpenSSL/py3/OpenSSL/debug.py b/contrib/python/pyOpenSSL/py3/OpenSSL/debug.py index e39b128a7e2..e0ed3f81d62 100644 --- a/contrib/python/pyOpenSSL/py3/OpenSSL/debug.py +++ b/contrib/python/pyOpenSSL/py3/OpenSSL/debug.py @@ -1,17 +1,13 @@ -from __future__ import print_function - import ssl import sys import cffi - import cryptography import OpenSSL.SSL from . import version - _env_info = """\ pyOpenSSL: {pyopenssl} cryptography: {cryptography} diff --git a/contrib/python/pyOpenSSL/py3/OpenSSL/version.py b/contrib/python/pyOpenSSL/py3/OpenSSL/version.py index 61ec17b2d8f..4120cff6dda 100644 --- a/contrib/python/pyOpenSSL/py3/OpenSSL/version.py +++ b/contrib/python/pyOpenSSL/py3/OpenSSL/version.py @@ -17,7 +17,7 @@ __all__ = [ "__version__", ] -__version__ = "23.0.0" +__version__ = "23.3.0" __title__ = "pyOpenSSL" __uri__ = "https://pyopenssl.org/" @@ -25,4 +25,4 @@ __summary__ = "Python wrapper module around the OpenSSL library" __author__ = "The pyOpenSSL developers" __email__ = "[email protected]" __license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2001-2023 {0}".format(__author__) +__copyright__ = f"Copyright 2001-2023 {__author__}" diff --git a/contrib/python/pyOpenSSL/py3/tests/conftest.py b/contrib/python/pyOpenSSL/py3/tests/conftest.py index 5bae6b8f717..573b4b3d602 100644 --- a/contrib/python/pyOpenSSL/py3/tests/conftest.py +++ b/contrib/python/pyOpenSSL/py3/tests/conftest.py @@ -7,9 +7,10 @@ import pytest def pytest_report_header(config): - import OpenSSL.SSL import cryptography + import OpenSSL.SSL + return "OpenSSL: {openssl}\ncryptography: {cryptography}".format( openssl=OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION), cryptography=cryptography.__version__, diff --git a/contrib/python/pyOpenSSL/py3/tests/memdbg.py b/contrib/python/pyOpenSSL/py3/tests/memdbg.py index 0dd8c318e4e..5eed1ba8b0a 100644 --- a/contrib/python/pyOpenSSL/py3/tests/memdbg.py +++ b/contrib/python/pyOpenSSL/py3/tests/memdbg.py @@ -3,7 +3,6 @@ import traceback from cffi import api as _api - sys.modules["ssl"] = None sys.modules["_hashlib"] = None @@ -22,7 +21,7 @@ _ffi.cdef( char **backtrace_symbols(void *const *buffer, int size); void backtrace_symbols_fd(void *const *buffer, int size, int fd); """ -) # noqa +) _api = _ffi.verify( """ #include <openssl/crypto.h> @@ -79,7 +78,7 @@ def free(p): if p != _ffi.NULL: C.free(p) del heap[p] - log("free(0x%x)" % (int(_ffi.cast("int", p)),)) + log("free(0x{:x})".format(int(_ffi.cast("int", p)))) if _api.CRYPTO_set_mem_functions(malloc, realloc, free): diff --git a/contrib/python/pyOpenSSL/py3/tests/test_crypto.py b/contrib/python/pyOpenSSL/py3/tests/test_crypto.py index 44bbd0f2aba..6351f0d703c 100644 --- a/contrib/python/pyOpenSSL/py3/tests/test_crypto.py +++ b/contrib/python/pyOpenSSL/py3/tests/test_crypto.py @@ -6,34 +6,27 @@ Unit tests for :py:mod:`OpenSSL.crypto`. """ import base64 import sys -from datetime import datetime, timedelta +import warnings +from datetime import datetime, timedelta, timezone from subprocess import PIPE, Popen -from warnings import simplefilter - -from cryptography import x509 -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec, ed25519, ed448, rsa import flaky - import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec, ed448, ed25519, rsa -from OpenSSL._util import ffi as _ffi, lib as _lib +from OpenSSL._util import ffi as _ffi +from OpenSSL._util import lib as _lib from OpenSSL.crypto import ( - CRL, - Error, FILETYPE_ASN1, FILETYPE_PEM, FILETYPE_TEXT, - NetscapeSPKI, - PKCS12, - PKCS7, - PKey, - Revoked, TYPE_DSA, TYPE_RSA, X509, - X509Extension, + Error, + PKey, X509Name, X509Req, X509Store, @@ -42,26 +35,33 @@ from OpenSSL.crypto import ( X509StoreFlags, dump_certificate, dump_certificate_request, - dump_crl, dump_privatekey, dump_publickey, get_elliptic_curve, get_elliptic_curves, load_certificate, load_certificate_request, - load_crl, - load_pkcs12, - load_pkcs7_data, load_privatekey, load_publickey, sign, verify, ) +with pytest.warns(DeprecationWarning): + from OpenSSL.crypto import ( + CRL, + PKCS12, + NetscapeSPKI, + Revoked, + X509Extension, + dump_crl, + load_crl, + ) + from .util import ( - EqualityTestsMixin, NON_ASCII, WARNING_TYPE_EXPECTED, + EqualityTestsMixin, is_consistent_type, ) @@ -70,6 +70,10 @@ def normalize_privatekey_pem(pem): return dump_privatekey(FILETYPE_PEM, load_privatekey(FILETYPE_PEM, pem)) +def utcnow(): + return datetime.now(timezone.utc).replace(tzinfo=None) + + GOOD_CIPHER = "blowfish" BAD_CIPHER = "zippers" @@ -554,57 +558,6 @@ ywIDAQAB -----END PUBLIC KEY----- """ -# Some PKCS#7 stuff. Generated with the openssl command line: -# -# openssl crl2pkcs7 -inform pem -outform pem -certfile s.pem -nocrl -# -# with a certificate and key (but the key should be irrelevant) in s.pem -pkcs7Data = b"""\ ------BEGIN PKCS7----- -MIIDNwYJKoZIhvcNAQcCoIIDKDCCAyQCAQExADALBgkqhkiG9w0BBwGgggMKMIID -BjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzERMA8G -A1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQDExtN -MkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5ncHNA -cG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzELMAkG -A1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhvc3Qx -HTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEBBQAD -SwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh5kwI -zOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQCMAAw -LAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0G -A1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7hyNp -65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoTCE0y -Q3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlwdG8g -Q2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3QxLmNv -bYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6BoJu -VwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++7QGG -/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JEWUQ9 -Ho4EzbYCOaEAMQA= ------END PKCS7----- -""" - -pkcs7DataASN1 = base64.b64decode( - b""" -MIIDNwYJKoZIhvcNAQcCoIIDKDCCAyQCAQExADALBgkqhkiG9w0BBwGgggMKMIID -BjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzERMA8G -A1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQDExtN -MkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5ncHNA -cG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzELMAkG -A1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhvc3Qx -HTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEBBQAD -SwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh5kwI -zOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQCMAAw -LAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0G -A1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7hyNp -65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoTCE0y -Q3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlwdG8g -Q2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3QxLmNv -bYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6BoJu -VwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++7QGG -/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JEWUQ9 -Ho4EzbYCOaEAMQA= -""" -) - crlData = b"""\ -----BEGIN X509 CRL----- MIIBWzCBxTANBgkqhkiG9w0BAQQFADBYMQswCQYDVQQGEwJVUzELMAkGA1UECBMC @@ -1601,20 +1554,12 @@ class TestX509Req(_PKeyInteractionTestsMixin): """ `X509Req.set_version` sets the X.509 version of the certificate request. `X509Req.get_version` returns the X.509 version of the - certificate request. The only defined version is 0. Others may or - may not be supported depending on backend. + certificate request. The only defined version is 0. """ request = X509Req() assert request.get_version() == 0 request.set_version(0) assert request.get_version() == 0 - try: - request.set_version(1) - assert request.get_version() == 1 - request.set_version(3) - assert request.get_version() == 3 - except Error: - pass def test_version_wrong_args(self): """ @@ -1624,6 +1569,8 @@ class TestX509Req(_PKeyInteractionTestsMixin): request = X509Req() with pytest.raises(TypeError): request.set_version("foo") + with pytest.raises(ValueError): + request.set_version(2) def test_get_subject(self): """ @@ -1681,6 +1628,14 @@ class TestX509Req(_PKeyInteractionTestsMixin): exts = request.get_extensions() assert len(exts) == 2 + def test_undef_oid(self): + assert ( + X509Extension( + b"1.2.3.4.5.6.7", False, b"DER:05:00" + ).get_short_name() + == b"UNDEF" + ) + def test_add_extensions_wrong_args(self): """ `X509Req.add_extensions` raises `TypeError` if called with a @@ -1916,14 +1871,14 @@ class TestX509(_PKeyInteractionTestsMixin): current time plus the number of seconds passed in. """ cert = load_certificate(FILETYPE_PEM, self.pemData) - not_before_min = datetime.utcnow().replace(microsecond=0) + timedelta( + not_before_min = utcnow().replace(microsecond=0) + timedelta( seconds=100 ) cert.gmtime_adj_notBefore(100) not_before = datetime.strptime( cert.get_notBefore().decode(), "%Y%m%d%H%M%SZ" ) - not_before_max = datetime.utcnow() + timedelta(seconds=100) + not_before_max = utcnow() + timedelta(seconds=100) assert not_before_min <= not_before <= not_before_max def test_gmtime_adj_notAfter_wrong_args(self): @@ -1942,14 +1897,14 @@ class TestX509(_PKeyInteractionTestsMixin): to be the current time plus the number of seconds passed in. """ cert = load_certificate(FILETYPE_PEM, self.pemData) - not_after_min = datetime.utcnow().replace(microsecond=0) + timedelta( + not_after_min = utcnow().replace(microsecond=0) + timedelta( seconds=100 ) cert.gmtime_adj_notAfter(100) not_after = datetime.strptime( cert.get_notAfter().decode(), "%Y%m%d%H%M%SZ" ) - not_after_max = datetime.utcnow() + timedelta(seconds=100) + not_after_max = utcnow() + timedelta(seconds=100) assert not_after_min <= not_after <= not_after_max def test_has_expired(self): @@ -2384,7 +2339,7 @@ class TestX509Store: class TestPKCS12: """ - Test for `OpenSSL.crypto.PKCS12` and `OpenSSL.crypto.load_pkcs12`. + Test for `OpenSSL.crypto.PKCS12`. """ def test_type(self): @@ -2427,7 +2382,7 @@ class TestPKCS12: def test_key_only(self): """ A `PKCS12` with only a private key can be exported using - `PKCS12.export` and loaded again using `load_pkcs12`. + `PKCS12.export`. """ passwd = b"blah" p12 = PKCS12() @@ -2435,25 +2390,12 @@ class TestPKCS12: p12.set_privatekey(pkey) assert None is p12.get_certificate() assert pkey == p12.get_privatekey() - try: - dumped_p12 = p12.export(passphrase=passwd, iter=2, maciter=3) - except Error: - # Some versions of OpenSSL will throw an exception - # for this nearly useless PKCS12 we tried to generate: - # [('PKCS12 routines', 'PKCS12_create', 'invalid null argument')] - return - p12 = load_pkcs12(dumped_p12, passwd) - assert None is p12.get_ca_certificates() - assert None is p12.get_certificate() - - # OpenSSL fails to bring the key back to us. So sad. Perhaps in the - # future this will be improved. - assert isinstance(p12.get_privatekey(), (PKey, type(None))) + p12.export(passphrase=passwd, iter=2, maciter=3) def test_cert_only(self): """ A `PKCS12` with only a certificate can be exported using - `PKCS12.export` and loaded again using `load_pkcs12`. + `PKCS12.export`. """ passwd = b"blah" p12 = PKCS12() @@ -2461,51 +2403,26 @@ class TestPKCS12: p12.set_certificate(cert) assert cert == p12.get_certificate() assert None is p12.get_privatekey() - try: - dumped_p12 = p12.export(passphrase=passwd, iter=2, maciter=3) - except Error: - # Some versions of OpenSSL will throw an exception - # for this nearly useless PKCS12 we tried to generate: - # [('PKCS12 routines', 'PKCS12_create', 'invalid null argument')] - return - p12 = load_pkcs12(dumped_p12, passwd) - assert None is p12.get_privatekey() - - # OpenSSL fails to bring the cert back to us. Groany mcgroan. - assert isinstance(p12.get_certificate(), (X509, type(None))) - - # Oh ho. It puts the certificate into the ca certificates list, in - # fact. Totally bogus, I would think. Nevertheless, let's exploit - # that to check to see if it reconstructed the certificate we expected - # it to. At some point, hopefully this will change so that - # p12.get_certificate() is actually what returns the loaded - # certificate. - assert root_cert_pem == dump_certificate( - FILETYPE_PEM, p12.get_ca_certificates()[0] - ) + p12.export(passphrase=passwd, iter=2, maciter=3) - def gen_pkcs12( - self, cert_pem=None, key_pem=None, ca_pem=None, friendly_name=None - ): + def gen_pkcs12(self, cert_pem=None, key_pem=None, ca_pem=None): """ Generate a PKCS12 object with components from PEM. Verify that the set functions return None. """ p12 = PKCS12() - if cert_pem: - ret = p12.set_certificate(load_certificate(FILETYPE_PEM, cert_pem)) - assert ret is None - if key_pem: - ret = p12.set_privatekey(load_privatekey(FILETYPE_PEM, key_pem)) - assert ret is None + + ret = p12.set_certificate(load_certificate(FILETYPE_PEM, cert_pem)) + assert ret is None + + ret = p12.set_privatekey(load_privatekey(FILETYPE_PEM, key_pem)) + assert ret is None + if ca_pem: ret = p12.set_ca_certificates( (load_certificate(FILETYPE_PEM, ca_pem),) ) assert ret is None - if friendly_name: - ret = p12.set_friendlyname(friendly_name) - assert ret is None return p12 def check_recovery( @@ -2515,29 +2432,29 @@ class TestPKCS12: Use openssl program to confirm three components are recoverable from a PKCS12 string. """ - if key: - recovered_key = _runopenssl( - p12_str, - b"pkcs12", - b"-nocerts", - b"-nodes", - b"-passin", - b"pass:" + passwd, - *extra, - ) - assert recovered_key[-len(key) :] == key - if cert: - recovered_cert = _runopenssl( - p12_str, - b"pkcs12", - b"-clcerts", - b"-nodes", - b"-passin", - b"pass:" + passwd, - b"-nokeys", - *extra, - ) - assert recovered_cert[-len(cert) :] == cert + recovered_key = _runopenssl( + p12_str, + b"pkcs12", + b"-nocerts", + b"-nodes", + b"-passin", + b"pass:" + passwd, + *extra, + ).replace(b"\r\n", b"\n") + assert recovered_key[-len(key) :] == key + + recovered_cert = _runopenssl( + p12_str, + b"pkcs12", + b"-clcerts", + b"-nodes", + b"-passin", + b"pass:" + passwd, + b"-nokeys", + *extra, + ).replace(b"\r\n", b"\n") + assert recovered_cert[-len(cert) :] == cert + if ca: recovered_cert = _runopenssl( p12_str, @@ -2548,141 +2465,9 @@ class TestPKCS12: b"pass:" + passwd, b"-nokeys", *extra, - ) + ).replace(b"\r\n", b"\n") assert recovered_cert[-len(ca) :] == ca - def verify_pkcs12_container(self, p12): - """ - Verify that the PKCS#12 container contains the correct client - certificate and private key. - - :param p12: The PKCS12 instance to verify. - :type p12: `PKCS12` - """ - cert_pem = dump_certificate(FILETYPE_PEM, p12.get_certificate()) - key_pem = dump_privatekey(FILETYPE_PEM, p12.get_privatekey()) - assert (client_cert_pem, client_key_pem, None) == ( - cert_pem, - key_pem, - p12.get_ca_certificates(), - ) - - def test_load_pkcs12(self): - """ - A PKCS12 string generated using the openssl command line can be loaded - with `load_pkcs12` and its components extracted and examined. - """ - passwd = b"whatever" - pem = client_key_pem + client_cert_pem - p12_str = _runopenssl( - pem, - b"pkcs12", - b"-export", - b"-clcerts", - b"-passout", - b"pass:" + passwd, - ) - p12 = load_pkcs12(p12_str, passphrase=passwd) - self.verify_pkcs12_container(p12) - - def test_load_pkcs12_text_passphrase(self): - """ - A PKCS12 string generated using the openssl command line can be loaded - with `load_pkcs12` and its components extracted and examined. - Using text as passphrase instead of bytes. DeprecationWarning expected. - """ - pem = client_key_pem + client_cert_pem - passwd = b"whatever" - p12_str = _runopenssl( - pem, - b"pkcs12", - b"-export", - b"-clcerts", - b"-passout", - b"pass:" + passwd, - ) - with pytest.warns(DeprecationWarning) as w: - simplefilter("always") - p12 = load_pkcs12(p12_str, passphrase=b"whatever".decode("ascii")) - msg = "{0} for passphrase is no longer accepted, use bytes".format( - WARNING_TYPE_EXPECTED - ) - assert msg == str(w[-1].message) - - self.verify_pkcs12_container(p12) - - def test_load_pkcs12_no_passphrase(self): - """ - A PKCS12 string generated using openssl command line can be loaded with - `load_pkcs12` without a passphrase and its components extracted - and examined. - """ - pem = client_key_pem + client_cert_pem - p12_str = _runopenssl( - pem, b"pkcs12", b"-export", b"-clcerts", b"-passout", b"pass:" - ) - p12 = load_pkcs12(p12_str) - self.verify_pkcs12_container(p12) - - def _dump_and_load(self, dump_passphrase, load_passphrase): - """ - A helper method to dump and load a PKCS12 object. - """ - p12 = self.gen_pkcs12(client_cert_pem, client_key_pem) - dumped_p12 = p12.export(passphrase=dump_passphrase, iter=2, maciter=3) - return load_pkcs12(dumped_p12, passphrase=load_passphrase) - - def test_load_pkcs12_null_passphrase_load_empty(self): - """ - A PKCS12 string can be dumped with a null passphrase, loaded with an - empty passphrase with `load_pkcs12`, and its components - extracted and examined. - """ - self.verify_pkcs12_container( - self._dump_and_load(dump_passphrase=None, load_passphrase=b"") - ) - - def test_load_pkcs12_null_passphrase_load_null(self): - """ - A PKCS12 string can be dumped with a null passphrase, loaded with a - null passphrase with `load_pkcs12`, and its components - extracted and examined. - """ - self.verify_pkcs12_container( - self._dump_and_load(dump_passphrase=None, load_passphrase=None) - ) - - def test_load_pkcs12_empty_passphrase_load_empty(self): - """ - A PKCS12 string can be dumped with an empty passphrase, loaded with an - empty passphrase with `load_pkcs12`, and its components - extracted and examined. - """ - self.verify_pkcs12_container( - self._dump_and_load(dump_passphrase=b"", load_passphrase=b"") - ) - - def test_load_pkcs12_empty_passphrase_load_null(self): - """ - A PKCS12 string can be dumped with an empty passphrase, loaded with a - null passphrase with `load_pkcs12`, and its components - extracted and examined. - """ - self.verify_pkcs12_container( - self._dump_and_load(dump_passphrase=b"", load_passphrase=None) - ) - - def test_load_pkcs12_garbage(self): - """ - `load_pkcs12` raises `OpenSSL.crypto.Error` when passed - a string which is not a PKCS12 dump. - """ - passwd = b"whatever" - with pytest.raises(Error) as err: - load_pkcs12(b"fruit loops", passwd) - assert err.value.args[0][0][0] == "asn1 encoding routines" - assert len(err.value.args[0][0]) == 3 - def test_replace(self): """ `PKCS12.set_certificate` replaces the certificate in a PKCS12 @@ -2713,20 +2498,7 @@ class TestPKCS12: for friendly_name in [b"Serverlicious", None, b"###"]: p12.set_friendlyname(friendly_name) assert p12.get_friendlyname() == friendly_name - dumped_p12 = p12.export(passphrase=passwd, iter=2, maciter=3) - reloaded_p12 = load_pkcs12(dumped_p12, passwd) - assert p12.get_friendlyname() == reloaded_p12.get_friendlyname() - # We would use the openssl program to confirm the friendly - # name, but it is not possible. The pkcs12 command - # does not store the friendly name in the cert's - # alias, which we could then extract. - self.check_recovery( - dumped_p12, - key=server_key_pem, - cert=server_cert_pem, - ca=root_cert_pem, - passwd=passwd, - ) + p12.export(passphrase=passwd, iter=2, maciter=3) def test_various_empty_passphrases(self): """ @@ -2777,19 +2549,7 @@ class TestPKCS12: """ passwd = b"Lake Michigan" p12 = self.gen_pkcs12(server_cert_pem, server_key_pem, root_cert_pem) - dumped_p12 = p12.export(maciter=-1, passphrase=passwd, iter=2) - try: - recovered_p12 = load_pkcs12(dumped_p12, passwd) - # The person who generated this PCKS12 should be flogged, - # or better yet we should have a means to determine - # whether a PCKS12 had a MAC that was verified. - # Anyway, libopenssl chooses to allow it, so the - # pyopenssl binding does as well. - assert isinstance(recovered_p12, PKCS12) - except Error: - # Failing here with an exception is preferred as some openssl - # versions do. - pass + p12.export(maciter=-1, passphrase=passwd, iter=2) def test_zero_len_list_for_ca(self): """ @@ -2821,9 +2581,9 @@ class TestPKCS12: p12 = self.gen_pkcs12(server_cert_pem, server_key_pem, root_cert_pem) with pytest.warns(DeprecationWarning) as w: - simplefilter("always") + warnings.simplefilter("always") dumped_p12 = p12.export(passphrase=b"randomtext".decode("ascii")) - msg = "{0} for passphrase is no longer accepted, use bytes".format( + msg = "{} for passphrase is no longer accepted, use bytes".format( WARNING_TYPE_EXPECTED ) assert msg == str(w[-1].message) @@ -2849,7 +2609,7 @@ def _runopenssl(pem, *args): Run the command line openssl tool with the given arguments and write the given PEM to its stdin. Not safe for quotes. """ - proc = Popen([b"openssl"] + list(args), stdin=PIPE, stdout=PIPE) + proc = Popen([b"openssl", *list(args)], stdin=PIPE, stdout=PIPE) proc.stdin.write(pem) proc.stdin.close() output = proc.stdout.read() @@ -3288,36 +3048,6 @@ class TestFunction: with pytest.raises(Error): load_privatekey(FILETYPE_PEM, encrypted_key_pem, passphrase) - def test_load_pkcs7_data_pem(self): - """ - `load_pkcs7_data` accepts a PKCS#7 string and returns an instance of - `PKCS`. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert isinstance(pkcs7, PKCS7) - - def test_load_pkcs7_data_asn1(self): - """ - `load_pkcs7_data` accepts a bytes containing ASN1 data representing - PKCS#7 and returns an instance of `PKCS7`. - """ - pkcs7 = load_pkcs7_data(FILETYPE_ASN1, pkcs7DataASN1) - assert isinstance(pkcs7, PKCS7) - - def test_load_pkcs7_data_invalid(self): - """ - If the data passed to `load_pkcs7_data` is invalid, `Error` is raised. - """ - with pytest.raises(Error): - load_pkcs7_data(FILETYPE_PEM, b"foo") - - def test_load_pkcs7_type_invalid(self): - """ - If the type passed to `load_pkcs7_data`, `ValueError` is raised. - """ - with pytest.raises(ValueError): - load_pkcs7_data(object(), b"foo") - class TestLoadCertificate: """ @@ -3343,61 +3073,6 @@ class TestLoadCertificate: load_certificate(FILETYPE_ASN1, b"lol") -class TestPKCS7: - """ - Tests for `PKCS7`. - """ - - def test_type_is_signed(self): - """ - `PKCS7.type_is_signed` returns `True` if the PKCS7 object is of - the type *signed*. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert pkcs7.type_is_signed() - - def test_type_is_enveloped(self): - """ - `PKCS7.type_is_enveloped` returns `False` if the PKCS7 object is not - of the type *enveloped*. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert not pkcs7.type_is_enveloped() - - def test_type_is_signed_and_enveloped(self): - """ - `PKCS7.type_is_signedAndEnveloped` returns `False` - if the PKCS7 object is not of the type *signed and enveloped*. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert not pkcs7.type_is_signedAndEnveloped() - - def test_type_is_data(self): - """ - `PKCS7.type_is_data` returns `False` if the PKCS7 object is not of - the type data. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert not pkcs7.type_is_data() - - def test_get_type_name(self): - """ - `PKCS7.get_type_name` returns a `str` giving the - type name. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - assert pkcs7.get_type_name() == b"pkcs7-signedData" - - def test_attribute(self): - """ - If an attribute other than one of the methods tested here is accessed - on an instance of `PKCS7`, `AttributeError` is raised. - """ - pkcs7 = load_pkcs7_data(FILETYPE_PEM, pkcs7Data) - with pytest.raises(AttributeError): - pkcs7.foo - - class TestNetscapeSPKI(_PKeyInteractionTestsMixin): """ Tests for `OpenSSL.crypto.NetscapeSPKI`. @@ -3676,8 +3351,8 @@ class TestCRL: not emit a deprecation warning. """ crl = self._get_crl() - with pytest.warns(None) as catcher: - simplefilter("always") + with warnings.catch_warnings(record=True) as catcher: + warnings.simplefilter("always") assert 0 == len(catcher) dumped_crl = crl.export(self.cert, self.pkey, digest=b"md5") text = _runopenssl(dumped_crl, b"crl", b"-noout", b"-text") @@ -3830,7 +3505,8 @@ class TestCRL: buf = dump_crl(FILETYPE_PEM, crl) assert buf == crlData - def _make_test_crl(self, issuer_cert, issuer_key, certs=()): + @staticmethod + def _make_test_crl(issuer_cert, issuer_key, certs=()): """ Create a CRL. @@ -3856,7 +3532,55 @@ class TestCRL: crl.sign(issuer_cert, issuer_key, digest=b"sha512") return crl - def test_verify_with_revoked(self): + @staticmethod + def _make_test_crl_cryptography(issuer_cert, issuer_key, certs=()): + """ + Create a CRL using cryptography's API. + + :param list[X509] certs: A list of certificates to revoke. + :rtype: ``cryptography.x509.CertificateRevocationList`` + """ + from cryptography.x509.extensions import CRLReason, ReasonFlags + + builder = x509.CertificateRevocationListBuilder() + builder = builder.issuer_name( + X509.to_cryptography(issuer_cert).subject + ) + for cert in certs: + revoked = ( + x509.RevokedCertificateBuilder() + .serial_number(cert.get_serial_number()) + .revocation_date(datetime(2014, 6, 1, 0, 0, 0)) + .add_extension(CRLReason(ReasonFlags.unspecified), False) + .build() + ) + builder = builder.add_revoked_certificate(revoked) + + builder = builder.last_update(datetime(2014, 6, 1, 0, 0, 0)) + # The year 5000 is far into the future so that this CRL isn't + # considered to have expired. + builder = builder.next_update(datetime(5000, 6, 1, 0, 0, 0)) + + crl = builder.sign( + private_key=PKey.to_cryptography_key(issuer_key), + algorithm=hashes.SHA512(), + ) + return crl + + @pytest.mark.parametrize( + "create_crl", + [ + pytest.param( + _make_test_crl.__func__, + id="pyOpenSSL CRL", + ), + pytest.param( + _make_test_crl_cryptography.__func__, + id="cryptography CRL", + ), + ], + ) + def test_verify_with_revoked(self, create_crl): """ `verify_certificate` raises error when an intermediate certificate is revoked. @@ -3864,10 +3588,10 @@ class TestCRL: store = X509Store() store.add_cert(self.root_cert) store.add_cert(self.intermediate_cert) - root_crl = self._make_test_crl( + root_crl = create_crl( self.root_cert, self.root_key, certs=[self.intermediate_cert] ) - intermediate_crl = self._make_test_crl( + intermediate_crl = create_crl( self.intermediate_cert, self.intermediate_key, certs=[] ) store.add_crl(root_crl) @@ -3880,7 +3604,20 @@ class TestCRL: store_ctx.verify_certificate() assert str(err.value) == "certificate revoked" - def test_verify_with_missing_crl(self): + @pytest.mark.parametrize( + "create_crl", + [ + pytest.param( + _make_test_crl.__func__, + id="pyOpenSSL CRL", + ), + pytest.param( + _make_test_crl_cryptography.__func__, + id="cryptography CRL", + ), + ], + ) + def test_verify_with_missing_crl(self, create_crl): """ `verify_certificate` raises error when an intermediate certificate's CRL is missing. @@ -3888,7 +3625,7 @@ class TestCRL: store = X509Store() store.add_cert(self.root_cert) store.add_cert(self.intermediate_cert) - root_crl = self._make_test_crl( + root_crl = create_crl( self.root_cert, self.root_key, certs=[self.intermediate_cert] ) store.add_crl(root_crl) @@ -4235,7 +3972,7 @@ class TestX509StoreContext: @staticmethod def _create_ca_file(base_path, hash_directory, cacert): - ca_hash = "{:08x}.0".format(cacert.subject_name_hash()) + ca_hash = f"{cacert.subject_name_hash():08x}.0" cafile = base_path.join(hash_directory, ca_hash) cafile.write_binary( dump_certificate(FILETYPE_PEM, cacert), ensure=True @@ -4366,16 +4103,16 @@ class TestSignVerify: cert = load_certificate(FILETYPE_PEM, root_cert_pem) for digest in ["md5", "sha1", "sha256"]: with pytest.warns(DeprecationWarning) as w: - simplefilter("always") + warnings.simplefilter("always") sig = sign(priv_key, content, digest) - assert "{0} for data is no longer accepted, use bytes".format( + assert "{} for data is no longer accepted, use bytes".format( WARNING_TYPE_EXPECTED ) == str(w[-1].message) with pytest.warns(DeprecationWarning) as w: - simplefilter("always") + warnings.simplefilter("always") verify(cert, sig, content, digest) - assert "{0} for data is no longer accepted, use bytes".format( + assert "{} for data is no longer accepted, use bytes".format( WARNING_TYPE_EXPECTED ) == str(w[-1].message) @@ -4459,7 +4196,7 @@ class TestEllipticCurve: """ curves = get_elliptic_curves() curve = next(iter(curves)) - assert "<Curve %r>" % (curve.name,) == repr(curve) + assert f"<Curve {curve.name!r}>" == repr(curve) def test_to_EC_KEY(self): """ diff --git a/contrib/python/pyOpenSSL/py3/tests/test_ssl.py b/contrib/python/pyOpenSSL/py3/tests/test_ssl.py index fdedd7133a0..b5305eb9200 100644 --- a/contrib/python/pyOpenSSL/py3/tests/test_ssl.py +++ b/contrib/python/pyOpenSSL/py3/tests/test_ssl.py @@ -9,6 +9,7 @@ import datetime import gc import select import sys +import time import uuid from errno import ( EAFNOSUPPORT, @@ -26,42 +27,50 @@ from socket import ( AF_INET6, MSG_PEEK, SHUT_RDWR, - error, + gaierror, socket, ) from sys import getfilesystemencoding, platform from typing import Union -from warnings import simplefilter from weakref import ref +import flaky +import pytest from cryptography import x509 -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID - -import flaky - from pretend import raiser -import pytest - from OpenSSL import SSL +from OpenSSL._util import ffi as _ffi +from OpenSSL._util import lib as _lib +from OpenSSL.crypto import ( + FILETYPE_PEM, + TYPE_RSA, + X509, + PKey, + X509Store, + dump_certificate, + dump_privatekey, + get_elliptic_curves, + load_certificate, + load_privatekey, +) + +with pytest.warns(DeprecationWarning): + from OpenSSL.crypto import X509Extension + from OpenSSL.SSL import ( - Connection, - Context, DTLS_METHOD, - Error, MODE_RELEASE_BUFFERS, NO_OVERLAPPING_PROTOCOLS, - OPENSSL_VERSION_NUMBER, OP_COOKIE_EXCHANGE, OP_NO_COMPRESSION, OP_NO_QUERY_MTU, - OP_NO_SSLv2, - OP_NO_SSLv3, OP_NO_TICKET, OP_SINGLE_DH_USE, + OPENSSL_VERSION_NUMBER, RECEIVED_SHUTDOWN, SENT_SHUTDOWN, SESS_CACHE_BOTH, @@ -72,11 +81,6 @@ from OpenSSL.SSL import ( SESS_CACHE_NO_INTERNAL_STORE, SESS_CACHE_OFF, SESS_CACHE_SERVER, - SSLEAY_BUILT_ON, - SSLEAY_CFLAGS, - SSLEAY_DIR, - SSLEAY_PLATFORM, - SSLEAY_VERSION, SSL_CB_ACCEPT_EXIT, SSL_CB_ACCEPT_LOOP, SSL_CB_ALERT, @@ -93,45 +97,41 @@ from OpenSSL.SSL import ( SSL_ST_ACCEPT, SSL_ST_CONNECT, SSL_ST_MASK, - SSLeay_version, - SSLv23_METHOD, - Session, - SysCallError, + SSLEAY_BUILT_ON, + SSLEAY_CFLAGS, + SSLEAY_DIR, + SSLEAY_PLATFORM, + SSLEAY_VERSION, TLS1_1_VERSION, TLS1_2_VERSION, TLS1_3_VERSION, TLS_METHOD, - TLSv1_1_METHOD, - TLSv1_2_METHOD, - TLSv1_METHOD, VERIFY_CLIENT_ONCE, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_NONE, VERIFY_PEER, + Connection, + Context, + Error, + OP_NO_SSLv2, + OP_NO_SSLv3, + Session, + SSLeay_version, + SSLv23_METHOD, + SysCallError, + TLSv1_1_METHOD, + TLSv1_2_METHOD, + TLSv1_METHOD, WantReadError, WantWriteError, ZeroReturnError, _make_requires, ) -from OpenSSL._util import ffi as _ffi, lib as _lib -from OpenSSL.crypto import ( - FILETYPE_PEM, - PKey, - TYPE_RSA, - X509, - X509Extension, - X509Store, - dump_certificate, - dump_privatekey, - get_elliptic_curves, - load_certificate, - load_privatekey, -) try: from OpenSSL.SSL import ( - SSL_ST_INIT, SSL_ST_BEFORE, + SSL_ST_INIT, SSL_ST_OK, SSL_ST_RENEGOTIATE, ) @@ -153,7 +153,6 @@ from .test_crypto import ( ) from .util import NON_ASCII, WARNING_TYPE_EXPECTED, is_consistent_type - # openssl dhparam 2048 -out dh-2048.pem dhparam = """\ -----BEGIN DH PARAMETERS----- @@ -170,7 +169,7 @@ i5s5yYK7a/0eWxxRr2qraYaUj8RwDpH9CwIBAg== def socket_any_family(): try: return socket(AF_INET) - except error as e: + except OSError as e: if e.errno == EAFNOSUPPORT: return socket(AF_INET6) raise @@ -359,11 +358,10 @@ def interact_in_memory(client_conn, server_conn): # Copy stuff from each side's send buffer to the other side's # receive buffer. - for (read, write) in [ + for read, write in [ (client_conn, server_conn), (server_conn, client_conn), ]: - # Give the side a chance to generate some more bytes, or succeed. try: data = read.recv(2**16) @@ -642,7 +640,7 @@ class TestContext: key = PKey() key.generate_key(TYPE_RSA, 1024) - with open(pemfile, "wt") as pem: + with open(pemfile, "w") as pem: pem.write(dump_privatekey(FILETYPE_PEM, key).decode("ascii")) ctx = Context(SSLv23_METHOD) @@ -1140,23 +1138,30 @@ class TestContext: self._load_verify_locations_test(None, capath) - def test_load_verify_directory_bytes_capath(self, tmpfile): + @pytest.mark.parametrize( + "pathtype", + [ + "ascii_path", + pytest.param( + "unicode_path", + marks=pytest.mark.skipif( + platform == "win32", + reason="Unicode paths not supported on Windows", + ), + ), + ], + ) + @pytest.mark.parametrize("argtype", ["bytes_arg", "unicode_arg"]) + def test_load_verify_directory_capath(self, pathtype, argtype, tmpfile): """ `Context.load_verify_locations` accepts a directory name as a `bytes` instance and uses the certificates within for verification purposes. """ - self._load_verify_directory_locations_capath( - tmpfile + NON_ASCII.encode(getfilesystemencoding()) - ) - - def test_load_verify_directory_unicode_capath(self, tmpfile): - """ - `Context.load_verify_locations` accepts a directory name as a `unicode` - instance and uses the certificates within for verification purposes. - """ - self._load_verify_directory_locations_capath( - tmpfile.decode(getfilesystemencoding()) + NON_ASCII - ) + if pathtype == "unicode_path": + tmpfile += NON_ASCII.encode(getfilesystemencoding()) + if argtype == "unicode_arg": + tmpfile = tmpfile.decode(getfilesystemencoding()) + self._load_verify_directory_locations_capath(tmpfile) def test_load_verify_locations_wrong_args(self): """ @@ -1172,7 +1177,7 @@ class TestContext: @pytest.mark.skipif( not platform.startswith("linux"), reason="Loading fallback paths is a linux-specific behavior to " - "accommodate pyca/cryptography manylinux1 wheels", + "accommodate pyca/cryptography manylinux wheels", ) def test_fallback_default_verify_paths(self, monkeypatch): """ @@ -1265,7 +1270,10 @@ class TestContext: ) client = socket_any_family() - client.connect(("encrypted.google.com", 443)) + try: + client.connect(("encrypted.google.com", 443)) + except gaierror: + pytest.skip("cannot connect to encrypted.google.com") clientSSL = Connection(context, client) clientSSL.set_connect_state() clientSSL.set_tlsext_host_name(b"encrypted.google.com") @@ -1717,7 +1725,7 @@ class TestContext: """ context = Context(SSLv23_METHOD) with pytest.raises(TypeError): - context.set_tlsext_use_srtp(str("SRTP_AES128_CM_SHA1_80")) + context.set_tlsext_use_srtp("SRTP_AES128_CM_SHA1_80") def test_set_tlsext_use_srtp_invalid_profile(self): """ @@ -1776,7 +1784,7 @@ class TestServerNameCallback: if callback is not None: referrers = get_referrers(callback) if len(referrers) > 1: # pragma: nocover - pytest.fail("Some references remain: %r" % (referrers,)) + pytest.fail(f"Some references remain: {referrers!r}") def test_no_servername(self): """ @@ -2364,7 +2372,7 @@ class TestConnection: # 2.6: https://github.com/pytest-dev/pytest/issues/988 try: clientSSL.connect((loopback_address(client), 1)) - except error as e: + except OSError as e: exc = e assert exc.args[0] == ECONNREFUSED @@ -2380,10 +2388,6 @@ class TestConnection: clientSSL.connect((loopback_address(port), port.getsockname()[1])) # XXX An assertion? Or something? - @pytest.mark.skipif( - platform == "darwin", - reason="connect_ex sometimes causes a kernel panic on OS X 10.6.4", - ) def test_connect_ex(self): """ If there is a connection error, `Connection.connect_ex` returns the @@ -2706,7 +2710,7 @@ class TestConnection: if callback is not None: # pragma: nocover referrers = get_referrers(callback) if len(referrers) > 1: - pytest.fail("Some references remain: %r" % (referrers,)) + pytest.fail(f"Some references remain: {referrers!r}") def test_get_session_unconnected(self): """ @@ -2762,7 +2766,7 @@ class TestConnection: ctx = Context(TLSv1_2_METHOD) ctx.use_privatekey(key) ctx.use_certificate(cert) - ctx.set_session_id("unity-test") + ctx.set_session_id(b"unity-test") def makeServer(socket): server = Connection(ctx, socket) @@ -2838,23 +2842,24 @@ class TestConnection: """ client_socket, server_socket = socket_pair() # Fill up the client's send buffer so Connection won't be able to write - # anything. Only write a single byte at a time so we can be sure we + # anything. Start by sending larger chunks (Windows Socket I/O is slow) + # and continue by writing a single byte at a time so we can be sure we # completely fill the buffer. Even though the socket API is allowed to # signal a short write via its return value it seems this doesn't # always happen on all platforms (FreeBSD and OS X particular) for the # very last bit of available buffer space. - msg = b"x" - for i in range(1024 * 1024 * 64): - try: - client_socket.send(msg) - except error as e: - if e.errno == EWOULDBLOCK: - break - raise - else: - pytest.fail( - "Failed to fill socket buffer, cannot test BIO want write" - ) + for msg in [b"x" * 65536, b"x"]: + for i in range(1024 * 1024 * 64): + try: + client_socket.send(msg) + except OSError as e: + if e.errno == EWOULDBLOCK: + break + raise # pragma: no cover + else: # pragma: no cover + pytest.fail( + "Failed to fill socket buffer, cannot test BIO want write" + ) ctx = Context(SSLv23_METHOD) conn = Connection(ctx, client_socket) @@ -3117,9 +3122,8 @@ class TestConnectionSend: """ server, client = loopback() with pytest.warns(DeprecationWarning) as w: - simplefilter("always") count = server.send(b"xy".decode("ascii")) - assert "{0} for buf is no longer accepted, use bytes".format( + assert "{} for buf is no longer accepted, use bytes".format( WARNING_TYPE_EXPECTED ) == str(w[-1].message) assert count == 2 @@ -3325,9 +3329,8 @@ class TestConnectionSendall: """ server, client = loopback() with pytest.warns(DeprecationWarning) as w: - simplefilter("always") server.sendall(b"x".decode("ascii")) - assert "{0} for buf is no longer accepted, use bytes".format( + assert "{} for buf is no longer accepted, use bytes".format( WARNING_TYPE_EXPECTED ) == str(w[-1].message) assert client.recv(1) == b"x" @@ -3753,13 +3756,16 @@ class TestMemoryBIO: """ If the connection is lost before an orderly SSL shutdown occurs, `OpenSSL.SSL.SysCallError` is raised with a message of - "Unexpected EOF". + "Unexpected EOF" (or WSAECONNRESET on Windows). """ server_conn, client_conn = loopback() client_conn.sock_shutdown(SHUT_RDWR) with pytest.raises(SysCallError) as err: server_conn.recv(1024) - assert err.value.args == (-1, "Unexpected EOF") + if platform == "win32": + assert err.value.args == (10054, "WSAECONNRESET") + else: + assert err.value.args == (-1, "Unexpected EOF") def _check_client_ca_list(self, func): """ @@ -4370,10 +4376,11 @@ class TestDTLS: # new versions of OpenSSL, this is unnecessary, but harmless, because the # DTLS state machine treats it like a network hiccup that duplicated a # packet, which DTLS is robust against. - def test_it_works_at_all(self): - # arbitrary number larger than any conceivable handshake volley - LARGE_BUFFER = 65536 + # Arbitrary number larger than any conceivable handshake volley. + LARGE_BUFFER = 65536 + + def test_it_works_at_all(self): s_ctx = Context(DTLS_METHOD) def generate_cookie(ssl): @@ -4404,7 +4411,7 @@ class TestDTLS: def pump_membio(label, source, sink): try: - chunk = source.bio_read(LARGE_BUFFER) + chunk = source.bio_read(self.LARGE_BUFFER) except WantReadError: return False # I'm not sure this check is needed, but I'm not sure it's *not* @@ -4484,3 +4491,39 @@ class TestDTLS: assert 0 < c.get_cleartext_mtu() < 500 except NotImplementedError: # OpenSSL 1.1.0 and earlier pass + + def test_timeout(self, monkeypatch): + c_ctx = Context(DTLS_METHOD) + c = Connection(c_ctx) + + # No timeout before the handshake starts. + assert c.DTLSv1_get_timeout() is None + assert c.DTLSv1_handle_timeout() is False + + # Start handshake and check there is data to send. + c.set_connect_state() + try: + c.do_handshake() + except SSL.WantReadError: + pass + assert c.bio_read(self.LARGE_BUFFER) + + # There should now be an active timeout. + seconds = c.DTLSv1_get_timeout() + assert seconds is not None + + # Handle the timeout and check there is data to send. + time.sleep(seconds) + assert c.DTLSv1_handle_timeout() is True + assert c.bio_read(self.LARGE_BUFFER) + + # After the maximum number of allowed timeouts is reached, + # DTLSv1_handle_timeout will return -1. + # + # Testing this directly is prohibitively time consuming as the timeout + # duration is doubled on each retry, so the best we can do is to mock + # this condition. + monkeypatch.setattr(_lib, "DTLSv1_handle_timeout", lambda x: -1) + + with pytest.raises(Error): + c.DTLSv1_handle_timeout() diff --git a/contrib/python/pyOpenSSL/py3/ya.make b/contrib/python/pyOpenSSL/py3/ya.make index 08bed6fbc0f..28f890402b4 100644 --- a/contrib/python/pyOpenSSL/py3/ya.make +++ b/contrib/python/pyOpenSSL/py3/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() SUBSCRIBER(g:python-contrib) -VERSION(23.0.0) +VERSION(23.3.0) LICENSE(Apache-2.0) |