diff options
author | AlexSm <alex@ydb.tech> | 2024-01-09 18:56:40 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-09 18:56:40 +0100 |
commit | e95f266d2a3e48e62015220588a4fd73d5d5a5cb (patch) | |
tree | a8a784b6931fe52ad5f511cfef85af14e5f63991 /contrib/python | |
parent | 50a65e3b48a82d5b51f272664da389f2e0b0c99a (diff) | |
download | ydb-e95f266d2a3e48e62015220588a4fd73d5d5a5cb.tar.gz |
Library import 6 (#888)
Diffstat (limited to 'contrib/python')
160 files changed, 3114 insertions, 1208 deletions
diff --git a/contrib/python/PyJWT/py3/.dist-info/METADATA b/contrib/python/PyJWT/py3/.dist-info/METADATA index f5fbdf64a5..65b6141fa1 100644 --- a/contrib/python/PyJWT/py3/.dist-info/METADATA +++ b/contrib/python/PyJWT/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: PyJWT -Version: 2.3.0 +Version: 2.4.0 Summary: JSON Web Token implementation in Python Home-page: https://github.com/jpadilla/pyjwt Author: Jose Padilla @@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Topic :: Utilities Requires-Python: >=3.6 Description-Content-Type: text/x-rst diff --git a/contrib/python/PyJWT/py3/LICENSE b/contrib/python/PyJWT/py3/LICENSE index bdc7819ea1..fd0ecbc889 100644 --- a/contrib/python/PyJWT/py3/LICENSE +++ b/contrib/python/PyJWT/py3/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 José Padilla +Copyright (c) 2015-2022 José Padilla Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/contrib/python/PyJWT/py3/jwt/__init__.py b/contrib/python/PyJWT/py3/jwt/__init__.py index 3208c39f39..6b3f8ab160 100644 --- a/contrib/python/PyJWT/py3/jwt/__init__.py +++ b/contrib/python/PyJWT/py3/jwt/__init__.py @@ -25,19 +25,19 @@ from .exceptions import ( ) from .jwks_client import PyJWKClient -__version__ = "2.3.0" +__version__ = "2.4.0" __title__ = "PyJWT" __description__ = "JSON Web Token implementation in Python" __url__ = "https://pyjwt.readthedocs.io" __uri__ = __url__ -__doc__ = __description__ + " <" + __uri__ + ">" +__doc__ = f"{__description__} <{__uri__}>" __author__ = "José Padilla" __email__ = "hello@jpadilla.com" __license__ = "MIT" -__copyright__ = "Copyright 2015-2020 José Padilla" +__copyright__ = "Copyright 2015-2022 José Padilla" __all__ = [ diff --git a/contrib/python/PyJWT/py3/jwt/algorithms.py b/contrib/python/PyJWT/py3/jwt/algorithms.py index 1f8865afbd..46a1a532e7 100644 --- a/contrib/python/PyJWT/py3/jwt/algorithms.py +++ b/contrib/python/PyJWT/py3/jwt/algorithms.py @@ -9,6 +9,8 @@ from .utils import ( der_to_raw_signature, force_bytes, from_base64url_uint, + is_pem_format, + is_ssh_key, raw_to_der_signature, to_base64url_uint, ) @@ -183,14 +185,7 @@ class HMACAlgorithm(Algorithm): def prepare_key(self, key): key = force_bytes(key) - invalid_strings = [ - b"-----BEGIN PUBLIC KEY-----", - b"-----BEGIN CERTIFICATE-----", - b"-----BEGIN RSA PUBLIC KEY-----", - b"ssh-rsa", - ] - - if any(string_value in key for string_value in invalid_strings): + if is_pem_format(key) or is_ssh_key(key): raise InvalidKeyError( "The specified key is an asymmetric key or x509 certificate and" " should not be used as an HMAC secret." @@ -417,6 +412,12 @@ if has_crypto: except ValueError: key = load_pem_private_key(key, password=None) + # Explicit check the key to prevent confusing errors from cryptography + if not isinstance(key, (EllipticCurvePrivateKey, EllipticCurvePublicKey)): + raise InvalidKeyError( + "Expecting a EllipticCurvePrivateKey/EllipticCurvePublicKey. Wrong key provided for ECDSA algorithms" + ) + return key def sign(self, msg, key): @@ -545,26 +546,28 @@ if has_crypto: pass def prepare_key(self, key): - - if isinstance( - key, - (Ed25519PrivateKey, Ed25519PublicKey, Ed448PrivateKey, Ed448PublicKey), - ): - return key - if isinstance(key, (bytes, str)): if isinstance(key, str): key = key.encode("utf-8") str_key = key.decode("utf-8") if "-----BEGIN PUBLIC" in str_key: - return load_pem_public_key(key) - if "-----BEGIN PRIVATE" in str_key: - return load_pem_private_key(key, password=None) - if str_key[0:4] == "ssh-": - return load_ssh_public_key(key) + key = load_pem_public_key(key) + elif "-----BEGIN PRIVATE" in str_key: + key = load_pem_private_key(key, password=None) + elif str_key[0:4] == "ssh-": + key = load_ssh_public_key(key) + + # Explicit check the key to prevent confusing errors from cryptography + if not isinstance( + key, + (Ed25519PrivateKey, Ed25519PublicKey, Ed448PrivateKey, Ed448PublicKey), + ): + raise InvalidKeyError( + "Expecting a EllipticCurvePrivateKey/EllipticCurvePublicKey. Wrong key provided for EdDSA algorithms" + ) - raise TypeError("Expecting a PEM-formatted or OpenSSH key.") + return key def sign(self, msg, key): """ diff --git a/contrib/python/PyJWT/py3/jwt/api_jwk.py b/contrib/python/PyJWT/py3/jwt/api_jwk.py index a0f6364da0..31250d57f2 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jwk.py +++ b/contrib/python/PyJWT/py3/jwt/api_jwk.py @@ -11,7 +11,7 @@ class PyJWK: kty = self._jwk_data.get("kty", None) if not kty: - raise InvalidKeyError("kty is not found: %s" % self._jwk_data) + raise InvalidKeyError(f"kty is not found: {self._jwk_data}") if not algorithm and isinstance(self._jwk_data, dict): algorithm = self._jwk_data.get("alg", None) @@ -29,25 +29,25 @@ class PyJWK: elif crv == "secp256k1": algorithm = "ES256K" else: - raise InvalidKeyError("Unsupported crv: %s" % crv) + raise InvalidKeyError(f"Unsupported crv: {crv}") elif kty == "RSA": algorithm = "RS256" elif kty == "oct": algorithm = "HS256" elif kty == "OKP": if not crv: - raise InvalidKeyError("crv is not found: %s" % self._jwk_data) + raise InvalidKeyError(f"crv is not found: {self._jwk_data}") if crv == "Ed25519": algorithm = "EdDSA" else: - raise InvalidKeyError("Unsupported crv: %s" % crv) + raise InvalidKeyError(f"Unsupported crv: {crv}") else: - raise InvalidKeyError("Unsupported kty: %s" % kty) + raise InvalidKeyError(f"Unsupported kty: {kty}") self.Algorithm = self._algorithms.get(algorithm) if not self.Algorithm: - raise PyJWKError("Unable to find a algorithm for key: %s" % self._jwk_data) + raise PyJWKError(f"Unable to find a algorithm for key: {self._jwk_data}") self.key = self.Algorithm.from_jwk(self._jwk_data) @@ -95,3 +95,9 @@ class PyJWKSet: def from_json(data): obj = json.loads(data) return PyJWKSet.from_dict(obj) + + def __getitem__(self, kid): + for key in self.keys: + if key.key_id == kid: + return key + raise KeyError(f"keyset has no key for kid: {kid}") diff --git a/contrib/python/PyJWT/py3/jwt/api_jws.py b/contrib/python/PyJWT/py3/jwt/api_jws.py index f85072e05e..cbf4f6f5b8 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jws.py +++ b/contrib/python/PyJWT/py3/jwt/api_jws.py @@ -80,34 +80,54 @@ class PyJWS: algorithm: Optional[str] = "HS256", headers: Optional[Dict] = None, json_encoder: Optional[Type[json.JSONEncoder]] = None, + is_payload_detached: bool = False, ) -> str: segments = [] if algorithm is None: algorithm = "none" - # Prefer headers["alg"] if present to algorithm parameter. - if headers and "alg" in headers and headers["alg"]: - algorithm = headers["alg"] + # Prefer headers values if present to function parameters. + if headers: + headers_alg = headers.get("alg") + if headers_alg: + algorithm = headers["alg"] + + headers_b64 = headers.get("b64") + if headers_b64 is False: + is_payload_detached = True # Header - header = {"typ": self.header_typ, "alg": algorithm} + header = {"typ": self.header_typ, "alg": algorithm} # type: Dict[str, Any] if headers: self._validate_headers(headers) header.update(headers) - if not header["typ"]: - del header["typ"] + + if not header["typ"]: + del header["typ"] + + if is_payload_detached: + header["b64"] = False + elif "b64" in header: + # True is the standard value for b64, so no need for it + del header["b64"] json_header = json.dumps( header, separators=(",", ":"), cls=json_encoder ).encode() segments.append(base64url_encode(json_header)) - segments.append(base64url_encode(payload)) + + if is_payload_detached: + msg_payload = payload + else: + msg_payload = base64url_encode(payload) + segments.append(msg_payload) # Segments signing_input = b".".join(segments) + try: alg_obj = self._algorithms[algorithm] key = alg_obj.prepare_key(key) @@ -116,14 +136,15 @@ class PyJWS: except KeyError as e: if not has_crypto and algorithm in requires_cryptography: raise NotImplementedError( - "Algorithm '%s' could not be found. Do you have cryptography " - "installed?" % algorithm + f"Algorithm '{algorithm}' could not be found. Do you have cryptography installed?" ) from e - else: - raise NotImplementedError("Algorithm not supported") from e + raise NotImplementedError("Algorithm not supported") from e segments.append(base64url_encode(signature)) + # Don't put the payload content inside the encoded token when detached + if is_payload_detached: + segments[1] = b"" encoded_string = b".".join(segments) return encoded_string.decode("utf-8") @@ -132,8 +153,9 @@ class PyJWS: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, + detached_payload: Optional[bytes] = None, **kwargs, ) -> Dict[str, Any]: if options is None: @@ -148,6 +170,14 @@ class PyJWS: payload, signing_input, header, signature = self._load(jwt) + if header.get("b64", True) is False: + if detached_payload is None: + raise DecodeError( + 'It is required that you pass in a value for the "detached_payload" argument to decode a message having the b64 header set to false.' + ) + payload = detached_payload + signing_input = b".".join([signing_input.rsplit(b".", 1)[0], payload]) + if verify_signature: self._verify_signature(signing_input, header, signature, key, algorithms) @@ -161,8 +191,8 @@ class PyJWS: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> str: decoded = self.decode_complete(jwt, key, algorithms, options, **kwargs) @@ -200,7 +230,7 @@ class PyJWS: try: header = json.loads(header_data) except ValueError as e: - raise DecodeError("Invalid header string: %s" % e) from e + raise DecodeError(f"Invalid header string: {e}") from e if not isinstance(header, Mapping): raise DecodeError("Invalid header string: must be a json object") diff --git a/contrib/python/PyJWT/py3/jwt/api_jwt.py b/contrib/python/PyJWT/py3/jwt/api_jwt.py index f3b55d360e..7d2177bf53 100644 --- a/contrib/python/PyJWT/py3/jwt/api_jwt.py +++ b/contrib/python/PyJWT/py3/jwt/api_jwt.py @@ -1,4 +1,5 @@ import json +import warnings from calendar import timegm from collections.abc import Iterable, Mapping from datetime import datetime, timedelta, timezone @@ -66,14 +67,23 @@ class PyJWT: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> Dict[str, Any]: - if options is None: - options = {"verify_signature": True} - else: - options.setdefault("verify_signature", True) + options = dict(options or {}) # shallow-copy or initialize an empty dict + options.setdefault("verify_signature", True) + + # If the user has set the legacy `verify` argument, and it doesn't match + # what the relevant `options` entry for the argument is, inform the user + # that they're likely making a mistake. + if "verify" in kwargs and kwargs["verify"] != options["verify_signature"]: + warnings.warn( + "The `verify` argument to `decode` does nothing in PyJWT 2.0 and newer. " + "The equivalent is setting `verify_signature` to False in the `options` dictionary. " + "This invocation has a mismatch between the kwarg and the option entry.", + category=DeprecationWarning, + ) if not options["verify_signature"]: options.setdefault("verify_exp", False) @@ -98,7 +108,7 @@ class PyJWT: try: payload = json.loads(decoded["payload"]) except ValueError as e: - raise DecodeError("Invalid payload string: %s" % e) + raise DecodeError(f"Invalid payload string: {e}") if not isinstance(payload, dict): raise DecodeError("Invalid payload string: must be a json object") @@ -112,8 +122,8 @@ class PyJWT: self, jwt: str, key: str = "", - algorithms: List[str] = None, - options: Dict = None, + algorithms: Optional[List[str]] = None, + options: Optional[Dict] = None, **kwargs, ) -> Dict[str, Any]: decoded = self.decode_complete(jwt, key, algorithms, options, **kwargs) diff --git a/contrib/python/PyJWT/py3/jwt/exceptions.py b/contrib/python/PyJWT/py3/jwt/exceptions.py index 308899aa6a..ee201add1a 100644 --- a/contrib/python/PyJWT/py3/jwt/exceptions.py +++ b/contrib/python/PyJWT/py3/jwt/exceptions.py @@ -51,7 +51,7 @@ class MissingRequiredClaimError(InvalidTokenError): self.claim = claim def __str__(self): - return 'Token is missing the "%s" claim' % self.claim + return f'Token is missing the "{self.claim}" claim' class PyJWKError(PyJWTError): diff --git a/contrib/python/PyJWT/py3/jwt/help.py b/contrib/python/PyJWT/py3/jwt/help.py index d8f2302421..d5c3ebbfff 100644 --- a/contrib/python/PyJWT/py3/jwt/help.py +++ b/contrib/python/PyJWT/py3/jwt/help.py @@ -28,10 +28,10 @@ def info(): if implementation == "CPython": implementation_version = platform.python_version() elif implementation == "PyPy": - implementation_version = "{}.{}.{}".format( - sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro, + implementation_version = ( + f"{sys.pypy_version_info.major}." + f"{sys.pypy_version_info.minor}." + f"{sys.pypy_version_info.micro}" ) if sys.pypy_version_info.releaselevel != "final": implementation_version = "".join( diff --git a/contrib/python/PyJWT/py3/jwt/utils.py b/contrib/python/PyJWT/py3/jwt/utils.py index 9dde10cf8e..8ab73b42d3 100644 --- a/contrib/python/PyJWT/py3/jwt/utils.py +++ b/contrib/python/PyJWT/py3/jwt/utils.py @@ -1,5 +1,6 @@ import base64 import binascii +import re from typing import Any, Union try: @@ -97,3 +98,63 @@ def raw_to_der_signature(raw_sig: bytes, curve: EllipticCurve) -> bytes: s = bytes_to_number(raw_sig[num_bytes:]) return encode_dss_signature(r, s) + + +# Based on https://github.com/hynek/pem/blob/7ad94db26b0bc21d10953f5dbad3acfdfacf57aa/src/pem/_core.py#L224-L252 +_PEMS = { + b"CERTIFICATE", + b"TRUSTED CERTIFICATE", + b"PRIVATE KEY", + b"PUBLIC KEY", + b"ENCRYPTED PRIVATE KEY", + b"OPENSSH PRIVATE KEY", + b"DSA PRIVATE KEY", + b"RSA PRIVATE KEY", + b"RSA PUBLIC KEY", + b"EC PRIVATE KEY", + b"DH PARAMETERS", + b"NEW CERTIFICATE REQUEST", + b"CERTIFICATE REQUEST", + b"SSH2 PUBLIC KEY", + b"SSH2 ENCRYPTED PRIVATE KEY", + b"X509 CRL", +} + +_PEM_RE = re.compile( + b"----[- ]BEGIN (" + + b"|".join(_PEMS) + + b""")[- ]----\r? +.+?\r? +----[- ]END \\1[- ]----\r?\n?""", + re.DOTALL, +) + + +def is_pem_format(key: bytes) -> bool: + return bool(_PEM_RE.search(key)) + + +# Based on https://github.com/pyca/cryptography/blob/bcb70852d577b3f490f015378c75cba74986297b/src/cryptography/hazmat/primitives/serialization/ssh.py#L40-L46 +_CERT_SUFFIX = b"-cert-v01@openssh.com" +_SSH_PUBKEY_RC = re.compile(br"\A(\S+)[ \t]+(\S+)") +_SSH_KEY_FORMATS = [ + b"ssh-ed25519", + b"ssh-rsa", + b"ssh-dss", + b"ecdsa-sha2-nistp256", + b"ecdsa-sha2-nistp384", + b"ecdsa-sha2-nistp521", +] + + +def is_ssh_key(key: bytes) -> bool: + if any(string_value in key for string_value in _SSH_KEY_FORMATS): + return True + + ssh_pubkey_match = _SSH_PUBKEY_RC.match(key) + if ssh_pubkey_match: + key_type = ssh_pubkey_match.group(1) + if _CERT_SUFFIX == key_type[-len(_CERT_SUFFIX) :]: + return True + + return False diff --git a/contrib/python/PyJWT/py3/ya.make b/contrib/python/PyJWT/py3/ya.make index 0cbee2bb2e..3be98b643e 100644 --- a/contrib/python/PyJWT/py3/ya.make +++ b/contrib/python/PyJWT/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.3.0) +VERSION(2.4.0) LICENSE(MIT) diff --git a/contrib/python/contextlib2/py3/contextlib2/__init__.py b/contrib/python/contextlib2/py3/contextlib2/__init__.py index d6c0c4ac4a..63ecced65a 100644 --- a/contrib/python/contextlib2/py3/contextlib2/__init__.py +++ b/contrib/python/contextlib2/py3/contextlib2/__init__.py @@ -177,8 +177,10 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, # Need to force instantiation so we can reliably # tell if we get the same exception back value = type() + if traceback is not None: + value = value.with_traceback(traceback) try: - self.gen.throw(type, value, traceback) + self.gen.throw(value) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration diff --git a/contrib/python/fonttools/.dist-info/METADATA b/contrib/python/fonttools/.dist-info/METADATA index 9026fc11f8..f156c42697 100644 --- a/contrib/python/fonttools/.dist-info/METADATA +++ b/contrib/python/fonttools/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: fonttools -Version: 4.46.0 +Version: 4.47.0 Summary: Tools to manipulate font files Home-page: http://github.com/fonttools/fonttools Author: Just van Rossum @@ -34,6 +34,7 @@ Requires-Dist: fs <3,>=2.2.0 ; extra == 'all' Requires-Dist: lxml <5,>=4.0 ; extra == 'all' Requires-Dist: zopfli >=0.1.4 ; extra == 'all' Requires-Dist: lz4 >=1.7.4.2 ; extra == 'all' +Requires-Dist: pycairo ; extra == 'all' Requires-Dist: matplotlib ; extra == 'all' Requires-Dist: sympy ; extra == 'all' Requires-Dist: skia-pathops >=0.5.0 ; extra == 'all' @@ -47,6 +48,7 @@ Requires-Dist: xattr ; (sys_platform == "darwin") and extra == 'all' Provides-Extra: graphite Requires-Dist: lz4 >=1.7.4.2 ; extra == 'graphite' Provides-Extra: interpolatable +Requires-Dist: pycairo ; extra == 'interpolatable' Requires-Dist: scipy ; (platform_python_implementation != "PyPy") and extra == 'interpolatable' Requires-Dist: munkres ; (platform_python_implementation == "PyPy") and extra == 'interpolatable' Provides-Extra: lxml @@ -210,6 +212,13 @@ are required to unlock the extra features named "ufo", etc. * `munkres <https://pypi.python.org/pypi/munkres>`__: a pure-Python module that implements the Hungarian or Kuhn-Munkres algorithm. + To plot the results to a PDF or HTML format, you also need to install: + + * `pycairo <https://pypi.org/project/pycairo/>`__: Python bindings for the + Cairo graphics library. Note that wheels are currently only available for + Windows, for other platforms see pycairo's `installation instructions + <https://pycairo.readthedocs.io/en/latest/getting_started.html>`__. + *Extra:* ``interpolatable`` - ``Lib/fontTools/varLib/plot.py`` @@ -366,6 +375,16 @@ Have fun! Changelog ~~~~~~~~~ +4.47.0 (released 2023-12-18) +---------------------------- + +- [varLib.models] New API for VariationModel: ``getMasterScalars`` and + ``interpolateFromValuesAndScalars``. +- [varLib.interpolatable] Various bugfixes and rendering improvements. In particular, + add a Summary page in the front, and an Index and Table-of-Contents in the back. + Change the page size to Letter. +- [Docs/designspaceLib] Defined a new ``public.fontInfo`` lib key, not used anywhere yet (#3358). + 4.46.0 (released 2023-12-02) ---------------------------- diff --git a/contrib/python/fonttools/README.rst b/contrib/python/fonttools/README.rst index bcb7f0d47f..d84282fc76 100644 --- a/contrib/python/fonttools/README.rst +++ b/contrib/python/fonttools/README.rst @@ -138,6 +138,13 @@ are required to unlock the extra features named "ufo", etc. * `munkres <https://pypi.python.org/pypi/munkres>`__: a pure-Python module that implements the Hungarian or Kuhn-Munkres algorithm. + To plot the results to a PDF or HTML format, you also need to install: + + * `pycairo <https://pypi.org/project/pycairo/>`__: Python bindings for the + Cairo graphics library. Note that wheels are currently only available for + Windows, for other platforms see pycairo's `installation instructions + <https://pycairo.readthedocs.io/en/latest/getting_started.html>`__. + *Extra:* ``interpolatable`` - ``Lib/fontTools/varLib/plot.py`` diff --git a/contrib/python/fonttools/fontTools/__init__.py b/contrib/python/fonttools/fontTools/__init__.py index dfe589402f..6c00e567a4 100644 --- a/contrib/python/fonttools/fontTools/__init__.py +++ b/contrib/python/fonttools/fontTools/__init__.py @@ -3,6 +3,6 @@ from fontTools.misc.loggingTools import configLogger log = logging.getLogger(__name__) -version = __version__ = "4.46.0" +version = __version__ = "4.47.0" __all__ = ["version", "log", "configLogger"] diff --git a/contrib/python/fonttools/fontTools/afmLib.py b/contrib/python/fonttools/fontTools/afmLib.py index 935a1e8e0d..e89646951c 100644 --- a/contrib/python/fonttools/fontTools/afmLib.py +++ b/contrib/python/fonttools/fontTools/afmLib.py @@ -82,7 +82,10 @@ kernRE = re.compile( # regular expressions to parse composite info lines of the form: # Aacute 2 ; PCC A 0 0 ; PCC acute 182 211 ; compositeRE = re.compile( - r"([.A-Za-z0-9_]+)" r"\s+" r"(\d+)" r"\s*;\s*" # char name # number of parts + r"([.A-Za-z0-9_]+)" # char name + r"\s+" + r"(\d+)" # number of parts + r"\s*;\s*" ) componentRE = re.compile( r"PCC\s+" # PPC diff --git a/contrib/python/fonttools/fontTools/designspaceLib/__init__.py b/contrib/python/fonttools/fontTools/designspaceLib/__init__.py index 1c71fd002e..69d4912c09 100644 --- a/contrib/python/fonttools/fontTools/designspaceLib/__init__.py +++ b/contrib/python/fonttools/fontTools/designspaceLib/__init__.py @@ -312,7 +312,7 @@ class SourceDescriptor(SimpleDescriptor): return self.designLocation @location.setter - def location(self, location: Optional[AnisotropicLocationDict]): + def location(self, location: Optional[SimpleLocationDict]): self.designLocation = location or {} def setFamilyName(self, familyName, languageCode="en"): @@ -329,15 +329,13 @@ class SourceDescriptor(SimpleDescriptor): """ return self.localisedFamilyName.get(languageCode) - def getFullDesignLocation( - self, doc: "DesignSpaceDocument" - ) -> AnisotropicLocationDict: + def getFullDesignLocation(self, doc: "DesignSpaceDocument") -> SimpleLocationDict: """Get the complete design location of this source, from its :attr:`designLocation` and the document's axis defaults. .. versionadded:: 5.0 """ - result: AnisotropicLocationDict = {} + result: SimpleLocationDict = {} for axis in doc.axes: if axis.name in self.designLocation: result[axis.name] = self.designLocation[axis.name] diff --git a/contrib/python/fonttools/fontTools/pens/recordingPen.py b/contrib/python/fonttools/fontTools/pens/recordingPen.py index 2ed8d32ec7..e24b65265a 100644 --- a/contrib/python/fonttools/fontTools/pens/recordingPen.py +++ b/contrib/python/fonttools/fontTools/pens/recordingPen.py @@ -8,6 +8,7 @@ __all__ = [ "RecordingPen", "DecomposingRecordingPen", "RecordingPointPen", + "lerpRecordings", ] @@ -172,6 +173,34 @@ class RecordingPointPen(AbstractPointPen): drawPoints = replay +def lerpRecordings(recording1, recording2, factor=0.5): + """Linearly interpolate between two recordings. The recordings + must be decomposed, i.e. they must not contain any components. + + Factor is typically between 0 and 1. 0 means the first recording, + 1 means the second recording, and 0.5 means the average of the + two recordings. Other values are possible, and can be useful to + extrapolate. Defaults to 0.5. + + Returns a generator with the new recording. + """ + if len(recording1) != len(recording2): + raise ValueError( + "Mismatched lengths: %d and %d" % (len(recording1), len(recording2)) + ) + for (op1, args1), (op2, args2) in zip(recording1, recording2): + if op1 != op2: + raise ValueError("Mismatched operations: %s, %s" % (op1, op2)) + if op1 == "addComponent": + raise ValueError("Cannot interpolate components") + else: + mid_args = [ + (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) + for (x1, y1), (x2, y2) in zip(args1, args2) + ] + yield (op1, mid_args) + + if __name__ == "__main__": pen = RecordingPen() pen.moveTo((0, 0)) diff --git a/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py b/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py index 349cc2c73f..5d188d6a10 100644 --- a/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py +++ b/contrib/python/fonttools/fontTools/ttLib/ttGlyphSet.py @@ -9,6 +9,11 @@ from fontTools.misc.fixedTools import otRound from fontTools.misc.loggingTools import deprecateFunction from fontTools.misc.transform import Transform from fontTools.pens.transformPen import TransformPen, TransformPointPen +from fontTools.pens.recordingPen import ( + DecomposingRecordingPen, + lerpRecordings, + replayRecording, +) class _TTGlyphSet(Mapping): @@ -321,3 +326,52 @@ def _setCoordinates(glyph, coord, glyfTable, *, recalcBounds=True): verticalAdvanceWidth, topSideBearing, ) + + +class LerpGlyphSet(Mapping): + """A glyphset that interpolates between two other glyphsets. + + Factor is typically between 0 and 1. 0 means the first glyphset, + 1 means the second glyphset, and 0.5 means the average of the + two glyphsets. Other values are possible, and can be useful to + extrapolate. Defaults to 0.5. + """ + + def __init__(self, glyphset1, glyphset2, factor=0.5): + self.glyphset1 = glyphset1 + self.glyphset2 = glyphset2 + self.factor = factor + + def __getitem__(self, glyphname): + if glyphname in self.glyphset1 and glyphname in self.glyphset2: + return LerpGlyph(glyphname, self) + raise KeyError(glyphname) + + def __contains__(self, glyphname): + return glyphname in self.glyphset1 and glyphname in self.glyphset2 + + def __iter__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return iter(set1.intersection(set2)) + + def __len__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return len(set1.intersection(set2)) + + +class LerpGlyph: + def __init__(self, glyphname, glyphset): + self.glyphset = glyphset + self.glyphname = glyphname + + def draw(self, pen): + recording1 = DecomposingRecordingPen(self.glyphset.glyphset1) + self.glyphset.glyphset1[self.glyphname].draw(recording1) + recording2 = DecomposingRecordingPen(self.glyphset.glyphset2) + self.glyphset.glyphset2[self.glyphname].draw(recording2) + + factor = self.glyphset.factor + + replayRecording(lerpRecordings(recording1.value, recording2.value, factor), pen) diff --git a/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py b/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py index a887e5d38f..d1cde0df7a 100644 --- a/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py +++ b/contrib/python/fonttools/fontTools/varLib/instancer/__init__.py @@ -1433,7 +1433,7 @@ def parseArgs(args): nargs="*", help="List of space separated locations. A location consists of " "the tag of a variation axis, followed by '=' and the literal, " - "string 'drop', or comma-separate list of one to three values, " + "string 'drop', or colon-separated list of one to three values, " "each of which is the empty string, or a number. " "E.g.: wdth=100 or wght=75.0:125.0 or wght=100:400:700 or wght=:500: " "or wght=drop", diff --git a/contrib/python/fonttools/fontTools/varLib/instancer/solver.py b/contrib/python/fonttools/fontTools/varLib/instancer/solver.py index 9c568fe9a5..ba5231b796 100644 --- a/contrib/python/fonttools/fontTools/varLib/instancer/solver.py +++ b/contrib/python/fonttools/fontTools/varLib/instancer/solver.py @@ -178,7 +178,9 @@ def _solve(tent, axisLimit, negative=False): # newUpper = peak + (1 - gain) * (upper - peak) assert axisMax <= newUpper # Because outGain > gain - if newUpper <= axisDef + (axisMax - axisDef) * 2: + # Disabled because ots doesn't like us: + # https://github.com/fonttools/fonttools/issues/3350 + if False and newUpper <= axisDef + (axisMax - axisDef) * 2: upper = newUpper if not negative and axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper: # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatable.py b/contrib/python/fonttools/fontTools/varLib/interpolatable.py index f03e946207..0a9bbebc41 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatable.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatable.py @@ -9,7 +9,11 @@ $ fonttools varLib.interpolatable font1 font2 ... from .interpolatableHelpers import * from .interpolatableTestContourOrder import test_contour_order from .interpolatableTestStartingPoint import test_starting_point -from fontTools.pens.recordingPen import RecordingPen, DecomposingRecordingPen +from fontTools.pens.recordingPen import ( + RecordingPen, + DecomposingRecordingPen, + lerpRecordings, +) from fontTools.pens.transformPen import TransformPen from fontTools.pens.statisticsPen import StatisticsPen, StatisticsControlPen from fontTools.pens.momentsPen import OpenContourError @@ -22,6 +26,7 @@ from functools import wraps from pprint import pformat from math import sqrt, atan2, pi import logging +import os log = logging.getLogger("fontTools.varLib.interpolatable") @@ -34,11 +39,9 @@ DEFAULT_UPEM = 1000 class Glyph: ITEMS = ( "recordings", - "recordingsNormalized", "greenStats", "controlStats", "greenVectors", - "greenVectorsNormalized", "controlVectors", "nodeTypes", "isomorphisms", @@ -90,21 +93,6 @@ class Glyph: self.greenVectors.append(contour_vector_from_stats(greenStats)) self.controlVectors.append(contour_vector_from_stats(controlStats)) - # Save a "normalized" version of the outlines - try: - rpen = DecomposingRecordingPen(glyphset) - tpen = TransformPen( - rpen, transform_from_stats(greenStats, inverse=True) - ) - contour.replay(tpen) - self.recordingsNormalized.append(rpen) - except ZeroDivisionError: - self.recordingsNormalized.append(None) - - greenStats = StatisticsPen(glyphset=glyphset) - rpen.replay(greenStats) - self.greenVectorsNormalized.append(contour_vector_from_stats(greenStats)) - # Check starting point if nodeTypes[0] == "addComponent": self._fill_in(ix) @@ -186,7 +174,11 @@ def test_gen( if not ignore_missing: yield ( glyph_name, - {"type": "missing", "master": name, "master_idx": master_idx}, + { + "type": InterpolatableProblem.MISSING, + "master": name, + "master_idx": master_idx, + }, ) continue @@ -198,10 +190,10 @@ def test_gen( yield ( glyph_name, { + "type": InterpolatableProblem.OPEN_PATH, "master": name, "master_idx": master_idx, "contour": ix, - "type": "open_path", }, ) if has_open: @@ -230,7 +222,7 @@ def test_gen( yield ( glyph_name, { - "type": "path_count", + "type": InterpolatableProblem.PATH_COUNT, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -249,7 +241,7 @@ def test_gen( yield ( glyph_name, { - "type": "node_count", + "type": InterpolatableProblem.NODE_COUNT, "path": pathIx, "master_1": names[m0idx], "master_2": names[m1idx], @@ -265,7 +257,7 @@ def test_gen( yield ( glyph_name, { - "type": "node_incompatibility", + "type": InterpolatableProblem.NODE_INCOMPATIBILITY, "path": pathIx, "node": nodeIx, "master_1": names[m0idx], @@ -279,21 +271,15 @@ def test_gen( continue # - # "contour_order" check + # InterpolatableProblem.CONTOUR_ORDER check # - matching, matching_cost, identity_cost = test_contour_order(glyph0, glyph1) - if matching_cost < identity_cost * tolerance: - log.debug( - "matching_ratio %g", - matching_cost / identity_cost, - ) - this_tolerance = matching_cost / identity_cost - log.debug("tolerance: %g", this_tolerance) + this_tolerance, matching = test_contour_order(glyph0, glyph1) + if this_tolerance < tolerance: yield ( glyph_name, { - "type": "contour_order", + "type": InterpolatableProblem.CONTOUR_ORDER, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -306,19 +292,15 @@ def test_gen( matchings[m1idx] = matching # - # "wrong_start_point" / weight check + # wrong-start-point / weight check # m0Isomorphisms = glyph0.isomorphisms m1Isomorphisms = glyph1.isomorphisms m0Vectors = glyph0.greenVectors m1Vectors = glyph1.greenVectors - m0VectorsNormalized = glyph0.greenVectorsNormalized - m1VectorsNormalized = glyph1.greenVectorsNormalized recording0 = glyph0.recordings recording1 = glyph1.recordings - recording0Normalized = glyph0.recordingsNormalized - recording1Normalized = glyph1.recordingsNormalized # If contour-order is wrong, adjust it matching = matchings[m1idx] @@ -327,14 +309,14 @@ def test_gen( ): # m1 is empty for composite glyphs m1Isomorphisms = [m1Isomorphisms[i] for i in matching] m1Vectors = [m1Vectors[i] for i in matching] - m1VectorsNormalized = [m1VectorsNormalized[i] for i in matching] recording1 = [recording1[i] for i in matching] - recording1Normalized = [recording1Normalized[i] for i in matching] midRecording = [] for c0, c1 in zip(recording0, recording1): try: - midRecording.append(lerp_recordings(c0, c1)) + r = RecordingPen() + r.value = list(lerpRecordings(c0.value, c1.value)) + midRecording.append(r) except ValueError: # Mismatch because of the reordering above midRecording.append(None) @@ -352,118 +334,100 @@ def test_gen( # after reordering above. continue - proposed_point, reverse, min_cost, first_cost = test_starting_point( + this_tolerance, proposed_point, reverse = test_starting_point( glyph0, glyph1, ix, tolerance, matching ) - if proposed_point or reverse: - this_tolerance = min_cost / first_cost - log.debug("tolerance: %g", this_tolerance) - if min_cost < first_cost * tolerance: - yield ( - glyph_name, - { - "type": "wrong_start_point", - "contour": ix, - "master_1": names[m0idx], - "master_2": names[m1idx], - "master_1_idx": m0idx, - "master_2_idx": m1idx, - "value_1": 0, - "value_2": proposed_point, - "reversed": reverse, - "tolerance": this_tolerance, - }, - ) - else: - # Weight check. - # - # If contour could be mid-interpolated, and the two - # contours have the same area sign, proceeed. - # - # The sign difference can happen if it's a werido - # self-intersecting contour; ignore it. - contour = midRecording[ix] - - normalized = False - if contour and (m0Vectors[ix][0] < 0) == (m1Vectors[ix][0] < 0): - if normalized: - midStats = StatisticsPen(glyphset=None) - tpen = TransformPen( - midStats, transform_from_stats(midStats, inverse=True) - ) - contour.replay(tpen) - else: - midStats = StatisticsPen(glyphset=None) - contour.replay(midStats) - - midVector = contour_vector_from_stats(midStats) + if this_tolerance < tolerance: + yield ( + glyph_name, + { + "type": InterpolatableProblem.WRONG_START_POINT, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": 0, + "value_2": proposed_point, + "reversed": reverse, + "tolerance": this_tolerance, + }, + ) - m0Vec = ( - m0Vectors[ix] if not normalized else m0VectorsNormalized[ix] + # Weight check. + # + # If contour could be mid-interpolated, and the two + # contours have the same area sign, proceeed. + # + # The sign difference can happen if it's a weirdo + # self-intersecting contour; ignore it. + contour = midRecording[ix] + + if contour and (m0Vectors[ix][0] < 0) == (m1Vectors[ix][0] < 0): + midStats = StatisticsPen(glyphset=None) + contour.replay(midStats) + + midVector = contour_vector_from_stats(midStats) + + m0Vec = m0Vectors[ix] + m1Vec = m1Vectors[ix] + size0 = m0Vec[0] * m0Vec[0] + size1 = m1Vec[0] * m1Vec[0] + midSize = midVector[0] * midVector[0] + + power = 1 + t = tolerance**power + + for overweight, problem_type in enumerate( + ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, ) - m1Vec = ( - m1Vectors[ix] if not normalized else m1VectorsNormalized[ix] + ): + if overweight: + expectedSize = sqrt(size0 * size1) + expectedSize = (size0 + size1) - expectedSize + continue + else: + expectedSize = sqrt(size0 * size1) + + log.debug( + "%s: actual size %g; threshold size %g, master sizes: %g, %g", + problem_type, + midSize, + expectedSize, + size0, + size1, ) - size0 = m0Vec[0] * m0Vec[0] - size1 = m1Vec[0] * m1Vec[0] - midSize = midVector[0] * midVector[0] - - power = 1 - t = tolerance**power - - for overweight, problem_type in enumerate( - ("underweight", "overweight") - ): - if overweight: - expectedSize = sqrt(size0 * size1) - expectedSize = (size0 + size1) - expectedSize - expectedSize = size1 + (midSize - size1) - continue - else: - expectedSize = sqrt(size0 * size1) - - log.debug( - "%s: actual size %g; threshold size %g, master sizes: %g, %g", - problem_type, - midSize, - expectedSize, - size0, - size1, - ) - size0, size1 = sorted((size0, size1)) - - if ( - not overweight - and expectedSize * tolerance > midSize + 1e-5 - ) or ( - overweight and 1e-5 + expectedSize / tolerance < midSize - ): - try: - if overweight: - this_tolerance = (expectedSize / midSize) ** ( - 1 / power - ) - else: - this_tolerance = (midSize / expectedSize) ** ( - 1 / power - ) - except ZeroDivisionError: - this_tolerance = 0 - log.debug("tolerance %g", this_tolerance) - yield ( - glyph_name, - { - "type": problem_type, - "contour": ix, - "master_1": names[m0idx], - "master_2": names[m1idx], - "master_1_idx": m0idx, - "master_2_idx": m1idx, - "tolerance": this_tolerance, - }, - ) + if ( + not overweight and expectedSize * tolerance > midSize + 1e-5 + ) or (overweight and 1e-5 + expectedSize / tolerance < midSize): + try: + if overweight: + this_tolerance = (expectedSize / midSize) ** ( + 1 / power + ) + else: + this_tolerance = (midSize / expectedSize) ** ( + 1 / power + ) + except ZeroDivisionError: + this_tolerance = 0 + log.debug("tolerance %g", this_tolerance) + yield ( + glyph_name, + { + "type": problem_type, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "tolerance": this_tolerance, + }, + ) # # "kink" detector @@ -585,7 +549,7 @@ def test_gen( this_tolerance = t / (abs(sin_mid) * kinkiness) log.debug( - "deviation %g; deviation_ratio %g; sin_mid %g; r_diff %g", + "kink: deviation %g; deviation_ratio %g; sin_mid %g; r_diff %g", deviation, deviation_ratio, sin_mid, @@ -595,7 +559,7 @@ def test_gen( yield ( glyph_name, { - "type": "kink", + "type": InterpolatableProblem.KINK, "contour": ix, "master_1": names[m0idx], "master_2": names[m1idx], @@ -614,7 +578,7 @@ def test_gen( yield ( glyph_name, { - "type": "nothing", + "type": InterpolatableProblem.NOTHING, "master_1": names[m0idx], "master_2": names[m1idx], "master_1_idx": m0idx, @@ -640,6 +604,13 @@ def recursivelyAddGlyph(glyphname, glyphset, ttGlyphSet, glyf): recursivelyAddGlyph(component.glyphName, glyphset, ttGlyphSet, glyf) +def ensure_parent_dir(path): + dirname = os.path.dirname(path) + if dirname: + os.makedirs(dirname, exist_ok=True) + return path + + def main(args=None): """Test for interpolatability issues between fonts""" import argparse @@ -759,7 +730,7 @@ def main(args=None): for k, vv in axis_triples.items() } - elif args.inputs[0].endswith(".glyphs"): + elif args.inputs[0].endswith((".glyphs", ".glyphspackage")): from glyphsLib import GSFont, to_designspace gsfont = GSFont(args.inputs[0]) @@ -929,7 +900,11 @@ def main(args=None): ) problems = defaultdict(list) - f = sys.stdout if args.output is None else open(args.output, "w") + f = ( + sys.stdout + if args.output is None + else open(ensure_parent_dir(args.output), "w") + ) if not args.quiet: if args.json: @@ -963,16 +938,16 @@ def main(args=None): print(f" Masters: %s:" % ", ".join(master_names), file=f) last_master_idxs = master_idxs - if p["type"] == "missing": + if p["type"] == InterpolatableProblem.MISSING: print( " Glyph was missing in master %s" % p["master"], file=f ) - elif p["type"] == "open_path": + elif p["type"] == InterpolatableProblem.OPEN_PATH: print( " Glyph has an open path in master %s" % p["master"], file=f, ) - elif p["type"] == "path_count": + elif p["type"] == InterpolatableProblem.PATH_COUNT: print( " Path count differs: %i in %s, %i in %s" % ( @@ -983,7 +958,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "node_count": + elif p["type"] == InterpolatableProblem.NODE_COUNT: print( " Node count differs in path %i: %i in %s, %i in %s" % ( @@ -995,7 +970,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "node_incompatibility": + elif p["type"] == InterpolatableProblem.NODE_INCOMPATIBILITY: print( " Node %o incompatible in path %i: %s in %s, %s in %s" % ( @@ -1008,7 +983,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "contour_order": + elif p["type"] == InterpolatableProblem.CONTOUR_ORDER: print( " Contour order differs: %s in %s, %s in %s" % ( @@ -1019,7 +994,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "wrong_start_point": + elif p["type"] == InterpolatableProblem.WRONG_START_POINT: print( " Contour %d start point differs: %s in %s, %s in %s; reversed: %s" % ( @@ -1032,7 +1007,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "underweight": + elif p["type"] == InterpolatableProblem.UNDERWEIGHT: print( " Contour %d interpolation is underweight: %s, %s" % ( @@ -1042,7 +1017,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "overweight": + elif p["type"] == InterpolatableProblem.OVERWEIGHT: print( " Contour %d interpolation is overweight: %s, %s" % ( @@ -1052,7 +1027,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "kink": + elif p["type"] == InterpolatableProblem.KINK: print( " Contour %d has a kink at %s: %s, %s" % ( @@ -1063,7 +1038,7 @@ def main(args=None): ), file=f, ) - elif p["type"] == "nothing": + elif p["type"] == InterpolatableProblem.NOTHING: print( " Showing %s and %s" % ( @@ -1076,29 +1051,31 @@ def main(args=None): for glyphname, problem in problems_gen: problems[glyphname].append(problem) - if args.pdf: - log.info("Writing PDF to %s", args.pdf) - from .interpolatablePlot import InterpolatablePDF + problems = sort_problems(problems) - with InterpolatablePDF(args.pdf, glyphsets=glyphsets, names=names) as pdf: - pdf.add_title_page( - original_args_inputs, tolerance=tolerance, kinkiness=kinkiness - ) - pdf.add_problems(problems) - if not problems and not args.quiet: - pdf.draw_cupcake() + for p in "ps", "pdf": + arg = getattr(args, p) + if arg is None: + continue + log.info("Writing %s to %s", p.upper(), arg) + from .interpolatablePlot import InterpolatablePS, InterpolatablePDF - if args.ps: - log.info("Writing PS to %s", args.pdf) - from .interpolatablePlot import InterpolatablePS + PlotterClass = InterpolatablePS if p == "ps" else InterpolatablePDF - with InterpolatablePS(args.ps, glyphsets=glyphsets, names=names) as ps: - ps.add_title_page( + with PlotterClass( + ensure_parent_dir(arg), glyphsets=glyphsets, names=names + ) as doc: + doc.add_title_page( original_args_inputs, tolerance=tolerance, kinkiness=kinkiness ) - ps.add_problems(problems) + if problems: + doc.add_summary(problems) + doc.add_problems(problems) if not problems and not args.quiet: - ps.draw_cupcake() + doc.draw_cupcake() + if problems: + doc.add_index() + doc.add_table_of_contents() if args.html: log.info("Writing HTML to %s", args.html) @@ -1125,7 +1102,7 @@ def main(args=None): import base64 - with open(args.html, "wb") as f: + with open(ensure_parent_dir(args.html), "wb") as f: f.write(b"<!DOCTYPE html>\n") f.write( b'<html><body align="center" style="font-family: sans-serif; text-color: #222">\n' diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py b/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py index 513e5f7409..2a3540fff2 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableHelpers.py @@ -1,9 +1,11 @@ +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet from fontTools.pens.basePen import AbstractPen, BasePen, DecomposingPen from fontTools.pens.pointPen import AbstractPointPen, SegmentToPointPen from fontTools.pens.recordingPen import RecordingPen, DecomposingRecordingPen from fontTools.misc.transform import Transform from collections import defaultdict, deque from math import sqrt, copysign, atan2, pi +from enum import Enum import itertools import logging @@ -11,6 +13,50 @@ import logging log = logging.getLogger("fontTools.varLib.interpolatable") +class InterpolatableProblem: + NOTHING = "nothing" + MISSING = "missing" + OPEN_PATH = "open_path" + PATH_COUNT = "path_count" + NODE_COUNT = "node_count" + NODE_INCOMPATIBILITY = "node_incompatibility" + CONTOUR_ORDER = "contour_order" + WRONG_START_POINT = "wrong_start_point" + KINK = "kink" + UNDERWEIGHT = "underweight" + OVERWEIGHT = "overweight" + + severity = { + MISSING: 1, + OPEN_PATH: 2, + PATH_COUNT: 3, + NODE_COUNT: 4, + NODE_INCOMPATIBILITY: 5, + CONTOUR_ORDER: 6, + WRONG_START_POINT: 7, + KINK: 8, + UNDERWEIGHT: 9, + OVERWEIGHT: 10, + NOTHING: 11, + } + + +def sort_problems(problems): + """Sort problems by severity, then by glyph name, then by problem message.""" + return dict( + sorted( + problems.items(), + key=lambda _: -min( + ( + (InterpolatableProblem.severity[p["type"]] + p.get("tolerance", 0)) + for p in _[1] + ), + ), + reverse=True, + ) + ) + + def rot_list(l, k): """Rotate list by k items forward. Ie. item at position 0 will be at position k in returned list. Negative k is allowed.""" @@ -332,52 +378,3 @@ def transform_from_stats(stats, inverse=False): trans = trans.translate(stats.meanX, stats.meanY) return trans - - -class LerpGlyphSet: - def __init__(self, glyphset1, glyphset2, factor=0.5): - self.glyphset1 = glyphset1 - self.glyphset2 = glyphset2 - self.factor = factor - - def __getitem__(self, glyphname): - return LerpGlyph(glyphname, self) - - -class LerpGlyph: - def __init__(self, glyphname, glyphset): - self.glyphset = glyphset - self.glyphname = glyphname - - def draw(self, pen): - recording1 = DecomposingRecordingPen(self.glyphset.glyphset1) - self.glyphset.glyphset1[self.glyphname].draw(recording1) - recording2 = DecomposingRecordingPen(self.glyphset.glyphset2) - self.glyphset.glyphset2[self.glyphname].draw(recording2) - - factor = self.glyphset.factor - for (op1, args1), (op2, args2) in zip(recording1.value, recording2.value): - if op1 != op2: - raise ValueError("Mismatching operations: %s, %s" % (op1, op2)) - mid_args = [ - (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) - for (x1, y1), (x2, y2) in zip(args1, args2) - ] - getattr(pen, op1)(*mid_args) - - -def lerp_recordings(recording1, recording2, factor=0.5): - pen = RecordingPen() - value = pen.value - for (op1, args1), (op2, args2) in zip(recording1.value, recording2.value): - if op1 != op2: - raise ValueError("Mismatched operations: %s, %s" % (op1, op2)) - if op1 == "addComponent": - mid_args = args1 # XXX Interpolate transformation? - else: - mid_args = [ - (x1 + (x2 - x1) * factor, y1 + (y2 - y1) * factor) - for (x1, y1), (x2, y2) in zip(args1, args2) - ] - value.append((op1, mid_args)) - return pen diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py b/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py index eef4a47160..3c206c6ee2 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatablePlot.py @@ -1,4 +1,6 @@ +from .interpolatableHelpers import * from fontTools.ttLib import TTFont +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet from fontTools.pens.recordingPen import ( RecordingPen, DecomposingRecordingPen, @@ -11,10 +13,9 @@ from fontTools.pens.pointPen import ( PointToSegmentPen, ReverseContourPointPen, ) -from fontTools.varLib.interpolatable import ( +from fontTools.varLib.interpolatableHelpers import ( PerContourOrComponentPen, SimpleRecordingPointPen, - LerpGlyphSet, ) from itertools import cycle from functools import wraps @@ -36,33 +37,34 @@ class OverridingDict(dict): class InterpolatablePlot: - width = 640 - height = 480 - pad = 16 - line_height = 36 + width = 8.5 * 72 + height = 11 * 72 + pad = 0.1 * 72 + title_font_size = 24 + font_size = 16 page_number = 1 head_color = (0.3, 0.3, 0.3) label_color = (0.2, 0.2, 0.2) border_color = (0.9, 0.9, 0.9) - border_width = 1 + border_width = 0.5 fill_color = (0.8, 0.8, 0.8) stroke_color = (0.1, 0.1, 0.1) - stroke_width = 2 + stroke_width = 1 oncurve_node_color = (0, 0.8, 0, 0.7) - oncurve_node_diameter = 10 + oncurve_node_diameter = 6 offcurve_node_color = (0, 0.5, 0, 0.7) - offcurve_node_diameter = 8 + offcurve_node_diameter = 4 handle_color = (0, 0.5, 0, 0.7) - handle_width = 1 + handle_width = 0.5 corrected_start_point_color = (0, 0.9, 0, 0.7) - corrected_start_point_size = 15 + corrected_start_point_size = 7 wrong_start_point_color = (1, 0, 0, 0.7) start_point_color = (0, 0, 1, 0.7) - start_arrow_length = 20 - kink_point_size = 10 + start_arrow_length = 9 + kink_point_size = 7 kink_point_color = (1, 0, 1, 0.7) - kink_circle_size = 25 - kink_circle_stroke_width = 1.5 + kink_circle_size = 15 + kink_circle_stroke_width = 1 kink_circle_color = (1, 0, 1, 0.7) contour_colors = ((1, 0, 0), (0, 0, 1), (0, 1, 0), (1, 1, 0), (1, 0, 1), (0, 1, 1)) contour_alpha = 0.5 @@ -113,62 +115,59 @@ class InterpolatablePlot: self.out = out self.glyphsets = glyphsets self.names = names or [repr(g) for g in glyphsets] + self.toc = {} for k, v in kwargs.items(): if not hasattr(self, k): raise TypeError("Unknown keyword argument: %s" % k) setattr(self, k, v) + self.panel_width = self.width / 2 - self.pad * 3 + self.panel_height = ( + self.height / 2 - self.pad * 6 - self.font_size * 2 - self.title_font_size + ) + def __enter__(self): return self def __exit__(self, type, value, traceback): pass - def set_size(self, width, height): - raise NotImplementedError - def show_page(self): self.page_number += 1 - def total_width(self): - return self.width * 2 + self.pad * 3 - - def total_height(self): - return ( - self.pad - + self.line_height - + self.pad - + self.line_height - + self.pad - + 2 * (self.height + self.pad * 2 + self.line_height) - + self.pad - ) - def add_title_page( self, files, *, show_tolerance=True, tolerance=None, kinkiness=None ): - self.set_size(self.total_width(), self.total_height()) - pad = self.pad - width = self.total_width() - 3 * self.pad - height = self.total_height() - 2 * self.pad + width = self.width - 3 * self.pad + height = self.height - 2 * self.pad x = y = pad - self.draw_label("Problem report for:", x=x, y=y, bold=True, width=width) - y += self.line_height + self.draw_label( + "Problem report for:", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size import hashlib for file in files: base_file = os.path.basename(file) - y += self.line_height + y += self.font_size + self.pad self.draw_label(base_file, x=x, y=y, bold=True, width=width) - y += self.line_height + y += self.font_size + self.pad - h = hashlib.sha1(open(file, "rb").read()).hexdigest() - self.draw_label("sha1: %s" % h, x=x + pad, y=y, width=width) - y += self.line_height + try: + h = hashlib.sha1(open(file, "rb").read()).hexdigest() + self.draw_label("sha1: %s" % h, x=x + pad, y=y, width=width) + y += self.font_size + except IsADirectoryError: + pass if file.endswith(".ttf"): ttFont = TTFont(file) @@ -184,8 +183,8 @@ class InterpolatablePlot: self.draw_label( "%s: %s" % (what, n), x=x + pad, y=y, width=width ) - y += self.line_height - elif file.endswith(".glyphs"): + y += self.font_size + self.pad + elif file.endswith((".glyphs", ".glyphspackage")): from glyphsLib import GSFont f = GSFont(file) @@ -200,7 +199,7 @@ class InterpolatablePlot: y=y, width=width, ) - y += self.line_height + y += self.font_size + self.pad self.draw_legend( show_tolerance=show_tolerance, tolerance=tolerance, kinkiness=kinkiness @@ -211,8 +210,8 @@ class InterpolatablePlot: cr = cairo.Context(self.surface) x = self.pad - y = self.total_height() - self.pad - self.line_height * 2 - width = self.total_width() - 2 * self.pad + y = self.height - self.pad - self.font_size * 2 + width = self.width - 2 * self.pad xx = x + self.pad * 2 xxx = x + self.pad * 4 @@ -221,10 +220,10 @@ class InterpolatablePlot: self.draw_label( "Tolerance: badness; closer to zero the worse", x=xxx, y=y, width=width ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Underweight contours", x=xxx, y=y, width=width) - cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.line_height) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) cr.set_source_rgb(*self.fill_color) cr.fill_preserve() if self.stroke_color: @@ -233,12 +232,12 @@ class InterpolatablePlot: cr.stroke_preserve() cr.set_source_rgba(*self.weight_issue_contour_color) cr.fill() - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Colored contours: contours with the wrong order", x=xxx, y=y, width=width ) - cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.line_height) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) if self.fill_color: cr.set_source_rgb(*self.fill_color) cr.fill_preserve() @@ -248,38 +247,38 @@ class InterpolatablePlot: cr.stroke_preserve() cr.set_source_rgba(*self.contour_colors[0], self.contour_alpha) cr.fill() - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Kink artifact", x=xxx, y=y, width=width) self.draw_circle( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.kink_circle_size, stroke_width=self.kink_circle_stroke_width, color=self.kink_circle_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Point causing kink in the contour", x=xxx, y=y, width=width) self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.kink_point_size, color=self.kink_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Suggested new contour start point", x=xxx, y=y, width=width) self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.corrected_start_point_size, color=self.corrected_start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Contour start point in contours with wrong direction", @@ -290,10 +289,10 @@ class InterpolatablePlot: self.draw_arrow( cr, x=xx - self.start_arrow_length * 0.3, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, color=self.wrong_start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label( "Contour start point when the first two points overlap", @@ -304,23 +303,23 @@ class InterpolatablePlot: self.draw_dot( cr, x=xx, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, diameter=self.corrected_start_point_size, color=self.start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Contour start point and direction", x=xxx, y=y, width=width) self.draw_arrow( cr, x=xx - self.start_arrow_length * 0.3, - y=y + self.line_height * 0.5, + y=y + self.font_size * 0.5, color=self.start_point_color, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Legend:", x=x, y=y, width=width, bold=True) - y -= self.pad + self.line_height + y -= self.pad + self.font_size if kinkiness is not None: self.draw_label( @@ -329,7 +328,7 @@ class InterpolatablePlot: y=y, width=width, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size if tolerance is not None: self.draw_label( @@ -338,10 +337,87 @@ class InterpolatablePlot: y=y, width=width, ) - y -= self.pad + self.line_height + y -= self.pad + self.font_size self.draw_label("Parameters:", x=x, y=y, width=width, bold=True) - y -= self.pad + self.line_height + y -= self.pad + self.font_size + + def add_summary(self, problems): + pad = self.pad + width = self.width - 3 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + "Summary of problems", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size + + glyphs_per_problem = defaultdict(set) + for glyphname, problems in sorted(problems.items()): + for problem in problems: + glyphs_per_problem[problem["type"]].add(glyphname) + + if "nothing" in glyphs_per_problem: + del glyphs_per_problem["nothing"] + + for problem_type in sorted( + glyphs_per_problem, key=lambda x: InterpolatableProblem.severity[x] + ): + y += self.font_size + self.draw_label( + "%s: %d" % (problem_type, len(glyphs_per_problem[problem_type])), + x=x, + y=y, + width=width, + bold=True, + ) + y += self.font_size + + for glyphname in sorted(glyphs_per_problem[problem_type]): + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 2 * pad, y=y, width=width - 2 * pad) + y += self.font_size + + self.show_page() + + def _add_listing(self, title, items): + pad = self.pad + width = self.width - 2 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + title, x=x, y=y, bold=True, width=width, font_size=self.title_font_size + ) + y += self.title_font_size + self.pad + + last_glyphname = None + for page_no, (glyphname, problems) in items: + if glyphname == last_glyphname: + continue + last_glyphname = glyphname + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 5 * pad, y=y, width=width - 2 * pad) + self.draw_label(str(page_no), x=x, y=y, width=4 * pad, align=1) + y += self.font_size + + self.show_page() + + def add_table_of_contents(self): + self._add_listing("Table of contents", sorted(self.toc.items())) + + def add_index(self): + self._add_listing("Index", sorted(self.toc.items(), key=lambda x: x[1][0])) def add_problems(self, problems, *, show_tolerance=True, show_page_number=True): for glyph, glyph_problems in problems.items(): @@ -383,6 +459,8 @@ class InterpolatablePlot: if type(problems) not in (list, tuple): problems = [problems] + self.toc[self.page_number] = (glyphname, problems) + problem_type = problems[0]["type"] problem_types = set(problem["type"] for problem in problems) if not all(pt == problem_type for pt in problem_types): @@ -397,14 +475,12 @@ class InterpolatablePlot: ) master_indices = [problems[0][k] for k in master_keys] - if problem_type == "missing": + if problem_type == InterpolatableProblem.MISSING: sample_glyph = next( i for i, m in enumerate(self.glyphsets) if m[glyphname] is not None ) master_indices.insert(0, sample_glyph) - self.set_size(self.total_width(), self.total_height()) - x = self.pad y = self.pad @@ -415,6 +491,7 @@ class InterpolatablePlot: color=self.head_color, align=0, bold=True, + font_size=self.title_font_size, ) tolerance = min(p.get("tolerance", 1) for p in problems) if tolerance < 1 and show_tolerance: @@ -422,29 +499,35 @@ class InterpolatablePlot: "tolerance: %.2f" % tolerance, x=x, y=y, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, align=1, bold=True, ) - y += self.line_height + self.pad + y += self.title_font_size + self.pad self.draw_label( - problem_type, + "Problems: " + problem_type, x=x, y=y, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, color=self.head_color, - align=0.5, bold=True, ) - y += self.line_height + self.pad + y += self.font_size + self.pad * 2 scales = [] for which, master_idx in enumerate(master_indices): glyphset = self.glyphsets[master_idx] name = self.names[master_idx] - self.draw_label(name, x=x, y=y, color=self.label_color, align=0.5) - y += self.line_height + self.pad + self.draw_label( + name, + x=x, + y=y, + color=self.label_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad if glyphset[glyphname] is not None: scales.append( @@ -452,24 +535,24 @@ class InterpolatablePlot: ) else: self.draw_emoticon(self.shrug, x=x, y=y) - y += self.height + self.pad + y += self.panel_height + self.font_size + self.pad if any( pt in ( - "nothing", - "wrong_start_point", - "contour_order", - "kink", - "underweight", - "overweight", + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, ) for pt in problem_types ): - x = self.pad + self.width + self.pad + x = self.pad + self.panel_width + self.pad y = self.pad - y += self.line_height + self.pad - y += self.line_height + self.pad + y += self.title_font_size + self.pad * 2 + y += self.font_size + self.pad glyphset1 = self.glyphsets[master_indices[0]] glyphset2 = self.glyphsets[master_indices[1]] @@ -477,9 +560,14 @@ class InterpolatablePlot: # Draw the mid-way of the two masters self.draw_label( - "midway interpolation", x=x, y=y, color=self.head_color, align=0.5 + "midway interpolation", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, ) - y += self.line_height + self.pad + y += self.font_size + self.pad midway_glyphset = LerpGlyphSet(glyphset1, glyphset2) self.draw_glyph( @@ -489,7 +577,12 @@ class InterpolatablePlot: + [ p for p in problems - if p["type"] in ("kink", "underweight", "overweight") + if p["type"] + in ( + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ) ], None, x=x, @@ -497,21 +590,28 @@ class InterpolatablePlot: scale=min(scales), ) - y += self.height + self.pad + y += self.panel_height + self.font_size + self.pad if any( pt in ( - "wrong_start_point", - "contour_order", - "kink", + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, ) for pt in problem_types ): # Draw the proposed fix - self.draw_label("proposed fix", x=x, y=y, color=self.head_color, align=0.5) - y += self.line_height + self.pad + self.draw_label( + "proposed fix", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad overriding1 = OverridingDict(glyphset1) overriding2 = OverridingDict(glyphset2) @@ -525,14 +625,14 @@ class InterpolatablePlot: glyphset2[glyphname].draw(perContourPen2) for problem in problems: - if problem["type"] == "contour_order": + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: fixed_contours = [ perContourPen2.value[i] for i in problems[0]["value_2"] ] perContourPen2.value = fixed_contours for problem in problems: - if problem["type"] == "wrong_start_point": + if problem["type"] == InterpolatableProblem.WRONG_START_POINT: # Save the wrong contours wrongContour1 = perContourPen1.value[problem["contour"]] wrongContour2 = perContourPen2.value[problem["contour"]] @@ -578,7 +678,7 @@ class InterpolatablePlot: for problem in problems: # If we have a kink, try to fix it. - if problem["type"] == "kink": + if problem["type"] == InterpolatableProblem.KINK: # Save the wrong contours wrongContour1 = perContourPen1.value[problem["contour"]] wrongContour2 = perContourPen2.value[problem["contour"]] @@ -669,15 +769,15 @@ class InterpolatablePlot: ) except ValueError: self.draw_emoticon(self.shrug, x=x, y=y) - y += self.height + self.pad + y += self.panel_height + self.pad else: emoticon = self.shrug - if "underweight" in problem_types: + if InterpolatableProblem.UNDERWEIGHT in problem_types: emoticon = self.underweight - elif "overweight" in problem_types: + elif InterpolatableProblem.OVERWEIGHT in problem_types: emoticon = self.overweight - elif "nothing" in problem_types: + elif InterpolatableProblem.NOTHING in problem_types: emoticon = self.yay self.draw_emoticon(emoticon, x=x, y=y) @@ -685,8 +785,8 @@ class InterpolatablePlot: self.draw_label( str(self.page_number), x=0, - y=self.total_height() - self.line_height, - width=self.total_width(), + y=self.height - self.font_size - self.pad, + width=self.width, color=self.head_color, align=0.5, ) @@ -702,20 +802,23 @@ class InterpolatablePlot: bold=False, width=None, height=None, + font_size=None, ): if width is None: width = self.width if height is None: height = self.height + if font_size is None: + font_size = self.font_size cr = cairo.Context(self.surface) cr.select_font_face( "@cairo:", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD if bold else cairo.FONT_WEIGHT_NORMAL, ) - cr.set_font_size(self.line_height) + cr.set_font_size(font_size) font_extents = cr.font_extents() - font_size = self.line_height * self.line_height / font_extents[2] + font_size = font_size * font_size / font_extents[2] cr.set_font_size(font_size) font_extents = cr.font_extents() @@ -762,14 +865,14 @@ class InterpolatablePlot: if glyph_width: if scale is None: - scale = self.width / glyph_width + scale = self.panel_width / glyph_width else: - scale = min(scale, self.height / glyph_height) + scale = min(scale, self.panel_height / glyph_height) if glyph_height: if scale is None: - scale = self.height / glyph_height + scale = self.panel_height / glyph_height else: - scale = min(scale, self.height / glyph_height) + scale = min(scale, self.panel_height / glyph_height) if scale is None: scale = 1 @@ -777,8 +880,8 @@ class InterpolatablePlot: cr.translate(x, y) # Center cr.translate( - (self.width - glyph_width * scale) / 2, - (self.height - glyph_height * scale) / 2, + (self.panel_width - glyph_width * scale) / 2, + (self.panel_height - glyph_height * scale) / 2, ) cr.scale(scale, -scale) cr.translate(-bounds[0], -bounds[3]) @@ -793,7 +896,7 @@ class InterpolatablePlot: pen = CairoPen(glyphset, cr) decomposedRecording.replay(pen) - if self.fill_color and problem_type != "open_path": + if self.fill_color and problem_type != InterpolatableProblem.OPEN_PATH: cr.set_source_rgb(*self.fill_color) cr.fill_preserve() @@ -804,11 +907,17 @@ class InterpolatablePlot: cr.new_path() - if "underweight" in problem_types or "overweight" in problem_types: + if ( + InterpolatableProblem.UNDERWEIGHT in problem_types + or InterpolatableProblem.OVERWEIGHT in problem_types + ): perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset) recording.replay(perContourPen) for problem in problems: - if problem["type"] in ("underweight", "overweight"): + if problem["type"] in ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ): contour = perContourPen.value[problem["contour"]] contour.replay(CairoPen(glyphset, cr)) cr.set_source_rgba(*self.weight_issue_contour_color) @@ -817,9 +926,9 @@ class InterpolatablePlot: if any( t in problem_types for t in { - "nothing", - "node_count", - "node_incompatibility", + InterpolatableProblem.NOTHING, + InterpolatableProblem.NODE_COUNT, + InterpolatableProblem.NODE_INCOMPATIBILITY, } ): cr.set_line_cap(cairo.LINE_CAP_ROUND) @@ -873,7 +982,7 @@ class InterpolatablePlot: matching = None for problem in problems: - if problem["type"] == "contour_order": + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: matching = problem["value_2"] colors = cycle(self.contour_colors) perContourPen = PerContourOrComponentPen( @@ -889,7 +998,10 @@ class InterpolatablePlot: cr.fill() for problem in problems: - if problem["type"] in ("nothing", "wrong_start_point"): + if problem["type"] in ( + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + ): idx = problem.get("contour") # Draw suggested point @@ -967,7 +1079,7 @@ class InterpolatablePlot: cr.restore() - if problem["type"] == "kink": + if problem["type"] == InterpolatableProblem.KINK: idx = problem.get("contour") perContourPen = PerContourOrComponentPen( RecordingPen, glyphset=glyphset @@ -1053,19 +1165,19 @@ class InterpolatablePlot: text = text.splitlines() cr = cairo.Context(self.surface) cr.set_source_rgb(*color) - cr.set_font_size(self.line_height) + cr.set_font_size(self.font_size) cr.select_font_face( "@cairo:monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL ) text_width = 0 text_height = 0 font_extents = cr.font_extents() - font_line_height = font_extents[2] + font_font_size = font_extents[2] font_ascent = font_extents[0] for line in text: extents = cr.text_extents(line) text_width = max(text_width, extents.x_advance) - text_height += font_line_height + text_height += font_font_size if not text_width: return cr.translate(x, y) @@ -1080,45 +1192,44 @@ class InterpolatablePlot: for line in text: cr.move_to(0, 0) cr.show_text(line) - cr.translate(0, font_line_height) + cr.translate(0, font_font_size) def draw_cupcake(self): - self.set_size(self.total_width(), self.total_height()) - self.draw_label( self.no_issues_label, x=self.pad, y=self.pad, color=self.no_issues_label_color, - width=self.total_width() - 2 * self.pad, + width=self.width - 2 * self.pad, align=0.5, bold=True, + font_size=self.title_font_size, ) self.draw_text( self.cupcake, x=self.pad, - y=self.pad + self.line_height, - width=self.total_width() - 2 * self.pad, - height=self.total_height() - 2 * self.pad - self.line_height, + y=self.pad + self.font_size, + width=self.width - 2 * self.pad, + height=self.height - 2 * self.pad - self.font_size, color=self.cupcake_color, ) def draw_emoticon(self, emoticon, x=0, y=0): - self.draw_text(emoticon, x=x, y=y, color=self.emoticon_color) + self.draw_text( + emoticon, + x=x, + y=y, + color=self.emoticon_color, + width=self.panel_width, + height=self.panel_height, + ) class InterpolatablePostscriptLike(InterpolatablePlot): - @wraps(InterpolatablePlot.__init__) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def __exit__(self, type, value, traceback): self.surface.finish() - def set_size(self, width, height): - self.surface.set_size(width, height) - def show_page(self): super().show_page() self.surface.show_page() @@ -1141,24 +1252,18 @@ class InterpolatablePDF(InterpolatablePostscriptLike): class InterpolatableSVG(InterpolatablePlot): - @wraps(InterpolatablePlot.__init__) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def __enter__(self): - self.surface = None + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) return self def __exit__(self, type, value, traceback): if self.surface is not None: self.show_page() - def set_size(self, width, height): - self.sink = BytesIO() - self.surface = cairo.SVGSurface(self.sink, width, height) - def show_page(self): super().show_page() self.surface.finish() self.out.append(self.sink.getvalue()) - self.surface = None + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py b/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py index d089e43576..9edb1afcb5 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableTestContourOrder.py @@ -1,4 +1,7 @@ from .interpolatableHelpers import * +import logging + +log = logging.getLogger("fontTools.varLib.interpolatable") def test_contour_order(glyph0, glyph1): @@ -71,4 +74,9 @@ def test_contour_order(glyph0, glyph1): matching_cost = matching_cost_green identity_cost = identity_cost_green - return matching, matching_cost, identity_cost + this_tolerance = matching_cost / identity_cost if identity_cost else 1 + log.debug( + "test-contour-order: tolerance %g", + this_tolerance, + ) + return this_tolerance, matching diff --git a/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py b/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py index 9f742a14f5..e760006631 100644 --- a/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py +++ b/contrib/python/fonttools/fontTools/varLib/interpolatableTestStartingPoint.py @@ -9,18 +9,15 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): m0Vectors = glyph0.greenVectors m1Vectors = [glyph1.greenVectors[i] for i in matching] - proposed_point = 0 - reverse = False - min_cost = first_cost = 1 - c0 = contour0[0] # Next few lines duplicated below. costs = [vdiff_hypot2_complex(c0[0], c1[0]) for c1 in contour1] min_cost_idx, min_cost = min(enumerate(costs), key=lambda x: x[1]) first_cost = costs[0] + proposed_point = contour1[min_cost_idx][1] + reverse = contour1[min_cost_idx][2] if min_cost < first_cost * tolerance: - this_tolerance = min_cost / first_cost # c0 is the first isomorphism of the m0 master # contour1 is list of all isomorphisms of the m1 master # @@ -37,8 +34,6 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): # closest point again. If it matches this time, let it # pass. - proposed_point = contour1[min_cost_idx][1] - reverse = contour1[min_cost_idx][2] num_points = len(glyph1.points[ix]) leeway = 3 if not reverse and ( @@ -102,4 +97,9 @@ def test_starting_point(glyph0, glyph1, ix, tolerance, matching): # proposed_point = 0 # new_contour1[min_cost_idx][1] pass - return proposed_point, reverse, min_cost, first_cost + this_tolerance = min_cost / first_cost if first_cost else 1 + log.debug( + "test-starting-point: tolerance %g", + this_tolerance, + ) + return this_tolerance, proposed_point, reverse diff --git a/contrib/python/fonttools/fontTools/varLib/merger.py b/contrib/python/fonttools/fontTools/varLib/merger.py index b2c34016b3..96029166a7 100644 --- a/contrib/python/fonttools/fontTools/varLib/merger.py +++ b/contrib/python/fonttools/fontTools/varLib/merger.py @@ -1059,7 +1059,7 @@ class InstancerMerger(AligningMerger): Merger.__init__(self, font) self.model = model self.location = location - self.scalars = model.getScalars(location) + self.masterScalars = model.getMasterScalars(location) @InstancerMerger.merger(ot.CaretValue) @@ -1067,8 +1067,10 @@ def merge(merger, self, lst): assert self.Format == 1 Coords = [a.Coordinate for a in lst] model = merger.model - scalars = merger.scalars - self.Coordinate = otRound(model.interpolateFromMastersAndScalars(Coords, scalars)) + masterScalars = merger.masterScalars + self.Coordinate = otRound( + model.interpolateFromValuesAndScalars(Coords, masterScalars) + ) @InstancerMerger.merger(ot.Anchor) @@ -1077,15 +1079,19 @@ def merge(merger, self, lst): XCoords = [a.XCoordinate for a in lst] YCoords = [a.YCoordinate for a in lst] model = merger.model - scalars = merger.scalars - self.XCoordinate = otRound(model.interpolateFromMastersAndScalars(XCoords, scalars)) - self.YCoordinate = otRound(model.interpolateFromMastersAndScalars(YCoords, scalars)) + masterScalars = merger.masterScalars + self.XCoordinate = otRound( + model.interpolateFromValuesAndScalars(XCoords, masterScalars) + ) + self.YCoordinate = otRound( + model.interpolateFromValuesAndScalars(YCoords, masterScalars) + ) @InstancerMerger.merger(otBase.ValueRecord) def merge(merger, self, lst): model = merger.model - scalars = merger.scalars + masterScalars = merger.masterScalars # TODO Handle differing valueformats for name, tableName in [ ("XAdvance", "XAdvDevice"), @@ -1097,7 +1103,9 @@ def merge(merger, self, lst): if hasattr(self, name): values = [getattr(a, name, 0) for a in lst] - value = otRound(model.interpolateFromMastersAndScalars(values, scalars)) + value = otRound( + model.interpolateFromValuesAndScalars(values, masterScalars) + ) setattr(self, name, value) diff --git a/contrib/python/fonttools/fontTools/varLib/models.py b/contrib/python/fonttools/fontTools/varLib/models.py index 33deabe043..59815316f8 100644 --- a/contrib/python/fonttools/fontTools/varLib/models.py +++ b/contrib/python/fonttools/fontTools/varLib/models.py @@ -271,6 +271,12 @@ class VariationModel(object): self._subModels = {} def getSubModel(self, items): + """Return a sub-model and the items that are not None. + + The sub-model is necessary for working with the subset + of items when some are None. + + The sub-model is cached.""" if None not in items: return self, items key = tuple(v is not None for v in items) @@ -465,6 +471,10 @@ class VariationModel(object): return model.getDeltas(items, round=round), model.supports def getScalars(self, loc): + """Return scalars for each delta, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromMastersAndScalars().""" return [ supportScalar( loc, support, extrapolate=self.extrapolate, axisRanges=self.axisRanges @@ -472,29 +482,65 @@ class VariationModel(object): for support in self.supports ] + def getMasterScalars(self, targetLocation): + """Return multipliers for each master, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromValuesAndScalars(). + + Note that the scalars used in interpolateFromMastersAndScalars(), + are *not* the same as the ones returned here. They are the result + of getScalars().""" + out = self.getScalars(targetLocation) + for i, weights in reversed(list(enumerate(self.deltaWeights))): + for j, weight in weights.items(): + out[j] -= out[i] * weight + + out = [out[self.mapping[i]] for i in range(len(out))] + return out + @staticmethod - def interpolateFromDeltasAndScalars(deltas, scalars): + def interpolateFromValuesAndScalars(values, scalars): + """Interpolate from values and scalars coefficients. + + If the values are master-values, then the scalars should be + fetched from getMasterScalars(). + + If the values are deltas, then the scalars should be fetched + from getScalars(); in which case this is the same as + interpolateFromDeltasAndScalars(). + """ v = None - assert len(deltas) == len(scalars) - for delta, scalar in zip(deltas, scalars): + assert len(values) == len(scalars) + for value, scalar in zip(values, scalars): if not scalar: continue - contribution = delta * scalar + contribution = value * scalar if v is None: v = contribution else: v += contribution return v + @staticmethod + def interpolateFromDeltasAndScalars(deltas, scalars): + """Interpolate from deltas and scalars fetched from getScalars().""" + return VariationModel.interpolateFromValuesAndScalars(deltas, scalars) + def interpolateFromDeltas(self, loc, deltas): + """Interpolate from deltas, at location loc.""" scalars = self.getScalars(loc) return self.interpolateFromDeltasAndScalars(deltas, scalars) def interpolateFromMasters(self, loc, masterValues, *, round=noRound): - deltas = self.getDeltas(masterValues, round=round) - return self.interpolateFromDeltas(loc, deltas) + """Interpolate from master-values, at location loc.""" + scalars = self.getMasterScalars(loc) + return self.interpolateFromValuesAndScalars(masterValues, scalars) def interpolateFromMastersAndScalars(self, masterValues, scalars, *, round=noRound): + """Interpolate from master-values, and scalars fetched from + getScalars(), which is useful when you want to interpolate + multiple master-values with the same location.""" deltas = self.getDeltas(masterValues, round=round) return self.interpolateFromDeltasAndScalars(deltas, scalars) diff --git a/contrib/python/fonttools/ya.make b/contrib/python/fonttools/ya.make index c2de0cc284..6e76c94da1 100644 --- a/contrib/python/fonttools/ya.make +++ b/contrib/python/fonttools/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.46.0) +VERSION(4.47.0) LICENSE(MIT) diff --git a/contrib/python/freezegun/py3/.dist-info/METADATA b/contrib/python/freezegun/py3/.dist-info/METADATA index 0978a40fcb..d5971e9f78 100644 --- a/contrib/python/freezegun/py3/.dist-info/METADATA +++ b/contrib/python/freezegun/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: freezegun -Version: 1.3.1 +Version: 1.4.0 Summary: Let your Python tests travel through time Home-page: https://github.com/spulec/freezegun Author: Steve Pulec diff --git a/contrib/python/freezegun/py3/freezegun/__init__.py b/contrib/python/freezegun/py3/freezegun/__init__.py index 003b2bd878..8fcb18ddfe 100644 --- a/contrib/python/freezegun/py3/freezegun/__init__.py +++ b/contrib/python/freezegun/py3/freezegun/__init__.py @@ -9,7 +9,7 @@ from .api import freeze_time from .config import configure __title__ = 'freezegun' -__version__ = '1.3.1' +__version__ = '1.4.0' __author__ = 'Steve Pulec' __license__ = 'Apache License 2.0' __copyright__ = 'Copyright 2012 Steve Pulec' diff --git a/contrib/python/freezegun/py3/freezegun/api.py b/contrib/python/freezegun/py3/freezegun/api.py index f732ff8c24..2917fa1f75 100644 --- a/contrib/python/freezegun/py3/freezegun/api.py +++ b/contrib/python/freezegun/py3/freezegun/api.py @@ -544,7 +544,7 @@ class StepTickTimeFactory: class _freeze_time: - def __init__(self, time_to_freeze_str, tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds): + def __init__(self, time_to_freeze_str, tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio): self.time_to_freeze = _parse_time_to_freeze(time_to_freeze_str) self.tz_offset = _parse_tz_offset(tz_offset) self.ignore = tuple(ignore) @@ -554,6 +554,7 @@ class _freeze_time: self.modules_at_start = set() self.as_arg = as_arg self.as_kwarg = as_kwarg + self.real_asyncio = real_asyncio def __call__(self, func): if inspect.isclass(func): @@ -727,20 +728,21 @@ class _freeze_time: setattr(module, attribute_name, fake) add_change((module, attribute_name, attribute_value)) - # To avoid breaking `asyncio.sleep()`, let asyncio event loops see real - # monotonic time even though we've just frozen `time.monotonic()` which - # is normally used there. If we didn't do this, `await asyncio.sleep()` - # would be hanging forever breaking many tests that use `freeze_time`. - # - # Note that we cannot statically tell the class of asyncio event loops - # because it is not officially documented and can actually be changed - # at run time using `asyncio.set_event_loop_policy`. That's why we check - # the type by creating a loop here and destroying it immediately. - event_loop = asyncio.new_event_loop() - event_loop.close() - EventLoopClass = type(event_loop) - add_change((EventLoopClass, "time", EventLoopClass.time)) - EventLoopClass.time = lambda self: real_monotonic() + if self.real_asyncio: + # To avoid breaking `asyncio.sleep()`, let asyncio event loops see real + # monotonic time even though we've just frozen `time.monotonic()` which + # is normally used there. If we didn't do this, `await asyncio.sleep()` + # would be hanging forever breaking many tests that use `freeze_time`. + # + # Note that we cannot statically tell the class of asyncio event loops + # because it is not officially documented and can actually be changed + # at run time using `asyncio.set_event_loop_policy`. That's why we check + # the type by creating a loop here and destroying it immediately. + event_loop = asyncio.new_event_loop() + event_loop.close() + EventLoopClass = type(event_loop) + add_change((EventLoopClass, "time", EventLoopClass.time)) + EventLoopClass.time = lambda self: real_monotonic() return freeze_factory @@ -830,7 +832,7 @@ class _freeze_time: def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_arg=False, as_kwarg='', - auto_tick_seconds=0): + auto_tick_seconds=0, real_asyncio=False): acceptable_times = (type(None), str, datetime.date, datetime.timedelta, types.FunctionType, types.GeneratorType) @@ -845,14 +847,14 @@ def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_ar raise SystemError('Calling freeze_time with tick=True is only compatible with CPython') if isinstance(time_to_freeze, types.FunctionType): - return freeze_time(time_to_freeze(), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds) + return freeze_time(time_to_freeze(), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if isinstance(time_to_freeze, types.GeneratorType): - return freeze_time(next(time_to_freeze), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds) + return freeze_time(next(time_to_freeze), tz_offset, ignore, tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if MayaDT is not None and isinstance(time_to_freeze, MayaDT): return freeze_time(time_to_freeze.datetime(), tz_offset, ignore, - tick, as_arg, as_kwarg, auto_tick_seconds) + tick, as_arg, as_kwarg, auto_tick_seconds, real_asyncio=real_asyncio) if ignore is None: ignore = [] @@ -868,6 +870,7 @@ def freeze_time(time_to_freeze=None, tz_offset=0, ignore=None, tick=False, as_ar as_arg=as_arg, as_kwarg=as_kwarg, auto_tick_seconds=auto_tick_seconds, + real_asyncio=real_asyncio, ) diff --git a/contrib/python/freezegun/py3/ya.make b/contrib/python/freezegun/py3/ya.make index 4c2d15c637..d6c429a154 100644 --- a/contrib/python/freezegun/py3/ya.make +++ b/contrib/python/freezegun/py3/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() PROVIDES(freezegun) -VERSION(1.3.1) +VERSION(1.4.0) LICENSE(Apache-2.0) diff --git a/contrib/python/httpx/.dist-info/METADATA b/contrib/python/httpx/.dist-info/METADATA index 247ac2e118..ab2b707fcd 100644 --- a/contrib/python/httpx/.dist-info/METADATA +++ b/contrib/python/httpx/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: httpx -Version: 0.25.2 +Version: 0.26.0 Summary: The next generation HTTP client. Project-URL: Changelog, https://github.com/encode/httpx/blob/master/CHANGELOG.md Project-URL: Documentation, https://www.python-httpx.org @@ -31,8 +31,8 @@ Requires-Dist: httpcore==1.* Requires-Dist: idna Requires-Dist: sniffio Provides-Extra: brotli -Requires-Dist: brotli; platform_python_implementation == 'CPython' and extra == 'brotli' -Requires-Dist: brotlicffi; platform_python_implementation != 'CPython' and extra == 'brotli' +Requires-Dist: brotli; (platform_python_implementation == 'CPython') and extra == 'brotli' +Requires-Dist: brotlicffi; (platform_python_implementation != 'CPython') and extra == 'brotli' Provides-Extra: cli Requires-Dist: click==8.*; extra == 'cli' Requires-Dist: pygments==2.*; extra == 'cli' @@ -196,7 +196,19 @@ inspiration around the lower-level networking details. ### Added -* Add missing type hints to few `__init__()` methods. (#2938) +* The `proxy` argument was added. You should use the `proxy` argument instead of the deprecated `proxies`, or use `mounts=` for more complex configurations. (#2879) + +### Deprecated + +* The `proxies` argument is now deprecated. It will still continue to work, but it will be removed in the future. (#2879) + +### Fixed + +* Fix cases of double escaping of URL path components. Allow / as a safe character in the query portion. (#2990) +* Handle `NO_PROXY` envvar cases when a fully qualified URL is supplied as the value. (#2741) +* Allow URLs where username or password contains unescaped '@'. (#2986) +* Ensure ASGI `raw_path` does not include URL query component. (#2999) +* Ensure `Response.iter_text()` cannot yield empty strings. (#2998) --- diff --git a/contrib/python/httpx/httpx/__version__.py b/contrib/python/httpx/httpx/__version__.py index c6bc0ac023..3edc842c69 100644 --- a/contrib/python/httpx/httpx/__version__.py +++ b/contrib/python/httpx/httpx/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx" __description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.25.2" +__version__ = "0.26.0" diff --git a/contrib/python/httpx/httpx/_api.py b/contrib/python/httpx/httpx/_api.py index 571289cf2b..c7af947218 100644 --- a/contrib/python/httpx/httpx/_api.py +++ b/contrib/python/httpx/httpx/_api.py @@ -10,6 +10,7 @@ from ._types import ( CookieTypes, HeaderTypes, ProxiesTypes, + ProxyTypes, QueryParamTypes, RequestContent, RequestData, @@ -32,6 +33,7 @@ def request( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -63,6 +65,7 @@ def request( request. * **auth** - *(optional)* An authentication class to use when sending the request. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending the request. @@ -91,6 +94,7 @@ def request( """ with Client( cookies=cookies, + proxy=proxy, proxies=proxies, cert=cert, verify=verify, @@ -124,6 +128,7 @@ def stream( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -143,6 +148,7 @@ def stream( """ with Client( cookies=cookies, + proxy=proxy, proxies=proxies, cert=cert, verify=verify, @@ -171,6 +177,7 @@ def get( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -193,6 +200,7 @@ def get( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -209,6 +217,7 @@ def options( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -231,6 +240,7 @@ def options( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -247,6 +257,7 @@ def head( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -269,6 +280,7 @@ def head( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -289,6 +301,7 @@ def post( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -312,6 +325,7 @@ def post( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -332,6 +346,7 @@ def put( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -355,6 +370,7 @@ def put( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -375,6 +391,7 @@ def patch( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -398,6 +415,7 @@ def patch( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, @@ -414,6 +432,7 @@ def delete( headers: typing.Optional[HeaderTypes] = None, cookies: typing.Optional[CookieTypes] = None, auth: typing.Optional[AuthTypes] = None, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, follow_redirects: bool = False, cert: typing.Optional[CertTypes] = None, @@ -436,6 +455,7 @@ def delete( headers=headers, cookies=cookies, auth=auth, + proxy=proxy, proxies=proxies, follow_redirects=follow_redirects, cert=cert, diff --git a/contrib/python/httpx/httpx/_client.py b/contrib/python/httpx/httpx/_client.py index 2c7ae030f5..2813a84f01 100644 --- a/contrib/python/httpx/httpx/_client.py +++ b/contrib/python/httpx/httpx/_client.py @@ -36,6 +36,7 @@ from ._types import ( CookieTypes, HeaderTypes, ProxiesTypes, + ProxyTypes, QueryParamTypes, RequestContent, RequestData, @@ -597,6 +598,7 @@ class Client(BaseClient): to authenticate the client. Either a path to an SSL certificate file, or two-tuple of (certificate file, key file), or a three-tuple of (certificate file, key file, password). + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending @@ -628,8 +630,11 @@ class Client(BaseClient): cert: typing.Optional[CertTypes] = None, http1: bool = True, http2: bool = False, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, - mounts: typing.Optional[typing.Mapping[str, BaseTransport]] = None, + mounts: typing.Optional[ + typing.Mapping[str, typing.Optional[BaseTransport]] + ] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -666,8 +671,17 @@ class Client(BaseClient): "Make sure to install httpx using `pip install httpx[http2]`." ) from None + if proxies: + message = ( + "The 'proxies' argument is now deprecated." + " Use 'proxy' or 'mounts' instead." + ) + warnings.warn(message, DeprecationWarning) + if proxy: + raise RuntimeError("Use either `proxy` or 'proxies', not both.") + allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, @@ -1264,7 +1278,9 @@ class Client(BaseClient): if self._state != ClientState.UNOPENED: msg = { ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: "Cannot reopen a client instance, once it has been closed.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), }[self._state] raise RuntimeError(msg) @@ -1322,6 +1338,7 @@ class AsyncClient(BaseClient): file, key file, password). * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending @@ -1353,8 +1370,11 @@ class AsyncClient(BaseClient): cert: typing.Optional[CertTypes] = None, http1: bool = True, http2: bool = False, + proxy: typing.Optional[ProxyTypes] = None, proxies: typing.Optional[ProxiesTypes] = None, - mounts: typing.Optional[typing.Mapping[str, AsyncBaseTransport]] = None, + mounts: typing.Optional[ + typing.Mapping[str, typing.Optional[AsyncBaseTransport]] + ] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -1391,8 +1411,17 @@ class AsyncClient(BaseClient): "Make sure to install httpx using `pip install httpx[http2]`." ) from None + if proxies: + message = ( + "The 'proxies' argument is now deprecated." + " Use 'proxy' or 'mounts' instead." + ) + warnings.warn(message, DeprecationWarning) + if proxy: + raise RuntimeError("Use either `proxy` or 'proxies', not both.") + allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, @@ -1980,7 +2009,9 @@ class AsyncClient(BaseClient): if self._state != ClientState.UNOPENED: msg = { ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: "Cannot reopen a client instance, once it has been closed.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), }[self._state] raise RuntimeError(msg) diff --git a/contrib/python/httpx/httpx/_content.py b/contrib/python/httpx/httpx/_content.py index 0aaea33749..cd0d17f171 100644 --- a/contrib/python/httpx/httpx/_content.py +++ b/contrib/python/httpx/httpx/_content.py @@ -105,7 +105,7 @@ class UnattachedStream(AsyncByteStream, SyncByteStream): def encode_content( - content: Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] + content: Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]], ) -> Tuple[Dict[str, str], Union[SyncByteStream, AsyncByteStream]]: if isinstance(content, (bytes, str)): body = content.encode("utf-8") if isinstance(content, str) else content diff --git a/contrib/python/httpx/httpx/_decoders.py b/contrib/python/httpx/httpx/_decoders.py index b4ac9a44af..3f507c8e04 100644 --- a/contrib/python/httpx/httpx/_decoders.py +++ b/contrib/python/httpx/httpx/_decoders.py @@ -212,7 +212,7 @@ class TextChunker: def decode(self, content: str) -> typing.List[str]: if self._chunk_size is None: - return [content] + return [content] if content else [] self._buffer.write(content) if self._buffer.tell() >= self._chunk_size: @@ -259,7 +259,8 @@ class LineDecoder: """ Handles incrementally reading lines from text. - Has the same behaviour as the stdllib splitlines, but handling the input iteratively. + Has the same behaviour as the stdllib splitlines, + but handling the input iteratively. """ def __init__(self) -> None: @@ -279,7 +280,9 @@ class LineDecoder: text = text[:-1] if not text: - return [] + # NOTE: the edge case input of empty text doesn't occur in practice, + # because other httpx internals filter out this value + return [] # pragma: no cover trailing_newline = text[-1] in NEWLINE_CHARS lines = text.splitlines() diff --git a/contrib/python/httpx/httpx/_exceptions.py b/contrib/python/httpx/httpx/_exceptions.py index 24a4f8aba3..123692955b 100644 --- a/contrib/python/httpx/httpx/_exceptions.py +++ b/contrib/python/httpx/httpx/_exceptions.py @@ -313,7 +313,10 @@ class ResponseNotRead(StreamError): """ def __init__(self) -> None: - message = "Attempted to access streaming response content, without having called `read()`." + message = ( + "Attempted to access streaming response content," + " without having called `read()`." + ) super().__init__(message) @@ -323,7 +326,10 @@ class RequestNotRead(StreamError): """ def __init__(self) -> None: - message = "Attempted to access streaming request content, without having called `read()`." + message = ( + "Attempted to access streaming request content," + " without having called `read()`." + ) super().__init__(message) diff --git a/contrib/python/httpx/httpx/_main.py b/contrib/python/httpx/httpx/_main.py index 7c12ce841d..adb57d5fc0 100644 --- a/contrib/python/httpx/httpx/_main.py +++ b/contrib/python/httpx/httpx/_main.py @@ -63,20 +63,21 @@ def print_help() -> None: ) table.add_row( "--auth [cyan]<USER PASS>", - "Username and password to include in the request. Specify '-' for the password to use " - "a password prompt. Note that using --verbose/-v will expose the Authorization " - "header, including the password encoding in a trivially reversible format.", + "Username and password to include in the request. Specify '-' for the password" + " to use a password prompt. Note that using --verbose/-v will expose" + " the Authorization header, including the password encoding" + " in a trivially reversible format.", ) table.add_row( - "--proxies [cyan]URL", + "--proxy [cyan]URL", "Send the request via a proxy. Should be the URL giving the proxy address.", ) table.add_row( "--timeout [cyan]FLOAT", - "Timeout value to use for network operations, such as establishing the connection, " - "reading some data, etc... [Default: 5.0]", + "Timeout value to use for network operations, such as establishing the" + " connection, reading some data, etc... [Default: 5.0]", ) table.add_row("--follow-redirects", "Automatically follow redirects.") @@ -385,8 +386,8 @@ def handle_help( ), ) @click.option( - "--proxies", - "proxies", + "--proxy", + "proxy", type=str, default=None, help="Send the request via a proxy. Should be the URL giving the proxy address.", @@ -455,7 +456,7 @@ def main( headers: typing.List[typing.Tuple[str, str]], cookies: typing.List[typing.Tuple[str, str]], auth: typing.Optional[typing.Tuple[str, str]], - proxies: str, + proxy: str, timeout: float, follow_redirects: bool, verify: bool, @@ -472,7 +473,7 @@ def main( try: with Client( - proxies=proxies, + proxy=proxy, timeout=timeout, verify=verify, http2=http2, diff --git a/contrib/python/httpx/httpx/_models.py b/contrib/python/httpx/httpx/_models.py index 8a5e6280f3..b8617cdab5 100644 --- a/contrib/python/httpx/httpx/_models.py +++ b/contrib/python/httpx/httpx/_models.py @@ -358,7 +358,8 @@ class Request: # Using `content=...` implies automatically populated `Host` and content # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`. # - # Using `stream=...` will not automatically include *any* auto-populated headers. + # Using `stream=...` will not automatically include *any* + # auto-populated headers. # # As an end-user you don't really need `stream=...`. It's only # useful when: @@ -852,7 +853,7 @@ class Response: yield chunk text_content = decoder.flush() for chunk in chunker.decode(text_content): - yield chunk + yield chunk # pragma: no cover for chunk in chunker.flush(): yield chunk @@ -956,7 +957,7 @@ class Response: yield chunk text_content = decoder.flush() for chunk in chunker.decode(text_content): - yield chunk + yield chunk # pragma: no cover for chunk in chunker.flush(): yield chunk diff --git a/contrib/python/httpx/httpx/_multipart.py b/contrib/python/httpx/httpx/_multipart.py index 6d5baa8639..5122d5114f 100644 --- a/contrib/python/httpx/httpx/_multipart.py +++ b/contrib/python/httpx/httpx/_multipart.py @@ -48,7 +48,8 @@ class DataField: ) if value is not None and not isinstance(value, (str, bytes, int, float)): raise TypeError( - f"Invalid type for value. Expected primitive type, got {type(value)}: {value!r}" + "Invalid type for value. Expected primitive type," + f" got {type(value)}: {value!r}" ) self.name = name self.value: typing.Union[str, bytes] = ( @@ -96,11 +97,13 @@ class FileField: content_type: typing.Optional[str] = None # This large tuple based API largely mirror's requests' API - # It would be good to think of better APIs for this that we could include in httpx 2.0 - # since variable length tuples (especially of 4 elements) are quite unwieldly + # It would be good to think of better APIs for this that we could + # include in httpx 2.0 since variable length tuples(especially of 4 elements) + # are quite unwieldly if isinstance(value, tuple): if len(value) == 2: - # neither the 3rd parameter (content_type) nor the 4th (headers) was included + # neither the 3rd parameter (content_type) nor the 4th (headers) + # was included filename, fileobj = value # type: ignore elif len(value) == 3: filename, fileobj, content_type = value # type: ignore @@ -116,9 +119,9 @@ class FileField: has_content_type_header = any("content-type" in key.lower() for key in headers) if content_type is not None and not has_content_type_header: - # note that unlike requests, we ignore the content_type - # provided in the 3rd tuple element if it is also included in the headers - # requests does the opposite (it overwrites the header with the 3rd tuple element) + # note that unlike requests, we ignore the content_type provided in the 3rd + # tuple element if it is also included in the headers requests does + # the opposite (it overwrites the headerwith the 3rd tuple element) headers["Content-Type"] = content_type if isinstance(fileobj, io.StringIO): diff --git a/contrib/python/httpx/httpx/_transports/asgi.py b/contrib/python/httpx/httpx/_transports/asgi.py index f67f0fbd5b..08cd392f75 100644 --- a/contrib/python/httpx/httpx/_transports/asgi.py +++ b/contrib/python/httpx/httpx/_transports/asgi.py @@ -103,7 +103,7 @@ class ASGITransport(AsyncBaseTransport): "headers": [(k.lower(), v) for (k, v) in request.headers.raw], "scheme": request.url.scheme, "path": request.url.path, - "raw_path": request.url.raw_path, + "raw_path": request.url.raw_path.split(b"?")[0], "query_string": request.url.query, "server": (request.url.host, request.url.port), "client": self.client, diff --git a/contrib/python/httpx/httpx/_transports/default.py b/contrib/python/httpx/httpx/_transports/default.py index 343c588f9f..14a087389a 100644 --- a/contrib/python/httpx/httpx/_transports/default.py +++ b/contrib/python/httpx/httpx/_transports/default.py @@ -47,7 +47,8 @@ from .._exceptions import ( WriteTimeout, ) from .._models import Request, Response -from .._types import AsyncByteStream, CertTypes, SyncByteStream, VerifyTypes +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream, VerifyTypes +from .._urls import URL from .base import AsyncBaseTransport, BaseTransport T = typing.TypeVar("T", bound="HTTPTransport") @@ -124,13 +125,14 @@ class HTTPTransport(BaseTransport): http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - proxy: typing.Optional[Proxy] = None, + proxy: typing.Optional[ProxyTypes] = None, uds: typing.Optional[str] = None, local_address: typing.Optional[str] = None, retries: int = 0, socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if proxy is None: self._pool = httpcore.ConnectionPool( @@ -190,7 +192,8 @@ class HTTPTransport(BaseTransport): ) else: # pragma: no cover raise ValueError( - f"Proxy protocol must be either 'http', 'https', or 'socks5', but got {proxy.url.scheme!r}." + "Proxy protocol must be either 'http', 'https', or 'socks5'," + f" but got {proxy.url.scheme!r}." ) def __enter__(self: T) -> T: # Use generics for subclass support. @@ -263,13 +266,14 @@ class AsyncHTTPTransport(AsyncBaseTransport): http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - proxy: typing.Optional[Proxy] = None, + proxy: typing.Optional[ProxyTypes] = None, uds: typing.Optional[str] = None, local_address: typing.Optional[str] = None, retries: int = 0, socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if proxy is None: self._pool = httpcore.AsyncConnectionPool( @@ -328,7 +332,8 @@ class AsyncHTTPTransport(AsyncBaseTransport): ) else: # pragma: no cover raise ValueError( - f"Proxy protocol must be either 'http', 'https', or 'socks5', but got {proxy.url.scheme!r}." + "Proxy protocol must be either 'http', 'https', or 'socks5'," + " but got {proxy.url.scheme!r}." ) async def __aenter__(self: A) -> A: # Use generics for subclass support. diff --git a/contrib/python/httpx/httpx/_types.py b/contrib/python/httpx/httpx/_types.py index 83cf35a32a..649d101d54 100644 --- a/contrib/python/httpx/httpx/_types.py +++ b/contrib/python/httpx/httpx/_types.py @@ -78,7 +78,8 @@ TimeoutTypes = Union[ Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], "Timeout", ] -ProxiesTypes = Union[URLTypes, "Proxy", Dict[URLTypes, Union[None, URLTypes, "Proxy"]]] +ProxyTypes = Union[URLTypes, "Proxy"] +ProxiesTypes = Union[ProxyTypes, Dict[URLTypes, Union[None, ProxyTypes]]] AuthTypes = Union[ Tuple[Union[str, bytes], Union[str, bytes]], diff --git a/contrib/python/httpx/httpx/_urlparse.py b/contrib/python/httpx/httpx/_urlparse.py index 8e060424e8..07bbea9070 100644 --- a/contrib/python/httpx/httpx/_urlparse.py +++ b/contrib/python/httpx/httpx/_urlparse.py @@ -62,8 +62,8 @@ AUTHORITY_REGEX = re.compile( ( r"(?:(?P<userinfo>{userinfo})@)?" r"(?P<host>{host})" r":?(?P<port>{port})?" ).format( - userinfo="[^@]*", # Any character sequence not including '@'. - host="(\\[.*\\]|[^:]*)", # Either any character sequence not including ':', + userinfo=".*", # Any character sequence. + host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@', # or an IPv6 address enclosed within square brackets. port=".*", # Any character sequence. ) @@ -260,10 +260,8 @@ def urlparse(url: str = "", **kwargs: typing.Optional[str]) -> ParseResult: # For 'path' we need to drop ? and # from the GEN_DELIMS set. parsed_path: str = quote(path, safe=SUB_DELIMS + ":/[]@") # For 'query' we need to drop '#' from the GEN_DELIMS set. - # We also exclude '/' because it is more robust to replace it with a percent - # encoding despite it not being a requirement of the spec. parsed_query: typing.Optional[str] = ( - None if query is None else quote(query, safe=SUB_DELIMS + ":?[]@") + None if query is None else quote(query, safe=SUB_DELIMS + ":/?[]@") ) # For 'fragment' we can include all of the GEN_DELIMS set. parsed_fragment: typing.Optional[str] = ( @@ -360,24 +358,25 @@ def normalize_port( def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None: """ - Path validation rules that depend on if the URL contains a scheme or authority component. + Path validation rules that depend on if the URL contains + a scheme or authority component. See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3 """ if has_authority: - # > If a URI contains an authority component, then the path component - # > must either be empty or begin with a slash ("/") character." + # If a URI contains an authority component, then the path component + # must either be empty or begin with a slash ("/") character." if path and not path.startswith("/"): raise InvalidURL("For absolute URLs, path must be empty or begin with '/'") else: - # > If a URI does not contain an authority component, then the path cannot begin - # > with two slash characters ("//"). + # If a URI does not contain an authority component, then the path cannot begin + # with two slash characters ("//"). if path.startswith("//"): raise InvalidURL( "URLs with no authority component cannot have a path starting with '//'" ) - # > In addition, a URI reference (Section 4.1) may be a relative-path reference, in which - # > case the first path segment cannot contain a colon (":") character. + # In addition, a URI reference (Section 4.1) may be a relative-path reference, + # in which case the first path segment cannot contain a colon (":") character. if path.startswith(":") and not has_scheme: raise InvalidURL( "URLs with no scheme component cannot have a path starting with ':'" @@ -431,13 +430,12 @@ def is_safe(string: str, safe: str = "/") -> bool: if char not in NON_ESCAPED_CHARS: return False - # Any '%' characters must be valid '%xx' escape sequences. - return string.count("%") == len(PERCENT_ENCODED_REGEX.findall(string)) + return True -def quote(string: str, safe: str = "/") -> str: +def percent_encoded(string: str, safe: str = "/") -> str: """ - Use percent-encoding to quote a string if required. + Use percent-encoding to quote a string. """ if is_safe(string, safe=safe): return string @@ -448,17 +446,57 @@ def quote(string: str, safe: str = "/") -> str: ) +def quote(string: str, safe: str = "/") -> str: + """ + Use percent-encoding to quote a string, omitting existing '%xx' escape sequences. + + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1 + + * `string`: The string to be percent-escaped. + * `safe`: A string containing characters that may be treated as safe, and do not + need to be escaped. Unreserved characters are always treated as safe. + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3 + """ + parts = [] + current_position = 0 + for match in re.finditer(PERCENT_ENCODED_REGEX, string): + start_position, end_position = match.start(), match.end() + matched_text = match.group(0) + # Add any text up to the '%xx' escape sequence. + if start_position != current_position: + leading_text = string[current_position:start_position] + parts.append(percent_encoded(leading_text, safe=safe)) + + # Add the '%xx' escape sequence. + parts.append(matched_text) + current_position = end_position + + # Add any text after the final '%xx' escape sequence. + if current_position != len(string): + trailing_text = string[current_position:] + parts.append(percent_encoded(trailing_text, safe=safe)) + + return "".join(parts) + + def urlencode(items: typing.List[typing.Tuple[str, str]]) -> str: - # We can use a much simpler version of the stdlib urlencode here because - # we don't need to handle a bunch of different typing cases, such as bytes vs str. - # - # https://github.com/python/cpython/blob/b2f7b2ef0b5421e01efb8c7bee2ef95d3bab77eb/Lib/urllib/parse.py#L926 - # - # Note that we use '%20' encoding for spaces. and '%2F for '/'. - # This is slightly different than `requests`, but is the behaviour that browsers use. - # - # See - # - https://github.com/encode/httpx/issues/2536 - # - https://github.com/encode/httpx/issues/2721 - # - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode - return "&".join([quote(k, safe="") + "=" + quote(v, safe="") for k, v in items]) + """ + We can use a much simpler version of the stdlib urlencode here because + we don't need to handle a bunch of different typing cases, such as bytes vs str. + + https://github.com/python/cpython/blob/b2f7b2ef0b5421e01efb8c7bee2ef95d3bab77eb/Lib/urllib/parse.py#L926 + + Note that we use '%20' encoding for spaces. and '%2F for '/'. + This is slightly different than `requests`, but is the behaviour that browsers use. + + See + - https://github.com/encode/httpx/issues/2536 + - https://github.com/encode/httpx/issues/2721 + - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode + """ + return "&".join( + [ + percent_encoded(k, safe="") + "=" + percent_encoded(v, safe="") + for k, v in items + ] + ) diff --git a/contrib/python/httpx/httpx/_urls.py b/contrib/python/httpx/httpx/_urls.py index b023941b62..26202e95db 100644 --- a/contrib/python/httpx/httpx/_urls.py +++ b/contrib/python/httpx/httpx/_urls.py @@ -51,21 +51,23 @@ class URL: assert url.raw_host == b"xn--fiqs8s.icom.museum" * `url.port` is either None or an integer. URLs that include the default port for - "http", "https", "ws", "wss", and "ftp" schemes have their port normalized to `None`. + "http", "https", "ws", "wss", and "ftp" schemes have their port + normalized to `None`. assert httpx.URL("http://example.com") == httpx.URL("http://example.com:80") assert httpx.URL("http://example.com").port is None assert httpx.URL("http://example.com:80").port is None - * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work with - `url.username` and `url.password` instead, which handle the URL escaping. + * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work + with `url.username` and `url.password` instead, which handle the URL escaping. * `url.raw_path` is raw bytes of both the path and query, without URL escaping. This portion is used as the target when constructing HTTP requests. Usually you'll want to work with `url.path` instead. - * `url.query` is raw bytes, without URL escaping. A URL query string portion can only - be properly URL escaped when decoding the parameter names and values themselves. + * `url.query` is raw bytes, without URL escaping. A URL query string portion can + only be properly URL escaped when decoding the parameter names and values + themselves. """ def __init__( @@ -115,7 +117,8 @@ class URL: self._uri_reference = url._uri_reference.copy_with(**kwargs) else: raise TypeError( - f"Invalid type for url. Expected str or httpx.URL, got {type(url)}: {url!r}" + "Invalid type for url. Expected str or httpx.URL," + f" got {type(url)}: {url!r}" ) @property @@ -305,7 +308,8 @@ class URL: Provides the (scheme, host, port, target) for the outgoing request. In older versions of `httpx` this was used in the low-level transport API. - We no longer use `RawURL`, and this property will be deprecated in a future release. + We no longer use `RawURL`, and this property will be deprecated + in a future release. """ return RawURL( self.raw_scheme, @@ -342,7 +346,9 @@ class URL: For example: - url = httpx.URL("https://www.example.com").copy_with(username="jo@gmail.com", password="a secret") + url = httpx.URL("https://www.example.com").copy_with( + username="jo@gmail.com", password="a secret" + ) assert url == "https://jo%40email.com:a%20secret@www.example.com" """ return URL(self, **kwargs) diff --git a/contrib/python/httpx/httpx/_utils.py b/contrib/python/httpx/httpx/_utils.py index ba5807c048..bc3cb001dd 100644 --- a/contrib/python/httpx/httpx/_utils.py +++ b/contrib/python/httpx/httpx/_utils.py @@ -152,7 +152,7 @@ SENSITIVE_HEADERS = {"authorization", "proxy-authorization"} def obfuscate_sensitive_headers( - items: typing.Iterable[typing.Tuple[typing.AnyStr, typing.AnyStr]] + items: typing.Iterable[typing.Tuple[typing.AnyStr, typing.AnyStr]], ) -> typing.Iterator[typing.Tuple[typing.AnyStr, typing.AnyStr]]: for k, v in items: if to_str(k.lower()) in SENSITIVE_HEADERS: @@ -227,7 +227,9 @@ def get_environment_proxies() -> typing.Dict[str, typing.Optional[str]]: # (But not "wwwgoogle.com") # NO_PROXY can include domains, IPv6, IPv4 addresses and "localhost" # NO_PROXY=example.com,::1,localhost,192.168.0.0/16 - if is_ipv4_hostname(hostname): + if "://" in hostname: + mounts[hostname] = None + elif is_ipv4_hostname(hostname): mounts[f"all://{hostname}"] = None elif is_ipv6_hostname(hostname): mounts[f"all://[{hostname}]"] = None @@ -293,14 +295,10 @@ class Timer: import trio return trio.current_time() - elif library == "curio": # pragma: no cover - import curio - - return typing.cast(float, await curio.clock()) - - import asyncio + else: + import asyncio - return asyncio.get_event_loop().time() + return asyncio.get_event_loop().time() def sync_start(self) -> None: self.started = time.perf_counter() diff --git a/contrib/python/httpx/ya.make b/contrib/python/httpx/ya.make index f07d3a08ce..2f8eaa9087 100644 --- a/contrib/python/httpx/ya.make +++ b/contrib/python/httpx/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.25.2) +VERSION(0.26.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index c243f74513..b70d38e2c2 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.92.0 +Version: 6.92.1 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py index 3eaed1eba1..41e8ce61d1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/compat.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/compat.py @@ -9,6 +9,8 @@ # obtain one at https://mozilla.org/MPL/2.0/. import codecs +import copy +import dataclasses import inspect import platform import sys @@ -188,3 +190,46 @@ def bad_django_TestCase(runner): from hypothesis.extra.django._impl import HypothesisTestCase return not isinstance(runner, HypothesisTestCase) + + +# see issue #3812 +if sys.version_info[:2] < (3, 12): + + def dataclass_asdict(obj, *, dict_factory=dict): + """ + A vendored variant of dataclasses.asdict. Includes the bugfix for + defaultdicts (cpython/32056) for all versions. See also issues/3812. + + This should be removed whenever we drop support for 3.11. We can use the + standard dataclasses.asdict after that point. + """ + if not dataclasses._is_dataclass_instance(obj): # pragma: no cover + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + +else: # pragma: no cover + dataclass_asdict = dataclasses.asdict + + +def _asdict_inner(obj, dict_factory): + if dataclasses._is_dataclass_instance(obj): + return dict_factory( + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in dataclasses.fields(obj) + ) + elif isinstance(obj, tuple) and hasattr(obj, "_fields"): + return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + if hasattr(type(obj), "default_factory"): + result = type(obj)(obj.default_factory) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result + return type(obj)( + (_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) + for k, v in obj.items() + ) + else: + return copy.deepcopy(obj) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py index a5a862635a..03653c61e1 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py @@ -77,6 +77,7 @@ from hypothesis.internal.compat import ( from hypothesis.internal.conjecture.utils import calc_label_from_cls, check_sample from hypothesis.internal.entropy import get_seeder_and_restorer from hypothesis.internal.floats import float_of +from hypothesis.internal.observability import TESTCASE_CALLBACKS from hypothesis.internal.reflection import ( define_function_signature, get_pretty_function_description, @@ -2103,7 +2104,9 @@ class DataObject: self.count += 1 printer = RepresentationPrinter(context=current_build_context()) desc = f"Draw {self.count}{'' if label is None else f' ({label})'}: " - self.conjecture_data._observability_args[desc] = to_jsonable(result) + if TESTCASE_CALLBACKS: + self.conjecture_data._observability_args[desc] = to_jsonable(result) + printer.text(desc) printer.pretty(result) note(printer.getvalue()) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py index 995b179b40..b2a7661cd6 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/utils.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Callable, Dict import attr from hypothesis.internal.cache import LRUReusedCache +from hypothesis.internal.compat import dataclass_asdict from hypothesis.internal.floats import float_to_int from hypothesis.internal.reflection import proxies from hypothesis.vendor.pretty import pretty @@ -177,7 +178,7 @@ def to_jsonable(obj: object) -> object: and dcs.is_dataclass(obj) and not isinstance(obj, type) ): - return to_jsonable(dcs.asdict(obj)) + return to_jsonable(dataclass_asdict(obj)) if attr.has(type(obj)): return to_jsonable(attr.asdict(obj, recurse=False)) # type: ignore if (pyd := sys.modules.get("pydantic")) and isinstance(obj, pyd.BaseModel): diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index 8357097704..937332138d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 92, 0) +__version_info__ = (6, 92, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index 456dabd865..80e21c22c2 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.92.0) +VERSION(6.92.1) LICENSE(MPL-2.0) diff --git a/contrib/python/importlib-metadata/py3/.dist-info/METADATA b/contrib/python/importlib-metadata/py3/.dist-info/METADATA index a344859f17..c2bab66615 100644 --- a/contrib/python/importlib-metadata/py3/.dist-info/METADATA +++ b/contrib/python/importlib-metadata/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: importlib-metadata -Version: 7.0.0 +Version: 7.0.1 Summary: Read metadata from Python packages Home-page: https://github.com/python/importlib_metadata Author: Jason R. Coombs @@ -42,7 +42,7 @@ Requires-Dist: importlib-resources >=1.3 ; (python_version < "3.9") and extra == .. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg -.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg +.. image:: https://github.com/python/importlib_metadata/actions/workflows/main.yml/badge.svg :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22 :alt: tests @@ -82,6 +82,8 @@ were contributed to different versions in the standard library: * - importlib_metadata - stdlib + * - 7.0 + - 3.13 * - 6.5 - 3.12 * - 4.13 diff --git a/contrib/python/importlib-metadata/py3/README.rst b/contrib/python/importlib-metadata/py3/README.rst index 73ddc5fab4..6788a7e3d3 100644 --- a/contrib/python/importlib-metadata/py3/README.rst +++ b/contrib/python/importlib-metadata/py3/README.rst @@ -3,7 +3,7 @@ .. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg -.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg +.. image:: https://github.com/python/importlib_metadata/actions/workflows/main.yml/badge.svg :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22 :alt: tests @@ -43,6 +43,8 @@ were contributed to different versions in the standard library: * - importlib_metadata - stdlib + * - 7.0 + - 3.13 * - 6.5 - 3.12 * - 4.13 diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py index cd015707b1..4debbecb9d 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import abc @@ -20,7 +22,6 @@ from . import _adapters, _meta, _py39compat from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, - StrPath, install, ) from ._functools import method_cache, pass_none @@ -320,14 +321,12 @@ class PackagePath(pathlib.PurePosixPath): dist: "Distribution" def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] - with self.locate().open(encoding=encoding) as stream: - return stream.read() + return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: - with self.locate().open('rb') as stream: - return stream.read() + return self.locate().read_bytes() - def locate(self) -> pathlib.Path: + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) @@ -341,6 +340,7 @@ class FileHash: class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -360,20 +360,42 @@ class DeprecatedNonAbstract: class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points defined by + the Setuptools spec in an ini format with sections + representing the groups. + - RECORD: A record of files as installed by a typical + installer. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path: StrPath) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @@ -396,16 +418,18 @@ class Distribution(DeprecatedNonAbstract): raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs) -> Iterable["Distribution"]: + def discover( + cls, *, context: Optional['DistributionFinder.Context'] = None, **kwargs + ) -> Iterable["Distribution"]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -414,8 +438,8 @@ class Distribution(DeprecatedNonAbstract): ) @staticmethod - def at(path: StrPath) -> "Distribution": - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> "Distribution": + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -424,7 +448,7 @@ class Distribution(DeprecatedNonAbstract): @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -822,7 +846,7 @@ class PathDistribution(Distribution): """ self._path = path - def read_text(self, filename: StrPath) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -836,7 +860,7 @@ class PathDistribution(Distribution): read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path: StrPath) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py b/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py index a48b609be6..1a3578e60f 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/_compat.py @@ -1,9 +1,6 @@ -import os import sys import platform -from typing import Union - __all__ = ['install', 'NullFinder'] @@ -44,7 +41,7 @@ def disable_stdlib_finder(): class NullFinder: """ - A "Finder" (aka "MetaClassFinder") that never finds any modules, + A "Finder" (aka "MetaPathFinder") that never finds any modules, but may find distributions. """ @@ -61,10 +58,3 @@ def pypy_partial(val): """ is_pypy = platform.python_implementation() == 'PyPy' return val + is_pypy - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] # pragma: no cover diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py b/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py index f670016de7..1342d839ba 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -44,20 +47,26 @@ class PackageMetadata(Protocol): """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self, other: Union[str, _T]) -> _T: + def joinpath(self, other: Union[str, os.PathLike[str]]) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: + def __truediv__(self, other: Union[str, os.PathLike[str]]) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: + def parent(self) -> SimplePath: + ... # pragma: no cover + + def read_text(self, encoding=None) -> str: + ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: + def exists(self) -> bool: ... # pragma: no cover diff --git a/contrib/python/importlib-metadata/py3/ya.make b/contrib/python/importlib-metadata/py3/ya.make index 4e4813824c..f967f7b41b 100644 --- a/contrib/python/importlib-metadata/py3/ya.make +++ b/contrib/python/importlib-metadata/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.0.0) +VERSION(7.0.1) LICENSE(Apache-2.0) diff --git a/contrib/python/ipython/py3/.dist-info/METADATA b/contrib/python/ipython/py3/.dist-info/METADATA index ded1f01e88..ff27b2e371 100644 --- a/contrib/python/ipython/py3/.dist-info/METADATA +++ b/contrib/python/ipython/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ipython -Version: 8.18.1 +Version: 8.19.0 Summary: IPython: Productive Interactive Computing Home-page: https://ipython.org Author: The IPython Development Team @@ -23,7 +23,7 @@ Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Topic :: System :: Shells -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: decorator @@ -46,10 +46,9 @@ Requires-Dist: sphinx-rtd-theme ; extra == 'all' Requires-Dist: docrepr ; extra == 'all' Requires-Dist: matplotlib ; extra == 'all' Requires-Dist: stack-data ; extra == 'all' -Requires-Dist: pytest <7 ; extra == 'all' +Requires-Dist: pytest ; extra == 'all' Requires-Dist: typing-extensions ; extra == 'all' Requires-Dist: exceptiongroup ; extra == 'all' -Requires-Dist: pytest <7.1 ; extra == 'all' Requires-Dist: pytest-asyncio <0.22 ; extra == 'all' Requires-Dist: testpath ; extra == 'all' Requires-Dist: pickleshare ; extra == 'all' @@ -61,7 +60,7 @@ Requires-Dist: ipyparallel ; extra == 'all' Requires-Dist: qtconsole ; extra == 'all' Requires-Dist: curio ; extra == 'all' Requires-Dist: matplotlib !=3.2.0 ; extra == 'all' -Requires-Dist: numpy >=1.22 ; extra == 'all' +Requires-Dist: numpy >=1.23 ; extra == 'all' Requires-Dist: pandas ; extra == 'all' Requires-Dist: trio ; extra == 'all' Provides-Extra: black @@ -74,10 +73,9 @@ Requires-Dist: sphinx-rtd-theme ; extra == 'doc' Requires-Dist: docrepr ; extra == 'doc' Requires-Dist: matplotlib ; extra == 'doc' Requires-Dist: stack-data ; extra == 'doc' -Requires-Dist: pytest <7 ; extra == 'doc' +Requires-Dist: pytest ; extra == 'doc' Requires-Dist: typing-extensions ; extra == 'doc' Requires-Dist: exceptiongroup ; extra == 'doc' -Requires-Dist: pytest <7.1 ; extra == 'doc' Requires-Dist: pytest-asyncio <0.22 ; extra == 'doc' Requires-Dist: testpath ; extra == 'doc' Requires-Dist: pickleshare ; extra == 'doc' @@ -96,19 +94,19 @@ Provides-Extra: qtconsole Requires-Dist: qtconsole ; extra == 'qtconsole' Provides-Extra: terminal Provides-Extra: test -Requires-Dist: pytest <7.1 ; extra == 'test' +Requires-Dist: pytest ; extra == 'test' Requires-Dist: pytest-asyncio <0.22 ; extra == 'test' Requires-Dist: testpath ; extra == 'test' Requires-Dist: pickleshare ; extra == 'test' Provides-Extra: test_extra -Requires-Dist: pytest <7.1 ; extra == 'test_extra' +Requires-Dist: pytest ; extra == 'test_extra' Requires-Dist: pytest-asyncio <0.22 ; extra == 'test_extra' Requires-Dist: testpath ; extra == 'test_extra' Requires-Dist: pickleshare ; extra == 'test_extra' Requires-Dist: curio ; extra == 'test_extra' Requires-Dist: matplotlib !=3.2.0 ; extra == 'test_extra' Requires-Dist: nbformat ; extra == 'test_extra' -Requires-Dist: numpy >=1.22 ; extra == 'test_extra' +Requires-Dist: numpy >=1.23 ; extra == 'test_extra' Requires-Dist: pandas ; extra == 'test_extra' Requires-Dist: trio ; extra == 'test_extra' diff --git a/contrib/python/ipython/py3/IPython/__init__.py b/contrib/python/ipython/py3/IPython/__init__.py index 3322562a16..b7235481f2 100644 --- a/contrib/python/ipython/py3/IPython/__init__.py +++ b/contrib/python/ipython/py3/IPython/__init__.py @@ -26,9 +26,10 @@ import sys #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 9): +if sys.version_info < (3, 10): raise ImportError( """ +IPython 8.19+ supports Python 3.10 and above, following SPEC0. IPython 8.13+ supports Python 3.9 and above, following NEP 29. IPython 8.0-8.12 supports Python 3.8 and above, following NEP 29. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. diff --git a/contrib/python/ipython/py3/IPython/core/completer.py b/contrib/python/ipython/py3/IPython/core/completer.py index cc5f6c4270..8e2bb7f810 100644 --- a/contrib/python/ipython/py3/IPython/core/completer.py +++ b/contrib/python/ipython/py3/IPython/core/completer.py @@ -2237,7 +2237,7 @@ class IPCompleter(Completer): self, cursor_column: int, cursor_line: int, text: str ) -> Iterator[_JediCompletionLike]: """ - Return a list of :any:`jedi.api.Completion`s object from a ``text`` and + Return a list of :any:`jedi.api.Completion`\\s object from a ``text`` and cursor position. Parameters diff --git a/contrib/python/ipython/py3/IPython/core/interactiveshell.py b/contrib/python/ipython/py3/IPython/core/interactiveshell.py index 67727404f9..1a73571f78 100644 --- a/contrib/python/ipython/py3/IPython/core/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/core/interactiveshell.py @@ -2639,7 +2639,10 @@ class InteractiveShell(SingletonConfigurable): """ cmd = self.var_expand(cmd, depth=1) # warn if there is an IPython magic alternative. - main_cmd = cmd.split()[0] + if cmd == "": + main_cmd = "" + else: + main_cmd = cmd.split()[0] has_magic_alternatives = ("pip", "conda", "cd") if main_cmd in has_magic_alternatives: diff --git a/contrib/python/ipython/py3/IPython/core/oinspect.py b/contrib/python/ipython/py3/IPython/core/oinspect.py index d77a732267..a8be281975 100644 --- a/contrib/python/ipython/py3/IPython/core/oinspect.py +++ b/contrib/python/ipython/py3/IPython/core/oinspect.py @@ -28,10 +28,7 @@ import warnings from typing import Any, Optional, Dict, Union, List, Tuple -if sys.version_info <= (3, 10): - from typing_extensions import TypeAlias -else: - from typing import TypeAlias +from typing import TypeAlias # IPython's own from IPython.core import page diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py index d012ec426b..70d5a675a3 100644 --- a/contrib/python/ipython/py3/IPython/core/release.py +++ b/contrib/python/ipython/py3/IPython/core/release.py @@ -16,8 +16,8 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 18 -_version_patch = 1 +_version_minor = 19 +_version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" _version_extra = "" # Uncomment this for full releases diff --git a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py index 997fd82250..532287f5e7 100644 --- a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py @@ -588,6 +588,17 @@ class TerminalInteractiveShell(InteractiveShell): help="Display the current vi mode (when using vi editing mode)." ).tag(config=True) + prompt_line_number_format = Unicode( + "", + help="The format for line numbering, will be passed `line` (int, 1 based)" + " the current line number and `rel_line` the relative line number." + " for example to display both you can use the following template string :" + " c.TerminalInteractiveShell.prompt_line_number_format='{line: 4d}/{rel_line:+03d} | '" + " This will display the current line number, with leading space and a width of at least 4" + " character, as well as the relative line number 0 padded and always with a + or - sign." + " Note that when using Emacs mode the prompt of the first line may not update.", + ).tag(config=True) + @observe('term_title') def init_term_title(self, change=None): # Enable or disable the terminal title. @@ -736,7 +747,7 @@ class TerminalInteractiveShell(InteractiveShell): def get_message(): return PygmentsTokens(self.prompts.in_prompt_tokens()) - if self.editing_mode == 'emacs': + if self.editing_mode == "emacs" and self.prompt_line_number_format == "": # with emacs mode the prompt is (usually) static, so we call only # the function once. With VI mode it can toggle between [ins] and # [nor] so we can't precompute. @@ -753,7 +764,7 @@ class TerminalInteractiveShell(InteractiveShell): "message": get_message, "prompt_continuation": ( lambda width, lineno, is_soft_wrap: PygmentsTokens( - self.prompts.continuation_prompt_tokens(width) + self.prompts.continuation_prompt_tokens(width, lineno=lineno) ) ), "multiline": True, diff --git a/contrib/python/ipython/py3/IPython/terminal/prompts.py b/contrib/python/ipython/py3/IPython/terminal/prompts.py index 3f5c07b980..ca56d91a40 100644 --- a/contrib/python/ipython/py3/IPython/terminal/prompts.py +++ b/contrib/python/ipython/py3/IPython/terminal/prompts.py @@ -25,11 +25,21 @@ class Prompts(object): return '['+mode+'] ' return '' + def current_line(self) -> int: + if self.shell.pt_app is not None: + return self.shell.pt_app.default_buffer.document.cursor_position_row or 0 + return 0 def in_prompt_tokens(self): return [ - (Token.Prompt, self.vi_mode() ), - (Token.Prompt, 'In ['), + (Token.Prompt, self.vi_mode()), + ( + Token.Prompt, + self.shell.prompt_line_number_format.format( + line=1, rel_line=-self.current_line() + ), + ), + (Token.Prompt, "In ["), (Token.PromptNum, str(self.shell.execution_count)), (Token.Prompt, ']: '), ] @@ -37,11 +47,20 @@ class Prompts(object): def _width(self): return fragment_list_width(self.in_prompt_tokens()) - def continuation_prompt_tokens(self, width=None): + def continuation_prompt_tokens(self, width=None, *, lineno=None): if width is None: width = self._width() + line = lineno + 1 if lineno is not None else 0 + prefix = " " * len( + self.vi_mode() + ) + self.shell.prompt_line_number_format.format( + line=line, rel_line=line - self.current_line() - 1 + ) return [ - (Token.Prompt, (' ' * (width - 5)) + '...: '), + ( + Token.Prompt, + prefix + (" " * (width - len(prefix) - 5)) + "...: ", + ), ] def rewrite_prompt_tokens(self): diff --git a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py index 8cb62ae3d0..02820dc9c1 100644 --- a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py +++ b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py @@ -1,2 +1,2 @@ # GENERATED BY setup.py -commit = "49914f938" +commit = "dc4369111" diff --git a/contrib/python/ipython/py3/IPython/utils/tz.py b/contrib/python/ipython/py3/IPython/utils/tz.py index dbe1e8e732..ba1199d274 100644 --- a/contrib/python/ipython/py3/IPython/utils/tz.py +++ b/contrib/python/ipython/py3/IPython/utils/tz.py @@ -3,29 +3,56 @@ Timezone utilities Just UTC-awareness right now + +Deprecated since IPython 8.19.0. """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (C) 2013 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Imports -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +import warnings from datetime import tzinfo, timedelta, datetime -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Code -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +__all__ = ["tzUTC", "utc_aware", "utcfromtimestamp", "utcnow"] + + # constant for zero offset ZERO = timedelta(0) + +def __getattr__(name): + if name not in __all__: + err = f"IPython.utils.tz is deprecated and has no attribute {name}" + raise AttributeError(err) + + _warn_deprecated() + + return getattr(name) + + +def _warn_deprecated(): + msg = "The module `IPython.utils.tz` is deprecated and will be completely removed." + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + + class tzUTC(tzinfo): - """tzinfo object for UTC (zero offset)""" + """tzinfo object for UTC (zero offset) + + Deprecated since IPython 8.19.0. + """ + + _warn_deprecated() def utcoffset(self, d): return ZERO @@ -38,11 +65,18 @@ UTC = tzUTC() # type: ignore[abstract] def utc_aware(unaware): - """decorator for adding UTC tzinfo to datetime's utcfoo methods""" + """decorator for adding UTC tzinfo to datetime's utcfoo methods + + Deprecated since IPython 8.19.0. + """ + def utc_method(*args, **kwargs): + _warn_deprecated() dt = unaware(*args, **kwargs) return dt.replace(tzinfo=UTC) + return utc_method + utcfromtimestamp = utc_aware(datetime.utcfromtimestamp) utcnow = utc_aware(datetime.utcnow) diff --git a/contrib/python/ipython/py3/ya.make b/contrib/python/ipython/py3/ya.make index 46c7cd63c3..e6f0887b76 100644 --- a/contrib/python/ipython/py3/ya.make +++ b/contrib/python/ipython/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.18.1) +VERSION(8.19.0) LICENSE(BSD-3-Clause) diff --git a/contrib/python/numpy/py3/.dist-info/METADATA b/contrib/python/numpy/py3/.dist-info/METADATA index 7341e57f9d..5e515025ec 100644 --- a/contrib/python/numpy/py3/.dist-info/METADATA +++ b/contrib/python/numpy/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: numpy -Version: 1.26.2 +Version: 1.26.3 Summary: Fundamental package for array computing in Python Home-page: https://numpy.org Author: Travis E. Oliphant et al. diff --git a/contrib/python/numpy/py3/numpy/__init__.py b/contrib/python/numpy/py3/numpy/__init__.py index 4120c2c116..a2fbd368be 100644 --- a/contrib/python/numpy/py3/numpy/__init__.py +++ b/contrib/python/numpy/py3/numpy/__init__.py @@ -388,20 +388,25 @@ else: pass if sys.platform == "darwin": + from . import exceptions with warnings.catch_warnings(record=True) as w: #_mac_os_check() # Throw runtime error, if the test failed Check for warning and error_message - error_message = "" if len(w) > 0: - error_message = "{}: {}".format(w[-1].category.__name__, str(w[-1].message)) - msg = ( - "Polyfit sanity test emitted a warning, most likely due " - "to using a buggy Accelerate backend." - "\nIf you compiled yourself, more information is available at:" - "\nhttps://numpy.org/doc/stable/user/building.html#accelerated-blas-lapack-libraries" - "\nOtherwise report this to the vendor " - "that provided NumPy.\n{}\n".format(error_message)) - raise RuntimeError(msg) + for _wn in w: + if _wn.category is exceptions.RankWarning: + # Ignore other warnings, they may not be relevant (see gh-25433). + error_message = f"{_wn.category.__name__}: {str(_wn.message)}" + msg = ( + "Polyfit sanity test emitted a warning, most likely due " + "to using a buggy Accelerate backend." + "\nIf you compiled yourself, more information is available at:" + "\nhttps://numpy.org/devdocs/building/index.html" + "\nOtherwise report this to the vendor " + "that provided NumPy.\n\n{}\n".format(error_message)) + raise RuntimeError(msg) + del _wn + del w del _mac_os_check # We usually use madvise hugepages support, but on some old kernels it diff --git a/contrib/python/numpy/py3/numpy/array_api/__init__.py b/contrib/python/numpy/py3/numpy/array_api/__init__.py index 964873faab..77f227882e 100644 --- a/contrib/python/numpy/py3/numpy/array_api/__init__.py +++ b/contrib/python/numpy/py3/numpy/array_api/__init__.py @@ -125,7 +125,7 @@ __array_api_version__ = "2022.12" __all__ = ["__array_api_version__"] -from ._constants import e, inf, nan, pi +from ._constants import e, inf, nan, pi, newaxis __all__ += ["e", "inf", "nan", "pi"] diff --git a/contrib/python/numpy/py3/numpy/array_api/_array_object.py b/contrib/python/numpy/py3/numpy/array_api/_array_object.py index ec465208e8..5aff9863d8 100644 --- a/contrib/python/numpy/py3/numpy/array_api/_array_object.py +++ b/contrib/python/numpy/py3/numpy/array_api/_array_object.py @@ -543,7 +543,11 @@ class Array: def __getitem__( self: Array, key: Union[ - int, slice, ellipsis, Tuple[Union[int, slice, ellipsis], ...], Array + int, + slice, + ellipsis, + Tuple[Union[int, slice, ellipsis, None], ...], + Array, ], /, ) -> Array: diff --git a/contrib/python/numpy/py3/numpy/array_api/_constants.py b/contrib/python/numpy/py3/numpy/array_api/_constants.py index 9541941e7c..15ab81d16d 100644 --- a/contrib/python/numpy/py3/numpy/array_api/_constants.py +++ b/contrib/python/numpy/py3/numpy/array_api/_constants.py @@ -4,3 +4,4 @@ e = np.e inf = np.inf nan = np.nan pi = np.pi +newaxis = np.newaxis diff --git a/contrib/python/numpy/py3/numpy/array_api/linalg.py b/contrib/python/numpy/py3/numpy/array_api/linalg.py index 58320db55c..09af9dfc3a 100644 --- a/contrib/python/numpy/py3/numpy/array_api/linalg.py +++ b/contrib/python/numpy/py3/numpy/array_api/linalg.py @@ -318,8 +318,9 @@ def _solve(a, b): # This does nothing currently but is left in because it will be relevant # when complex dtype support is added to the spec in 2022. signature = 'DD->D' if isComplexType(t) else 'dd->d' - extobj = get_linalg_error_extobj(_raise_linalgerror_singular) - r = gufunc(a, b, signature=signature, extobj=extobj) + with np.errstate(call=_raise_linalgerror_singular, invalid='call', + over='ignore', divide='ignore', under='ignore'): + r = gufunc(a, b, signature=signature) return wrap(r.astype(result_t, copy=False)) diff --git a/contrib/python/numpy/py3/numpy/core/src/common/half.hpp b/contrib/python/numpy/py3/numpy/core/src/common/half.hpp index ff9a547766..484750ad84 100644 --- a/contrib/python/numpy/py3/numpy/core/src/common/half.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/common/half.hpp @@ -59,7 +59,11 @@ class Half final { __vector float vf32 = vec_splats(f); __vector unsigned short vf16; __asm__ __volatile__ ("xvcvsphp %x0,%x1" : "=wa" (vf16) : "wa" (vf32)); + #ifdef __BIG_ENDIAN__ + bits_ = vec_extract(vf16, 1); + #else bits_ = vec_extract(vf16, 0); + #endif #else bits_ = half_private::FromFloatBits(BitCast<uint32_t>(f)); #endif diff --git a/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h b/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h index 2dc21e5a43..e7503b822e 100644 --- a/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h +++ b/contrib/python/numpy/py3/numpy/core/src/common/simd/neon/memory.h @@ -52,21 +52,12 @@ NPYV_IMPL_NEON_MEM(f64, double) ***************************/ NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) { - switch (stride) { - case 2: - return vld2q_s32((const int32_t*)ptr).val[0]; - case 3: - return vld3q_s32((const int32_t*)ptr).val[0]; - case 4: - return vld4q_s32((const int32_t*)ptr).val[0]; - default:; - int32x2_t ax = vcreate_s32(*ptr); - int32x4_t a = vcombine_s32(ax, ax); - a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); - a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); - a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); - return a; - } + int32x4_t a = vdupq_n_s32(0); + a = vld1q_lane_s32((const int32_t*)ptr, a, 0); + a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); + a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); + a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); + return a; } NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c index 58e8b3778b..efa4c271cf 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c @@ -21408,7 +21408,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - memcpy(&obj, ip, sizeof(obj)); + memcpy(&obj, (void *)ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src index bc3f743727..f3feee63da 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/arraytypes.c.src @@ -2889,7 +2889,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - memcpy(&obj, ip, sizeof(obj)); + memcpy(&obj, (void *)ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } diff --git a/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c b/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c index dfa84c51cd..427dd3d876 100644 --- a/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c +++ b/contrib/python/numpy/py3/numpy/core/src/multiarray/nditer_constr.c @@ -198,6 +198,9 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, /* Allocate memory for the iterator */ iter = (NpyIter*) PyObject_Malloc(NIT_SIZEOF_ITERATOR(itflags, ndim, nop)); + if (iter == NULL) { + return NULL; + } NPY_IT_TIME_POINT(c_malloc); diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp b/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp index 3f5099758c..a75f882ff4 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp @@ -9,6 +9,7 @@ #if defined(NPY_HAVE_AVX512_SPR) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512fp16-16bit-qsort.hpp" + #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #elif defined(NPY_HAVE_AVX512_ICL) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #endif diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp index 606f870645..2eb445422b 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512-16bit-qsort.hpp @@ -350,7 +350,7 @@ struct zmm_vector<uint16_t> { }; template <> -bool comparison_func<zmm_vector<float16>>(const uint16_t &a, const uint16_t &b) +inline bool comparison_func<zmm_vector<float16>>(const uint16_t &a, const uint16_t &b) { uint16_t signa = a & 0x8000, signb = b & 0x8000; uint16_t expa = a & 0x7c00, expb = b & 0x7c00; @@ -406,7 +406,7 @@ replace_inf_with_nan(uint16_t *arr, int64_t arrsize, int64_t nan_count) } template <> -void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { qselect_16bit_<zmm_vector<int16_t>, int16_t>( @@ -415,7 +415,7 @@ void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { qselect_16bit_<zmm_vector<uint16_t>, uint16_t>( @@ -423,7 +423,7 @@ void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize) } } -void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); @@ -434,7 +434,7 @@ void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qsort(int16_t *arr, int64_t arrsize) +inline void avx512_qsort(int16_t *arr, int64_t arrsize) { if (arrsize > 1) { qsort_16bit_<zmm_vector<int16_t>, int16_t>( @@ -443,7 +443,7 @@ void avx512_qsort(int16_t *arr, int64_t arrsize) } template <> -void avx512_qsort(uint16_t *arr, int64_t arrsize) +inline void avx512_qsort(uint16_t *arr, int64_t arrsize) { if (arrsize > 1) { qsort_16bit_<zmm_vector<uint16_t>, uint16_t>( @@ -451,7 +451,7 @@ void avx512_qsort(uint16_t *arr, int64_t arrsize) } } -void avx512_qsort_fp16(uint16_t *arr, int64_t arrsize) +inline void avx512_qsort_fp16(uint16_t *arr, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); diff --git a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp index 8a9a49ede9..1206f8223d 100644 --- a/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp +++ b/contrib/python/numpy/py3/numpy/core/src/npysort/x86-simd-sort/src/avx512fp16-16bit-qsort.hpp @@ -145,7 +145,7 @@ replace_inf_with_nan(_Float16 *arr, int64_t arrsize, int64_t nan_count) } template <> -void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) +inline void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); @@ -156,7 +156,7 @@ void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize) } template <> -void avx512_qsort(_Float16 *arr, int64_t arrsize) +inline void avx512_qsort(_Float16 *arr, int64_t arrsize) { if (arrsize > 1) { int64_t nan_count = replace_nan_with_inf(arr, arrsize); diff --git a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c index 695d488250..19706ae07f 100644 --- a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c +++ b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c @@ -68,7 +68,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_add) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -132,8 +154,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_add) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -204,7 +228,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_subtract) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -268,8 +314,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_subtract) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -340,7 +388,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_multiply) #endif return; } -#if NPY_SIMD_F32 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -404,8 +474,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_multiply) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 1 + #if 1 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 0 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -476,7 +548,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_divide) #endif return; } -#if NPY_SIMD_F32 +#if 1 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F32 if (len > npyv_nlanes_f32*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -540,8 +634,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_divide) npyv_store_f32((npy_float*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 1 || 0 + #if 0 npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, 1.0f); + #elif 1 + npyv_f32 a = npyv_load_till_f32((const npy_float*)src0, len, NPY_NANF); #else npyv_f32 a = npyv_load_tillz_f32((const npy_float*)src0, len); #endif @@ -614,7 +710,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_add) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -678,8 +796,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_add) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -750,7 +870,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_subtract) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -814,8 +956,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_subtract) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -886,7 +1030,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_multiply) #endif return; } -#if NPY_SIMD_F64 +#if 0 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -950,8 +1116,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_multiply) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 0 || 1 + #if 1 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 0 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -1022,7 +1190,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_divide) #endif return; } -#if NPY_SIMD_F64 +#if 1 && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif NPY_SIMD_F64 if (len > npyv_nlanes_f64*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -1086,8 +1276,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_divide) npyv_store_f64((npy_double*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if 1 || 0 + #if 0 npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, 1.0); + #elif 1 + npyv_f64 a = npyv_load_till_f64((const npy_double*)src0, len, NPY_NAN); #else npyv_f64 a = npyv_load_tillz_f64((const npy_double*)src0, len); #endif @@ -1220,7 +1412,7 @@ simd_csquare_f64(npyv_f64 x) { return simd_cmul_f64(x, x); } #endif -#line 286 +#line 310 #if NPY_SIMD_F32 NPY_FINLINE npyv_f32 simd_cabsolute_f32(npyv_f32 re, npyv_f32 im) @@ -1264,7 +1456,7 @@ simd_cabsolute_f32(npyv_f32 re, npyv_f32 im) } #endif // VECTOR -#line 286 +#line 310 #if NPY_SIMD_F64 NPY_FINLINE npyv_f64 simd_cabsolute_f64(npyv_f64 re, npyv_f64 im) @@ -1312,8 +1504,8 @@ simd_cabsolute_f64(npyv_f64 re, npyv_f64 im) /******************************************************************************** ** Defining ufunc inner functions ********************************************************************************/ -#line 342 -#line 350 +#line 366 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_add) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -1565,7 +1757,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_add_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_subtract) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -1817,7 +2009,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_subtract_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_multiply) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2070,7 +2262,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CFLOAT_multiply_indexed) } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_conjugate) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2158,7 +2350,7 @@ loop_scalar: } } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CFLOAT_square) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2247,8 +2439,8 @@ loop_scalar: } -#line 342 -#line 350 +#line 366 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_add) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2500,7 +2692,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_add_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_subtract) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2752,7 +2944,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_subtract_indexed) return 0; } -#line 350 +#line 374 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_multiply) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -3005,7 +3197,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(CDOUBLE_multiply_indexed) } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_conjugate) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -3093,7 +3285,7 @@ loop_scalar: } } -#line 606 +#line 630 NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(CDOUBLE_square) (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { diff --git a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src index 30111258d6..0d0de90125 100644 --- a/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src +++ b/contrib/python/numpy/py3/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src @@ -74,7 +74,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) #endif return; } -#if @VECTOR@ +#if @is_div@ && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif @VECTOR@ if (len > npyv_nlanes_@sfx@*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) @@ -138,8 +160,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) npyv_store_@sfx@((@type@*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if @is_div@ || @is_mul@ + #if @is_mul@ npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, 1.0@c@); + #elif @is_div@ + npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, NPY_NAN@C@); #else npyv_@sfx@ a = npyv_load_tillz_@sfx@((const @type@*)src0, len); #endif diff --git a/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build b/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build index 12fc640b88..836b74ac38 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build +++ b/contrib/python/numpy/py3/numpy/core/tests/examples/cython/meson.build @@ -14,6 +14,15 @@ npy_include_path = run_command(py, [ 'import os; os.chdir(".."); import numpy; print(os.path.abspath(numpy.get_include()))' ], check: true).stdout().strip() +npy_path = run_command(py, [ + '-c', + 'import os; os.chdir(".."); import numpy; print(os.path.dirname(numpy.__file__).removesuffix("numpy"))' + ], check: true).stdout().strip() + +# TODO: This is a hack due to gh-25135, where cython may not find the right +# __init__.pyd file. +add_project_arguments('-I', npy_path, language : 'cython') + py.extension_module( 'checks', 'checks.pyx', diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py b/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py index 3a7c7a80fd..da010df63e 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_cpu_features.py @@ -352,6 +352,7 @@ class Test_X86_Features(AbstractTest): SSE3="PNI", SSE41="SSE4_1", SSE42="SSE4_2", FMA3="FMA", AVX512VNNI="AVX512_VNNI", AVX512BITALG="AVX512_BITALG", AVX512VBMI2="AVX512_VBMI2", AVX5124FMAPS="AVX512_4FMAPS", AVX5124VNNIW="AVX512_4VNNIW", AVX512VPOPCNTDQ="AVX512_VPOPCNTDQ", + AVX512FP16="AVX512_FP16", ) def load_flags(self): self.load_flags_cpuinfo("flags") diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_cython.py b/contrib/python/numpy/py3/numpy/core/tests/test_cython.py index 99dd57e4c6..0e0d00c250 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_cython.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_cython.py @@ -28,14 +28,14 @@ else: pytestmark = pytest.mark.skipif(cython is None, reason="requires cython") -@pytest.fixture -def install_temp(tmp_path): +@pytest.fixture(scope='module') +def install_temp(tmpdir_factory): # Based in part on test_cython from random.tests.test_extending if IS_WASM: pytest.skip("No subprocess") srcdir = os.path.join(os.path.dirname(__file__), 'examples', 'cython') - build_dir = tmp_path / "build" + build_dir = tmpdir_factory.mktemp("cython_test") / "build" os.makedirs(build_dir, exist_ok=True) try: subprocess.check_call(["meson", "--version"]) diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py b/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py index bc3f330dc1..a381fa1d89 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_mem_policy.py @@ -440,3 +440,4 @@ def test_owner_is_base(get_module): with pytest.warns(UserWarning, match='warn_on_free'): del a gc.collect() + gc.collect() diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py b/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py index 35bd6e97e0..8c1a770cd2 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_nditer.py @@ -3203,7 +3203,6 @@ def test_warn_noclose(): assert len(sup.log) == 1 -@pytest.mark.skip @pytest.mark.skipif(sys.version_info[:2] == (3, 9) and sys.platform == "win32", reason="Errors with Python 3.9 on Windows") @pytest.mark.parametrize(["in_dtype", "buf_dtype"], diff --git a/contrib/python/numpy/py3/numpy/core/tests/test_umath.py b/contrib/python/numpy/py3/numpy/core/tests/test_umath.py index 59c670ffed..963e740d8d 100644 --- a/contrib/python/numpy/py3/numpy/core/tests/test_umath.py +++ b/contrib/python/numpy/py3/numpy/core/tests/test_umath.py @@ -17,7 +17,8 @@ from numpy.testing import ( assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_array_max_ulp, assert_allclose, assert_no_warnings, suppress_warnings, - _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL + _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL, + IS_PYPY ) from numpy.testing._private.utils import _glibc_older_than @@ -1825,6 +1826,18 @@ class TestSpecialFloats: with assert_no_warnings(): ufunc(array) + @pytest.mark.parametrize("dtype", ('e', 'f', 'd')) + def test_divide_spurious_fpexception(self, dtype): + dt = np.dtype(dtype) + dt_info = np.finfo(dt) + subnorm = dt_info.smallest_subnormal + # Verify a bug fix caused due to filling the remaining lanes of the + # partially loaded dividend SIMD vector with ones, which leads to + # raising an overflow warning when the divisor is denormal. + # see https://github.com/numpy/numpy/issues/25097 + with assert_no_warnings(): + np.zeros(128 + 1, dtype=dt) / subnorm + class TestFPClass: @pytest.mark.parametrize("stride", [-5, -4, -3, -2, -1, 1, 2, 4, 5, 6, 7, 8, 9, 10]) @@ -4180,7 +4193,10 @@ class TestComplexFunctions: for p in points: a = complex(func(np.complex_(p))) b = cfunc(p) - assert_(abs(a - b) < atol, "%s %s: %s; cmath: %s" % (fname, p, a, b)) + assert_( + abs(a - b) < atol, + "%s %s: %s; cmath: %s" % (fname, p, a, b) + ) @pytest.mark.xfail( # manylinux2014 uses glibc2.17 diff --git a/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c b/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c index b3f4f69765..cb55e57aa2 100644 --- a/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c +++ b/contrib/python/numpy/py3/numpy/distutils/checks/cpu_avx512_knl.c @@ -15,7 +15,7 @@ int main(int argc, char **argv) { - int base[128]; + int base[128]={}; __m512d ad = _mm512_loadu_pd((const __m512d*)argv[argc-1]); /* ER */ __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); diff --git a/contrib/python/numpy/py3/numpy/f2py/__init__.py b/contrib/python/numpy/py3/numpy/f2py/__init__.py index dbe3df27f6..e583250f70 100644 --- a/contrib/python/numpy/py3/numpy/f2py/__init__.py +++ b/contrib/python/numpy/py3/numpy/f2py/__init__.py @@ -1,13 +1,21 @@ #!/usr/bin/env python3 """Fortran to Python Interface Generator. +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the terms +of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ __all__ = ['run_main', 'compile', 'get_include'] import sys import subprocess import os +import warnings +from numpy.exceptions import VisibleDeprecationWarning from . import f2py2e from . import diagnose diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py b/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py index e548fc5430..e9b22a3921 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/_distutils.py @@ -13,7 +13,7 @@ import warnings class DistutilsBackend(Backend): def __init__(sef, *args, **kwargs): warnings.warn( - "distutils has been deprecated since NumPy 1.26." + "distutils has been deprecated since NumPy 1.26.x" "Use the Meson backend instead, or generate wrappers" "without -c and use a custom build script", VisibleDeprecationWarning, diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py b/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py index 3176a5e08f..f324e0f595 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/_meson.py @@ -1,12 +1,15 @@ from __future__ import annotations +import os import errno import shutil import subprocess +import sys from pathlib import Path from ._backend import Backend from string import Template +from itertools import chain import warnings @@ -19,10 +22,14 @@ class MesonTemplate: modulename: str, sources: list[Path], deps: list[str], + libraries: list[str], + library_dirs: list[Path], + include_dirs: list[Path], object_files: list[Path], linker_args: list[str], c_args: list[str], build_type: str, + python_exe: str, ): self.modulename = modulename self.build_template_path = ( @@ -30,14 +37,23 @@ class MesonTemplate: ) self.sources = sources self.deps = deps + self.libraries = libraries + self.library_dirs = library_dirs + if include_dirs is not None: + self.include_dirs = include_dirs + else: + self.include_dirs = [] self.substitutions = {} self.objects = object_files self.pipeline = [ self.initialize_template, self.sources_substitution, self.deps_substitution, + self.include_substitution, + self.libraries_substitution, ] self.build_type = build_type + self.python_exe = python_exe def meson_build_template(self) -> str: if not self.build_template_path.is_file(): @@ -52,17 +68,47 @@ class MesonTemplate: def initialize_template(self) -> None: self.substitutions["modulename"] = self.modulename self.substitutions["buildtype"] = self.build_type + self.substitutions["python"] = self.python_exe def sources_substitution(self) -> None: indent = " " * 21 self.substitutions["source_list"] = f",\n{indent}".join( - [f"'{source}'" for source in self.sources] + [f"{indent}'{source}'" for source in self.sources] ) def deps_substitution(self) -> None: indent = " " * 21 self.substitutions["dep_list"] = f",\n{indent}".join( - [f"dependency('{dep}')" for dep in self.deps] + [f"{indent}dependency('{dep}')" for dep in self.deps] + ) + + def libraries_substitution(self) -> None: + self.substitutions["lib_dir_declarations"] = "\n".join( + [ + f"lib_dir_{i} = declare_dependency(link_args : ['-L{lib_dir}'])" + for i, lib_dir in enumerate(self.library_dirs) + ] + ) + + self.substitutions["lib_declarations"] = "\n".join( + [ + f"{lib} = declare_dependency(link_args : ['-l{lib}'])" + for lib in self.libraries + ] + ) + + indent = " " * 21 + self.substitutions["lib_list"] = f"\n{indent}".join( + [f"{indent}{lib}," for lib in self.libraries] + ) + self.substitutions["lib_dir_list"] = f"\n{indent}".join( + [f"{indent}lib_dir_{i}," for i in range(len(self.library_dirs))] + ) + + def include_substitution(self) -> None: + indent = " " * 21 + self.substitutions["inc_list"] = f",\n{indent}".join( + [f"{indent}'{inc}'" for inc in self.include_dirs] ) def generate_meson_build(self): @@ -83,16 +129,18 @@ class MesonBackend(Backend): def _move_exec_to_root(self, build_dir: Path): walk_dir = Path(build_dir) / self.meson_build_dir - path_objects = walk_dir.glob(f"{self.modulename}*.so") + path_objects = chain( + walk_dir.glob(f"{self.modulename}*.so"), + walk_dir.glob(f"{self.modulename}*.pyd"), + ) + # Same behavior as distutils + # https://github.com/numpy/numpy/issues/24874#issuecomment-1835632293 for path_object in path_objects: - shutil.move(path_object, Path.cwd()) - - def _get_build_command(self): - return [ - "meson", - "setup", - self.meson_build_dir, - ] + dest_path = Path.cwd() / path_object.name + if dest_path.exists(): + dest_path.unlink() + shutil.copy2(path_object, dest_path) + os.remove(path_object) def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" @@ -100,10 +148,14 @@ class MesonBackend(Backend): self.modulename, self.sources, self.dependencies, + self.libraries, + self.library_dirs, + self.include_dirs, self.extra_objects, self.flib_flags, self.fc_flags, self.build_type, + sys.executable, ) src = meson_template.generate_meson_build() Path(build_dir).mkdir(parents=True, exist_ok=True) @@ -111,19 +163,14 @@ class MesonBackend(Backend): meson_build_file.write_text(src) return meson_build_file + def _run_subprocess_command(self, command, cwd): + subprocess.run(command, cwd=cwd, check=True) + def run_meson(self, build_dir: Path): - completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) - completed_process = subprocess.run( - ["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir - ) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) + setup_command = ["meson", "setup", self.meson_build_dir] + self._run_subprocess_command(setup_command, build_dir) + compile_command = ["meson", "compile", "-C", self.meson_build_dir] + self._run_subprocess_command(compile_command, build_dir) def compile(self) -> None: self.sources = _prepare_sources(self.modulename, self.sources, self.build_dir) @@ -137,7 +184,8 @@ def _prepare_sources(mname, sources, bdir): Path(bdir).mkdir(parents=True, exist_ok=True) # Copy sources for source in sources: - shutil.copy(source, bdir) + if Path(source).exists() and Path(source).is_file(): + shutil.copy(source, bdir) generated_sources = [ Path(f"{mname}module.c"), Path(f"{mname}-f2pywrappers2.f90"), diff --git a/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template b/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template index 545e399521..8e34fdc8d4 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template +++ b/contrib/python/numpy/py3/numpy/f2py/_backends/meson.build.template @@ -6,8 +6,9 @@ project('${modulename}', 'warning_level=1', 'buildtype=${buildtype}' ]) +fc = meson.get_compiler('fortran') -py = import('python').find_installation(pure: false) +py = import('python').find_installation('${python}', pure: false) py_dep = py.dependency() incdir_numpy = run_command(py, @@ -28,15 +29,26 @@ inc_f2py = include_directories(incdir_f2py) fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) +# gh-25000 +quadmath_dep = fc.find_library('quadmath', required: false) + +${lib_declarations} +${lib_dir_declarations} py.extension_module('${modulename}', [ ${source_list}, fortranobject_c ], - include_directories: [inc_np], + include_directories: [ + inc_np, +${inc_list} + ], dependencies : [ py_dep, + quadmath_dep, ${dep_list} +${lib_list} +${lib_dir_list} ], install : true) diff --git a/contrib/python/numpy/py3/numpy/f2py/_isocbind.py b/contrib/python/numpy/py3/numpy/f2py/_isocbind.py index 81f52fb4de..3043c5d916 100644 --- a/contrib/python/numpy/py3/numpy/f2py/_isocbind.py +++ b/contrib/python/numpy/py3/numpy/f2py/_isocbind.py @@ -1,45 +1,61 @@ +""" +ISO_C_BINDING maps for f2py2e. +Only required declarations/macros/functions will be used. + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. +""" +# These map to keys in c2py_map, via forced casting for now, see gh-25229 iso_c_binding_map = { 'integer': { 'c_int': 'int', - 'c_short': 'short int', - 'c_long': 'long int', - 'c_long_long': 'long long int', - 'c_signed_char': 'signed char', - 'c_size_t': 'size_t', - 'c_int8_t': 'int8_t', - 'c_int16_t': 'int16_t', - 'c_int32_t': 'int32_t', - 'c_int64_t': 'int64_t', - 'c_int_least8_t': 'int_least8_t', - 'c_int_least16_t': 'int_least16_t', - 'c_int_least32_t': 'int_least32_t', - 'c_int_least64_t': 'int_least64_t', - 'c_int_fast8_t': 'int_fast8_t', - 'c_int_fast16_t': 'int_fast16_t', - 'c_int_fast32_t': 'int_fast32_t', - 'c_int_fast64_t': 'int_fast64_t', - 'c_intmax_t': 'intmax_t', - 'c_intptr_t': 'intptr_t', - 'c_ptrdiff_t': 'intptr_t', + 'c_short': 'short', # 'short' <=> 'int' for now + 'c_long': 'long', # 'long' <=> 'int' for now + 'c_long_long': 'long_long', + 'c_signed_char': 'signed_char', + 'c_size_t': 'unsigned', # size_t <=> 'unsigned' for now + 'c_int8_t': 'signed_char', # int8_t <=> 'signed_char' for now + 'c_int16_t': 'short', # int16_t <=> 'short' for now + 'c_int32_t': 'int', # int32_t <=> 'int' for now + 'c_int64_t': 'long_long', + 'c_int_least8_t': 'signed_char', # int_least8_t <=> 'signed_char' for now + 'c_int_least16_t': 'short', # int_least16_t <=> 'short' for now + 'c_int_least32_t': 'int', # int_least32_t <=> 'int' for now + 'c_int_least64_t': 'long_long', + 'c_int_fast8_t': 'signed_char', # int_fast8_t <=> 'signed_char' for now + 'c_int_fast16_t': 'short', # int_fast16_t <=> 'short' for now + 'c_int_fast32_t': 'int', # int_fast32_t <=> 'int' for now + 'c_int_fast64_t': 'long_long', + 'c_intmax_t': 'long_long', # intmax_t <=> 'long_long' for now + 'c_intptr_t': 'long', # intptr_t <=> 'long' for now + 'c_ptrdiff_t': 'long', # ptrdiff_t <=> 'long' for now }, 'real': { 'c_float': 'float', 'c_double': 'double', - 'c_long_double': 'long double' + 'c_long_double': 'long_double' }, 'complex': { - 'c_float_complex': 'float _Complex', - 'c_double_complex': 'double _Complex', - 'c_long_double_complex': 'long double _Complex' + 'c_float_complex': 'complex_float', + 'c_double_complex': 'complex_double', + 'c_long_double_complex': 'complex_long_double' }, 'logical': { - 'c_bool': '_Bool' + 'c_bool': 'unsigned_char' # _Bool <=> 'unsigned_char' for now }, 'character': { 'c_char': 'char' } } +# TODO: See gh-25229 +isoc_c2pycode_map = {} +iso_c2py_map = {} + isoc_kindmap = {} for fortran_type, c_type_dict in iso_c_binding_map.items(): for c_type in c_type_dict.keys(): diff --git a/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py b/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py new file mode 100644 index 0000000000..6247b95bfe --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/_src_pyf.py @@ -0,0 +1,239 @@ +import re + +# START OF CODE VENDORED FROM `numpy.distutils.from_template` +############################################################# +""" +process_file(filename) + + takes templated file .xxx.src and produces .xxx file where .xxx + is .pyf .f90 or .f using the following template rules: + + '<..>' denotes a template. + + All function and subroutine blocks in a source file with names that + contain '<..>' will be replicated according to the rules in '<..>'. + + The number of comma-separated words in '<..>' will determine the number of + replicates. + + '<..>' may have two different forms, named and short. For example, + + named: + <p=d,s,z,c> where anywhere inside a block '<p>' will be replaced with + 'd', 's', 'z', and 'c' for each replicate of the block. + + <_c> is already defined: <_c=s,d,c,z> + <_t> is already defined: <_t=real,double precision,complex,double complex> + + short: + <s,d,c,z>, a short form of the named, useful when no <p> appears inside + a block. + + In general, '<..>' contains a comma separated list of arbitrary + expressions. If these expression must contain a comma|leftarrow|rightarrow, + then prepend the comma|leftarrow|rightarrow with a backslash. + + If an expression matches '\\<index>' then it will be replaced + by <index>-th expression. + + Note that all '<..>' forms in a block must have the same number of + comma-separated entries. + + Predefined named template rules: + <prefix=s,d,c,z> + <ftype=real,double precision,complex,double complex> + <ftypereal=real,double precision,\\0,\\1> + <ctype=float,double,complex_float,complex_double> + <ctypereal=float,double,\\0,\\1> +""" + +routine_start_re = re.compile(r'(\n|\A)(( (\$|\*))|)\s*(subroutine|function)\b', re.I) +routine_end_re = re.compile(r'\n\s*end\s*(subroutine|function)\b.*(\n|\Z)', re.I) +function_start_re = re.compile(r'\n (\$|\*)\s*function\b', re.I) + +def parse_structure(astr): + """ Return a list of tuples for each function or subroutine each + tuple is the start and end of a subroutine or function to be + expanded. + """ + + spanlist = [] + ind = 0 + while True: + m = routine_start_re.search(astr, ind) + if m is None: + break + start = m.start() + if function_start_re.match(astr, start, m.end()): + while True: + i = astr.rfind('\n', ind, start) + if i==-1: + break + start = i + if astr[i:i+7]!='\n $': + break + start += 1 + m = routine_end_re.search(astr, m.end()) + ind = end = m and m.end()-1 or len(astr) + spanlist.append((start, end)) + return spanlist + +template_re = re.compile(r"<\s*(\w[\w\d]*)\s*>") +named_re = re.compile(r"<\s*(\w[\w\d]*)\s*=\s*(.*?)\s*>") +list_re = re.compile(r"<\s*((.*?))\s*>") + +def find_repl_patterns(astr): + reps = named_re.findall(astr) + names = {} + for rep in reps: + name = rep[0].strip() or unique_key(names) + repl = rep[1].replace(r'\,', '@comma@') + thelist = conv(repl) + names[name] = thelist + return names + +def find_and_remove_repl_patterns(astr): + names = find_repl_patterns(astr) + astr = re.subn(named_re, '', astr)[0] + return astr, names + +item_re = re.compile(r"\A\\(?P<index>\d+)\Z") +def conv(astr): + b = astr.split(',') + l = [x.strip() for x in b] + for i in range(len(l)): + m = item_re.match(l[i]) + if m: + j = int(m.group('index')) + l[i] = l[j] + return ','.join(l) + +def unique_key(adict): + """ Obtain a unique key given a dictionary.""" + allkeys = list(adict.keys()) + done = False + n = 1 + while not done: + newkey = '__l%s' % (n) + if newkey in allkeys: + n += 1 + else: + done = True + return newkey + + +template_name_re = re.compile(r'\A\s*(\w[\w\d]*)\s*\Z') +def expand_sub(substr, names): + substr = substr.replace(r'\>', '@rightarrow@') + substr = substr.replace(r'\<', '@leftarrow@') + lnames = find_repl_patterns(substr) + substr = named_re.sub(r"<\1>", substr) # get rid of definition templates + + def listrepl(mobj): + thelist = conv(mobj.group(1).replace(r'\,', '@comma@')) + if template_name_re.match(thelist): + return "<%s>" % (thelist) + name = None + for key in lnames.keys(): # see if list is already in dictionary + if lnames[key] == thelist: + name = key + if name is None: # this list is not in the dictionary yet + name = unique_key(lnames) + lnames[name] = thelist + return "<%s>" % name + + substr = list_re.sub(listrepl, substr) # convert all lists to named templates + # newnames are constructed as needed + + numsubs = None + base_rule = None + rules = {} + for r in template_re.findall(substr): + if r not in rules: + thelist = lnames.get(r, names.get(r, None)) + if thelist is None: + raise ValueError('No replicates found for <%s>' % (r)) + if r not in names and not thelist.startswith('_'): + names[r] = thelist + rule = [i.replace('@comma@', ',') for i in thelist.split(',')] + num = len(rule) + + if numsubs is None: + numsubs = num + rules[r] = rule + base_rule = r + elif num == numsubs: + rules[r] = rule + else: + print("Mismatch in number of replacements (base <{}={}>) " + "for <{}={}>. Ignoring.".format(base_rule, ','.join(rules[base_rule]), r, thelist)) + if not rules: + return substr + + def namerepl(mobj): + name = mobj.group(1) + return rules.get(name, (k+1)*[name])[k] + + newstr = '' + for k in range(numsubs): + newstr += template_re.sub(namerepl, substr) + '\n\n' + + newstr = newstr.replace('@rightarrow@', '>') + newstr = newstr.replace('@leftarrow@', '<') + return newstr + +def process_str(allstr): + newstr = allstr + writestr = '' + + struct = parse_structure(newstr) + + oldend = 0 + names = {} + names.update(_special_names) + for sub in struct: + cleanedstr, defs = find_and_remove_repl_patterns(newstr[oldend:sub[0]]) + writestr += cleanedstr + names.update(defs) + writestr += expand_sub(newstr[sub[0]:sub[1]], names) + oldend = sub[1] + writestr += newstr[oldend:] + + return writestr + +include_src_re = re.compile(r"(\n|\A)\s*include\s*['\"](?P<name>[\w\d./\\]+\.src)['\"]", re.I) + +def resolve_includes(source): + d = os.path.dirname(source) + with open(source) as fid: + lines = [] + for line in fid: + m = include_src_re.match(line) + if m: + fn = m.group('name') + if not os.path.isabs(fn): + fn = os.path.join(d, fn) + if os.path.isfile(fn): + lines.extend(resolve_includes(fn)) + else: + lines.append(line) + else: + lines.append(line) + return lines + +def process_file(source): + lines = resolve_includes(source) + return process_str(''.join(lines)) + +_special_names = find_repl_patterns(''' +<_c=s,d,c,z> +<_t=real,double precision,complex,double complex> +<prefix=s,d,c,z> +<ftype=real,double precision,complex,double complex> +<ctype=float,double,complex_float,complex_double> +<ftypereal=real,double precision,\\0,\\1> +<ctypereal=float,double,\\0,\\1> +''') + +# END OF CODE VENDORED FROM `numpy.distutils.from_template` +########################################################### diff --git a/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py b/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py index 0c08e0a5e2..13a1074b44 100644 --- a/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py +++ b/contrib/python/numpy/py3/numpy/f2py/auxfuncs.py @@ -1,18 +1,12 @@ -#!/usr/bin/env python3 """ - Auxiliary functions for f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy (BSD style) LICENSE. - NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/24 19:01:55 $ -Pearu Peterson - """ import pprint import sys @@ -50,7 +44,7 @@ __all__ = [ 'isunsigned_long_longarray', 'isunsigned_short', 'isunsigned_shortarray', 'l_and', 'l_not', 'l_or', 'outmess', 'replace', 'show', 'stripcomma', 'throw_error', 'isattr_value', - 'deep_merge' + 'getuseblocks', 'process_f2cmap_dict' ] @@ -899,28 +893,6 @@ def applyrules(rules, d, var={}): del ret[k] return ret -def deep_merge(dict1, dict2): - """Recursively merge two dictionaries into a new dictionary. - - Parameters: - - dict1: The base dictionary. - - dict2: The dictionary to merge into a copy of dict1. - If a key exists in both, the dict2 value will take precedence. - - Returns: - - A new merged dictionary. - """ - merged_dict = deepcopy(dict1) - for key, value in dict2.items(): - if key in merged_dict: - if isinstance(merged_dict[key], dict) and isinstance(value, dict): - merged_dict[key] = deep_merge(merged_dict[key], value) - else: - merged_dict[key] = value - else: - merged_dict[key] = value - return merged_dict - _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]+)', re.I).match _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]*?' @@ -937,3 +909,80 @@ def get_f2py_modulename(source): name = m.group('name') break return name + +def getuseblocks(pymod): + all_uses = [] + for inner in pymod['body']: + for modblock in inner['body']: + if modblock.get('use'): + all_uses.extend([x for x in modblock.get("use").keys() if "__" not in x]) + return all_uses + +def process_f2cmap_dict(f2cmap_all, new_map, c2py_map, verbose = False): + """ + Update the Fortran-to-C type mapping dictionary with new mappings and + return a list of successfully mapped C types. + + This function integrates a new mapping dictionary into an existing + Fortran-to-C type mapping dictionary. It ensures that all keys are in + lowercase and validates new entries against a given C-to-Python mapping + dictionary. Redefinitions and invalid entries are reported with a warning. + + Parameters + ---------- + f2cmap_all : dict + The existing Fortran-to-C type mapping dictionary that will be updated. + It should be a dictionary of dictionaries where the main keys represent + Fortran types and the nested dictionaries map Fortran type specifiers + to corresponding C types. + + new_map : dict + A dictionary containing new type mappings to be added to `f2cmap_all`. + The structure should be similar to `f2cmap_all`, with keys representing + Fortran types and values being dictionaries of type specifiers and their + C type equivalents. + + c2py_map : dict + A dictionary used for validating the C types in `new_map`. It maps C + types to corresponding Python types and is used to ensure that the C + types specified in `new_map` are valid. + + verbose : boolean + A flag used to provide information about the types mapped + + Returns + ------- + tuple of (dict, list) + The updated Fortran-to-C type mapping dictionary and a list of + successfully mapped C types. + """ + f2cmap_mapped = [] + + new_map_lower = {} + for k, d1 in new_map.items(): + d1_lower = {k1.lower(): v1 for k1, v1 in d1.items()} + new_map_lower[k.lower()] = d1_lower + + for k, d1 in new_map_lower.items(): + if k not in f2cmap_all: + f2cmap_all[k] = {} + + for k1, v1 in d1.items(): + if v1 in c2py_map: + if k1 in f2cmap_all[k]: + outmess( + "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" + % (k, k1, f2cmap_all[k][k1], v1) + ) + f2cmap_all[k][k1] = v1 + if verbose: + outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k, k1, v1)) + f2cmap_mapped.append(v1) + else: + if verbose: + errmess( + "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" + % (k, k1, v1, v1, list(c2py_map.keys())) + ) + + return f2cmap_all, f2cmap_mapped diff --git a/contrib/python/numpy/py3/numpy/f2py/capi_maps.py b/contrib/python/numpy/py3/numpy/f2py/capi_maps.py index 32b6db5c59..fa477a5b9a 100644 --- a/contrib/python/numpy/py3/numpy/f2py/capi_maps.py +++ b/contrib/python/numpy/py3/numpy/f2py/capi_maps.py @@ -1,15 +1,10 @@ -#!/usr/bin/env python3 """ - -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version @@ -19,7 +14,7 @@ import re import os from .crackfortran import markoutercomma from . import cb_rules -from ._isocbind import iso_c_binding_map +from ._isocbind import iso_c_binding_map, isoc_c2pycode_map, iso_c2py_map # The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the @@ -29,7 +24,7 @@ from .auxfuncs import * __all__ = [ 'getctype', 'getstrlength', 'getarrdims', 'getpydocsign', 'getarrdocsign', 'getinit', 'sign2map', 'routsign2map', 'modsign2map', - 'cb_sign2map', 'cb_routsign2map', 'common_sign2map' + 'cb_sign2map', 'cb_routsign2map', 'common_sign2map', 'process_f2cmap_dict' ] @@ -131,13 +126,17 @@ f2cmap_all = {'real': {'': 'float', '4': 'float', '8': 'double', 'byte': {'': 'char'}, } -f2cmap_all = deep_merge(f2cmap_all, iso_c_binding_map) +# Add ISO_C handling +c2pycode_map.update(isoc_c2pycode_map) +c2py_map.update(iso_c2py_map) +f2cmap_all, _ = process_f2cmap_dict(f2cmap_all, iso_c_binding_map, c2py_map) +# End ISO_C handling f2cmap_default = copy.deepcopy(f2cmap_all) f2cmap_mapped = [] def load_f2cmap_file(f2cmap_file): - global f2cmap_all + global f2cmap_all, f2cmap_mapped f2cmap_all = copy.deepcopy(f2cmap_default) @@ -156,29 +155,11 @@ def load_f2cmap_file(f2cmap_file): outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) with open(f2cmap_file) as f: d = eval(f.read().lower(), {}, {}) - for k, d1 in d.items(): - for k1 in d1.keys(): - d1[k1.lower()] = d1[k1] - d[k.lower()] = d[k] - for k in d.keys(): - if k not in f2cmap_all: - f2cmap_all[k] = {} - for k1 in d[k].keys(): - if d[k][k1] in c2py_map: - if k1 in f2cmap_all[k]: - outmess( - "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" % (k, k1, f2cmap_all[k][k1], d[k][k1])) - f2cmap_all[k][k1] = d[k][k1] - outmess('\tMapping "%s(kind=%s)" to "%s"\n' % - (k, k1, d[k][k1])) - f2cmap_mapped.append(d[k][k1]) - else: - errmess("\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" % ( - k, k1, d[k][k1], d[k][k1], list(c2py_map.keys()))) + f2cmap_all, f2cmap_mapped = process_f2cmap_dict(f2cmap_all, d, c2py_map, True) outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: - errmess( - 'Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + errmess('Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + cformat_map = {'double': '%g', 'float': '%g', diff --git a/contrib/python/numpy/py3/numpy/f2py/cb_rules.py b/contrib/python/numpy/py3/numpy/f2py/cb_rules.py index 761831e004..721e075b6c 100644 --- a/contrib/python/numpy/py3/numpy/f2py/cb_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/cb_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build call-back mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/20 11:27:58 $ -Pearu Peterson - """ from . import __version__ from .auxfuncs import ( diff --git a/contrib/python/numpy/py3/numpy/f2py/cfuncs.py b/contrib/python/numpy/py3/numpy/f2py/cfuncs.py index f89793061b..4328a6e500 100644 --- a/contrib/python/numpy/py3/numpy/f2py/cfuncs.py +++ b/contrib/python/numpy/py3/numpy/f2py/cfuncs.py @@ -1,18 +1,14 @@ #!/usr/bin/env python3 """ - C declarations, CPP macros, and C functions for f2py2e. Only required declarations/macros/functions will be used. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 11:42:34 $ -Pearu Peterson - """ import sys import copy @@ -64,7 +60,7 @@ typedefs['unsigned_char'] = 'typedef unsigned char unsigned_char;' typedefs['unsigned_short'] = 'typedef unsigned short unsigned_short;' typedefs['unsigned_long'] = 'typedef unsigned long unsigned_long;' typedefs['signed_char'] = 'typedef signed char signed_char;' -typedefs['long_long'] = """\ +typedefs['long_long'] = """ #if defined(NPY_OS_WIN32) typedef __int64 long_long; #else @@ -72,14 +68,14 @@ typedef long long long_long; typedef unsigned long long unsigned_long_long; #endif """ -typedefs['unsigned_long_long'] = """\ +typedefs['unsigned_long_long'] = """ #if defined(NPY_OS_WIN32) typedef __uint64 long_long; #else typedef unsigned long long unsigned_long_long; #endif """ -typedefs['long_double'] = """\ +typedefs['long_double'] = """ #ifndef _LONG_DOUBLE typedef long double long_double; #endif @@ -93,7 +89,7 @@ typedefs['character'] = """typedef char character;""" ############### CPP macros #################### -cppmacros['CFUNCSMESS'] = """\ +cppmacros['CFUNCSMESS'] = """ #ifdef DEBUGCFUNCS #define CFUNCSMESS(mess) fprintf(stderr,\"debug-capi:\"mess); #define CFUNCSMESSPY(mess,obj) CFUNCSMESS(mess) \\ @@ -104,7 +100,7 @@ cppmacros['CFUNCSMESS'] = """\ #define CFUNCSMESSPY(mess,obj) #endif """ -cppmacros['F_FUNC'] = """\ +cppmacros['F_FUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -140,7 +136,7 @@ cppmacros['F_FUNC'] = """\ #define F_FUNC_US(f,F) F_FUNC(f,F) #endif """ -cppmacros['F_WRAPPEDFUNC'] = """\ +cppmacros['F_WRAPPEDFUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -176,7 +172,7 @@ cppmacros['F_WRAPPEDFUNC'] = """\ #define F_WRAPPEDFUNC_US(f,F) F_WRAPPEDFUNC(f,F) #endif """ -cppmacros['F_MODFUNC'] = """\ +cppmacros['F_MODFUNC'] = """ #if defined(F90MOD2CCONV1) /*E.g. Compaq Fortran */ #if defined(NO_APPEND_FORTRAN) #define F_MODFUNCNAME(m,f) $ ## m ## $ ## f @@ -210,12 +206,12 @@ cppmacros['F_MODFUNC'] = """\ #define F_MODFUNC(m,f) (*(f2pymodstruct##m##.##f)) """ -cppmacros['SWAPUNSAFE'] = """\ +cppmacros['SWAPUNSAFE'] = """ #define SWAP(a,b) (size_t)(a) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(b) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(a) = ((size_t)(a) ^ (size_t)(b)) """ -cppmacros['SWAP'] = """\ +cppmacros['SWAP'] = """ #define SWAP(a,b,t) {\\ t *c;\\ c = a;\\ @@ -224,13 +220,13 @@ cppmacros['SWAP'] = """\ """ # cppmacros['ISCONTIGUOUS']='#define ISCONTIGUOUS(m) (PyArray_FLAGS(m) & # NPY_ARRAY_C_CONTIGUOUS)' -cppmacros['PRINTPYOBJERR'] = """\ +cppmacros['PRINTPYOBJERR'] = """ #define PRINTPYOBJERR(obj)\\ fprintf(stderr,\"#modulename#.error is related to \");\\ PyObject_Print((PyObject *)obj,stderr,Py_PRINT_RAW);\\ fprintf(stderr,\"\\n\"); """ -cppmacros['MINMAX'] = """\ +cppmacros['MINMAX'] = """ #ifndef max #define max(a,b) ((a > b) ? (a) : (b)) #endif @@ -244,7 +240,7 @@ cppmacros['MINMAX'] = """\ #define MIN(a,b) ((a < b) ? (a) : (b)) #endif """ -cppmacros['len..'] = """\ +cppmacros['len..'] = """ /* See fortranobject.h for definitions. The macros here are provided for BC. */ #define rank f2py_rank #define shape f2py_shape @@ -254,16 +250,21 @@ cppmacros['len..'] = """\ #define slen f2py_slen #define size f2py_size """ -cppmacros[ - 'pyobj_from_char1'] = '#define pyobj_from_char1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_short1'] = '#define pyobj_from_short1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_char1'] = r""" +#define pyobj_from_char1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_short1'] = r""" +#define pyobj_from_short1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_int1'] = ['signed_char'] -cppmacros['pyobj_from_int1'] = '#define pyobj_from_int1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_long1'] = '#define pyobj_from_long1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_int1'] = r""" +#define pyobj_from_int1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_long1'] = r""" +#define pyobj_from_long1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_long_long1'] = ['long_long'] -cppmacros['pyobj_from_long_long1'] = """\ +cppmacros['pyobj_from_long_long1'] = """ #ifdef HAVE_LONG_LONG #define pyobj_from_long_long1(v) (PyLong_FromLongLong(v)) #else @@ -272,29 +273,29 @@ cppmacros['pyobj_from_long_long1'] = """\ #endif """ needs['pyobj_from_long_double1'] = ['long_double'] -cppmacros[ - 'pyobj_from_long_double1'] = '#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_double1'] = '#define pyobj_from_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_float1'] = '#define pyobj_from_float1(v) (PyFloat_FromDouble(v))' +cppmacros['pyobj_from_long_double1'] = """ +#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_double1'] = """ +#define pyobj_from_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_float1'] = """ +#define pyobj_from_float1(v) (PyFloat_FromDouble(v))""" needs['pyobj_from_complex_long_double1'] = ['complex_long_double'] -cppmacros[ - 'pyobj_from_complex_long_double1'] = '#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_long_double1'] = """ +#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_double1'] = ['complex_double'] -cppmacros[ - 'pyobj_from_complex_double1'] = '#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_double1'] = """ +#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_float1'] = ['complex_float'] -cppmacros[ - 'pyobj_from_complex_float1'] = '#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_float1'] = """ +#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_string1'] = ['string'] -cppmacros[ - 'pyobj_from_string1'] = '#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))' +cppmacros['pyobj_from_string1'] = """ +#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))""" needs['pyobj_from_string1size'] = ['string'] -cppmacros[ - 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))' +cppmacros['pyobj_from_string1size'] = """ +#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))""" needs['TRYPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYPYARRAYTEMPLATE'] = """\ +cppmacros['TRYPYARRAYTEMPLATE'] = """ /* New SciPy */ #define TRYPYARRAYTEMPLATECHAR case NPY_STRING: *(char *)(PyArray_DATA(arr))=*v; break; #define TRYPYARRAYTEMPLATELONG case NPY_LONG: *(long *)(PyArray_DATA(arr))=*v; break; @@ -331,7 +332,7 @@ cppmacros['TRYPYARRAYTEMPLATE'] = """\ """ needs['TRYCOMPLEXPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ +cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """ #define TRYCOMPLEXPYARRAYTEMPLATEOBJECT case NPY_OBJECT: PyArray_SETITEM(arr, PyArray_DATA(arr), pyobj_from_complex_ ## ctype ## 1((*v))); break; #define TRYCOMPLEXPYARRAYTEMPLATE(ctype,typecode)\\ PyArrayObject *arr = NULL;\\ @@ -372,7 +373,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ };\\ return -1; """ -# cppmacros['NUMFROMARROBJ']="""\ +# cppmacros['NUMFROMARROBJ']=""" # define NUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -388,7 +389,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ # } # """ # XXX: Note that CNUMFROMARROBJ is identical with NUMFROMARROBJ -# cppmacros['CNUMFROMARROBJ']="""\ +# cppmacros['CNUMFROMARROBJ']=""" # define CNUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -406,7 +407,7 @@ cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ needs['GETSTRFROMPYTUPLE'] = ['STRINGCOPYN', 'PRINTPYOBJERR'] -cppmacros['GETSTRFROMPYTUPLE'] = """\ +cppmacros['GETSTRFROMPYTUPLE'] = """ #define GETSTRFROMPYTUPLE(tuple,index,str,len) {\\ PyObject *rv_cb_str = PyTuple_GetItem((tuple),(index));\\ if (rv_cb_str == NULL)\\ @@ -421,7 +422,7 @@ cppmacros['GETSTRFROMPYTUPLE'] = """\ }\\ } """ -cppmacros['GETSCALARFROMPYTUPLE'] = """\ +cppmacros['GETSCALARFROMPYTUPLE'] = """ #define GETSCALARFROMPYTUPLE(tuple,index,var,ctype,mess) {\\ if ((capi_tmp = PyTuple_GetItem((tuple),(index)))==NULL) goto capi_fail;\\ if (!(ctype ## _from_pyobj((var),capi_tmp,mess)))\\ @@ -429,7 +430,7 @@ cppmacros['GETSCALARFROMPYTUPLE'] = """\ } """ -cppmacros['FAILNULL'] = """\\ +cppmacros['FAILNULL'] = """\ #define FAILNULL(p) do { \\ if ((p) == NULL) { \\ PyErr_SetString(PyExc_MemoryError, "NULL pointer found"); \\ @@ -438,11 +439,11 @@ cppmacros['FAILNULL'] = """\\ } while (0) """ needs['MEMCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['MEMCOPY'] = """\ +cppmacros['MEMCOPY'] = """ #define MEMCOPY(to,from,n)\\ do { FAILNULL(to); FAILNULL(from); (void)memcpy(to,from,n); } while (0) """ -cppmacros['STRINGMALLOC'] = """\ +cppmacros['STRINGMALLOC'] = """ #define STRINGMALLOC(str,len)\\ if ((str = (string)malloc(len+1)) == NULL) {\\ PyErr_SetString(PyExc_MemoryError, \"out of memory\");\\ @@ -451,11 +452,11 @@ cppmacros['STRINGMALLOC'] = """\ (str)[len] = '\\0';\\ } """ -cppmacros['STRINGFREE'] = """\ +cppmacros['STRINGFREE'] = """ #define STRINGFREE(str) do {if (!(str == NULL)) free(str);} while (0) """ needs['STRINGPADN'] = ['string.h'] -cppmacros['STRINGPADN'] = """\ +cppmacros['STRINGPADN'] = """ /* STRINGPADN replaces null values with padding values from the right. @@ -476,7 +477,7 @@ STRINGPADN(to, N, PADDING, NULLVALUE) is an inverse operation. } while (0) """ needs['STRINGCOPYN'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPYN'] = """\ +cppmacros['STRINGCOPYN'] = """ /* STRINGCOPYN copies N bytes. @@ -492,23 +493,23 @@ STRINGCOPYN copies N bytes. } while (0) """ needs['STRINGCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPY'] = """\ +cppmacros['STRINGCOPY'] = """ #define STRINGCOPY(to,from)\\ do { FAILNULL(to); FAILNULL(from); (void)strcpy(to,from); } while (0) """ -cppmacros['CHECKGENERIC'] = """\ +cppmacros['CHECKGENERIC'] = """ #define CHECKGENERIC(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKARRAY'] = """\ +cppmacros['CHECKARRAY'] = """ #define CHECKARRAY(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSTRING'] = """\ +cppmacros['CHECKSTRING'] = """ #define CHECKSTRING(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -516,7 +517,7 @@ cppmacros['CHECKSTRING'] = """\ PyErr_SetString(#modulename#_error, errstring);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSCALAR'] = """\ +cppmacros['CHECKSCALAR'] = """ #define CHECKSCALAR(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -524,7 +525,7 @@ cppmacros['CHECKSCALAR'] = """\ PyErr_SetString(#modulename#_error,errstring);\\ /*goto capi_fail;*/\\ } else """ -# cppmacros['CHECKDIMS']="""\ +# cppmacros['CHECKDIMS']=""" # define CHECKDIMS(dims,rank) \\ # for (int i=0;i<(rank);i++)\\ # if (dims[i]<0) {\\ @@ -534,12 +535,12 @@ cppmacros['CHECKSCALAR'] = """\ # """ cppmacros[ 'ARRSIZE'] = '#define ARRSIZE(dims,rank) (_PyArray_multiply_list(dims,rank))' -cppmacros['OLDPYNUM'] = """\ +cppmacros['OLDPYNUM'] = """ #ifdef OLDPYNUM #error You need to install NumPy version 0.13 or higher. See https://scipy.org/install.html #endif """ -cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ +cppmacros["F2PY_THREAD_LOCAL_DECL"] = """ #ifndef F2PY_THREAD_LOCAL_DECL #if defined(_MSC_VER) #define F2PY_THREAD_LOCAL_DECL __declspec(thread) @@ -565,21 +566,21 @@ cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ """ ################# C functions ############### -cfuncs['calcarrindex'] = """\ +cfuncs['calcarrindex'] = """ static int calcarrindex(int *i,PyArrayObject *arr) { int k,ii = i[0]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,k) - 1)+i[k]); /* assuming contiguous arr */ return ii; }""" -cfuncs['calcarrindextr'] = """\ +cfuncs['calcarrindextr'] = """ static int calcarrindextr(int *i,PyArrayObject *arr) { int k,ii = i[PyArray_NDIM(arr)-1]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,PyArray_NDIM(arr)-k-1) - 1)+i[PyArray_NDIM(arr)-k-1]); /* assuming contiguous arr */ return ii; }""" -cfuncs['forcomb'] = """\ +cfuncs['forcomb'] = """ static struct { int nd;npy_intp *d;int *i,*i_tr,tr; } forcombcache; static int initforcomb(npy_intp *dims,int nd,int tr) { int k; @@ -620,7 +621,7 @@ static int *nextforcomb(void) { return i; }""" needs['try_pyarr_from_string'] = ['STRINGCOPYN', 'PRINTPYOBJERR', 'string'] -cfuncs['try_pyarr_from_string'] = """\ +cfuncs['try_pyarr_from_string'] = """ /* try_pyarr_from_string copies str[:len(obj)] to the data of an `ndarray`. @@ -634,6 +635,9 @@ static int try_pyarr_from_string(PyObject *obj, fprintf(stderr, "try_pyarr_from_string(str='%s', len=%d, obj=%p)\\n", (char*)str,len, obj); #endif + if (!obj) return -2; /* Object missing */ + if (obj == Py_None) return -1; /* None */ + if (!PyArray_Check(obj)) goto capi_fail; /* not an ndarray */ if (PyArray_Check(obj)) { PyArrayObject *arr = (PyArrayObject *)obj; assert(ISCONTIGUOUS(arr)); @@ -656,7 +660,7 @@ capi_fail: } """ needs['string_from_pyobj'] = ['string', 'STRINGMALLOC', 'STRINGCOPYN'] -cfuncs['string_from_pyobj'] = """\ +cfuncs['string_from_pyobj'] = """ /* Create a new string buffer `str` of at most length `len` from a Python string-like object `obj`. @@ -756,7 +760,7 @@ capi_fail: } """ -cfuncs['character_from_pyobj'] = """\ +cfuncs['character_from_pyobj'] = """ static int character_from_pyobj(character* v, PyObject *obj, const char *errmess) { if (PyBytes_Check(obj)) { @@ -823,8 +827,10 @@ character_from_pyobj(character* v, PyObject *obj, const char *errmess) { } """ +# TODO: These should be dynamically generated, too many mapped to int things, +# see note in _isocbind.py needs['char_from_pyobj'] = ['int_from_pyobj'] -cfuncs['char_from_pyobj'] = """\ +cfuncs['char_from_pyobj'] = """ static int char_from_pyobj(char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -838,7 +844,7 @@ char_from_pyobj(char* v, PyObject *obj, const char *errmess) { needs['signed_char_from_pyobj'] = ['int_from_pyobj', 'signed_char'] -cfuncs['signed_char_from_pyobj'] = """\ +cfuncs['signed_char_from_pyobj'] = """ static int signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -852,7 +858,7 @@ signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { needs['short_from_pyobj'] = ['int_from_pyobj'] -cfuncs['short_from_pyobj'] = """\ +cfuncs['short_from_pyobj'] = """ static int short_from_pyobj(short* v, PyObject *obj, const char *errmess) { int i = 0; @@ -865,7 +871,7 @@ short_from_pyobj(short* v, PyObject *obj, const char *errmess) { """ -cfuncs['int_from_pyobj'] = """\ +cfuncs['int_from_pyobj'] = """ static int int_from_pyobj(int* v, PyObject *obj, const char *errmess) { @@ -915,7 +921,7 @@ int_from_pyobj(int* v, PyObject *obj, const char *errmess) """ -cfuncs['long_from_pyobj'] = """\ +cfuncs['long_from_pyobj'] = """ static int long_from_pyobj(long* v, PyObject *obj, const char *errmess) { PyObject* tmp = NULL; @@ -964,7 +970,7 @@ long_from_pyobj(long* v, PyObject *obj, const char *errmess) { needs['long_long_from_pyobj'] = ['long_long'] -cfuncs['long_long_from_pyobj'] = """\ +cfuncs['long_long_from_pyobj'] = """ static int long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) { @@ -1014,7 +1020,7 @@ long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) needs['long_double_from_pyobj'] = ['double_from_pyobj', 'long_double'] -cfuncs['long_double_from_pyobj'] = """\ +cfuncs['long_double_from_pyobj'] = """ static int long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) { @@ -1038,7 +1044,7 @@ long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) """ -cfuncs['double_from_pyobj'] = """\ +cfuncs['double_from_pyobj'] = """ static int double_from_pyobj(double* v, PyObject *obj, const char *errmess) { @@ -1082,7 +1088,7 @@ double_from_pyobj(double* v, PyObject *obj, const char *errmess) needs['float_from_pyobj'] = ['double_from_pyobj'] -cfuncs['float_from_pyobj'] = """\ +cfuncs['float_from_pyobj'] = """ static int float_from_pyobj(float* v, PyObject *obj, const char *errmess) { @@ -1098,7 +1104,7 @@ float_from_pyobj(float* v, PyObject *obj, const char *errmess) needs['complex_long_double_from_pyobj'] = ['complex_long_double', 'long_double', 'complex_double_from_pyobj', 'npy_math.h'] -cfuncs['complex_long_double_from_pyobj'] = """\ +cfuncs['complex_long_double_from_pyobj'] = """ static int complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char *errmess) { @@ -1125,7 +1131,7 @@ complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char needs['complex_double_from_pyobj'] = ['complex_double', 'npy_math.h'] -cfuncs['complex_double_from_pyobj'] = """\ +cfuncs['complex_double_from_pyobj'] = """ static int complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) { Py_complex c; @@ -1202,7 +1208,7 @@ complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) needs['complex_float_from_pyobj'] = [ 'complex_float', 'complex_double_from_pyobj'] -cfuncs['complex_float_from_pyobj'] = """\ +cfuncs['complex_float_from_pyobj'] = """ static int complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) { @@ -1217,7 +1223,7 @@ complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) """ -cfuncs['try_pyarr_from_character'] = """\ +cfuncs['try_pyarr_from_character'] = """ static int try_pyarr_from_character(PyObject* obj, character* v) { PyArrayObject *arr = (PyArrayObject*)obj; if (!obj) return -2; @@ -1282,7 +1288,7 @@ cfuncs[ needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] # create the list of arguments to be used when calling back to python -cfuncs['create_cb_arglist'] = """\ +cfuncs['create_cb_arglist'] = """ static int create_cb_arglist(PyObject* fun, PyTupleObject* xa , const int maxnofargs, const int nofoptargs, int *nofargs, PyTupleObject **args, diff --git a/contrib/python/numpy/py3/numpy/f2py/common_rules.py b/contrib/python/numpy/py3/numpy/f2py/common_rules.py index 5a488bf5a5..64347b7374 100644 --- a/contrib/python/numpy/py3/numpy/f2py/common_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/common_rules.py @@ -1,23 +1,18 @@ -#!/usr/bin/env python3 """ - Build common block mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version from .auxfuncs import ( - hasbody, hascommon, hasnote, isintent_hide, outmess + hasbody, hascommon, hasnote, isintent_hide, outmess, getuseblocks ) from . import capi_maps from . import func2subr @@ -78,6 +73,8 @@ def buildhooks(m): outmess('\t\tConstructing COMMON block support for "%s"...\n\t\t %s\n' % ( name, ','.join(inames))) fadd('subroutine f2pyinit%s(setupfunc)' % name) + for usename in getuseblocks(m): + fadd(f'use {usename}') fadd('external setupfunc') for n in vnames: fadd(func2subr.var2fixfortran(vars, n)) diff --git a/contrib/python/numpy/py3/numpy/f2py/crackfortran.py b/contrib/python/numpy/py3/numpy/f2py/crackfortran.py index f352bbaa27..8d3fc27608 100755 --- a/contrib/python/numpy/py3/numpy/f2py/crackfortran.py +++ b/contrib/python/numpy/py3/numpy/f2py/crackfortran.py @@ -2,14 +2,12 @@ """ crackfortran --- read fortran (77,90) code and extract declaration information. -Copyright 1999-2004 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/09/27 07:13:49 $ -Pearu Peterson Usage of crackfortran: @@ -995,6 +993,16 @@ def _resolvenameargspattern(line): def analyzeline(m, case, line): + """ + Reads each line in the input file in sequence and updates global vars. + + Effectively reads and collects information from the input file to the + global variable groupcache, a dictionary containing info about each part + of the fortran module. + + At the end of analyzeline, information is filtered into the correct dict + keys, but parameter values and dimensions are not yet interpreted. + """ global groupcounter, groupname, groupcache, grouplist, filepositiontext global currentfilename, f77modulename, neededinterface, neededmodule global expectbegin, gotnextfile, previous_context @@ -1681,10 +1689,18 @@ def markinnerspaces(line): def updatevars(typespec, selector, attrspec, entitydecl): + """ + Returns last_name, the variable name without special chars, parenthesis + or dimension specifiers. + + Alters groupcache to add the name, typespec, attrspec (and possibly value) + of current variable. + """ global groupcache, groupcounter last_name = None kindselect, charselect, typename = cracktypespec(typespec, selector) + # Clean up outer commas, whitespace and undesired chars from attrspec if attrspec: attrspec = [x.strip() for x in markoutercomma(attrspec).split('@,@')] l = [] @@ -2398,8 +2414,6 @@ def _calc_depend_dict(vars): def get_sorted_names(vars): - """ - """ depend_dict = _calc_depend_dict(vars) names = [] for name in list(depend_dict.keys()): @@ -2452,7 +2466,7 @@ def _selected_real_kind_func(p, r=0, radix=0): if p < 16: return 8 machine = platform.machine().lower() - if machine.startswith(('aarch64', 'arm64', 'loongarch', 'power', 'ppc', 'riscv', 's390x', 'sparc')): + if machine.startswith(('aarch64', 'alpha', 'arm64', 'loongarch', 'mips', 'power', 'ppc', 'riscv', 's390x', 'sparc')): if p <= 33: return 16 else: @@ -2491,6 +2505,7 @@ def get_parameters(vars, global_params={}): # TODO: test .eq., .neq., etc replacements. ]: v = v.replace(*repl) + v = kind_re.sub(r'kind("\1")', v) v = selected_int_kind_re.sub(r'selected_int_kind(\1)', v) @@ -2499,14 +2514,17 @@ def get_parameters(vars, global_params={}): # then we may easily remove those specifiers. # However, it may be that the user uses other specifiers...(!) is_replaced = False + if 'kindselector' in vars[n]: + # Remove kind specifier (including those defined + # by parameters) if 'kind' in vars[n]['kindselector']: orig_v_len = len(v) v = v.replace('_' + vars[n]['kindselector']['kind'], '') # Again, this will be true if even a single specifier # has been replaced, see comment above. is_replaced = len(v) < orig_v_len - + if not is_replaced: if not selected_kind_re.match(v): v_ = v.split('_') @@ -2533,6 +2551,10 @@ def get_parameters(vars, global_params={}): outmess(f'get_parameters[TODO]: ' f'implement evaluation of complex expression {v}\n') + dimspec = ([s.lstrip('dimension').strip() + for s in vars[n]['attrspec'] + if s.startswith('dimension')] or [None])[0] + # Handle _dp for gh-6624 # Also fixes gh-20460 if real16pattern.search(v): @@ -2540,11 +2562,11 @@ def get_parameters(vars, global_params={}): elif real8pattern.search(v): v = 4 try: - params[n] = eval(v, g_params, params) - + params[n] = param_eval(v, g_params, params, dimspec=dimspec) except Exception as msg: params[n] = v - outmess('get_parameters: got "%s" on %s\n' % (msg, repr(v))) + outmess(f'get_parameters: got "{msg}" on {n!r}\n') + if isstring(vars[n]) and isinstance(params[n], int): params[n] = chr(params[n]) nl = n.lower() @@ -2552,8 +2574,7 @@ def get_parameters(vars, global_params={}): params[nl] = params[n] else: print(vars[n]) - outmess( - 'get_parameters:parameter %s does not have value?!\n' % (repr(n))) + outmess(f'get_parameters:parameter {n!r} does not have value?!\n') return params @@ -2562,6 +2583,7 @@ def _eval_length(length, params): return '(*)' return _eval_scalar(length, params) + _is_kind_number = re.compile(r'\d+_').match @@ -2582,6 +2604,10 @@ def _eval_scalar(value, params): def analyzevars(block): + """ + Sets correct dimension information for each variable/parameter + """ + global f90modulevars setmesstext(block) @@ -2610,7 +2636,8 @@ def analyzevars(block): svars.append(n) params = get_parameters(vars, get_useparameters(block)) - + # At this point, params are read and interpreted, but + # the params used to define vars are not yet parsed dep_matches = {} name_match = re.compile(r'[A-Za-z][\w$]*').match for v in list(vars.keys()): @@ -2709,27 +2736,30 @@ def analyzevars(block): check = None if dim and 'dimension' not in vars[n]: vars[n]['dimension'] = [] - for d in rmbadname([x.strip() for x in markoutercomma(dim).split('@,@')]): - star = ':' if d == ':' else '*' + for d in rmbadname( + [x.strip() for x in markoutercomma(dim).split('@,@')] + ): + # d is the expression inside the dimension declaration # Evaluate `d` with respect to params - if d in params: - d = str(params[d]) - for p in params: - re_1 = re.compile(r'(?P<before>.*?)\b' + p + r'\b(?P<after>.*)', re.I) - m = re_1.match(d) - while m: - d = m.group('before') + \ - str(params[p]) + m.group('after') - m = re_1.match(d) - - if d == star: - dl = [star] + try: + # the dimension for this variable depends on a + # previously defined parameter + d = param_parse(d, params) + except (ValueError, IndexError, KeyError): + outmess( + ('analyzevars: could not parse dimension for ' + f'variable {d!r}\n') + ) + + dim_char = ':' if d == ':' else '*' + if d == dim_char: + dl = [dim_char] else: dl = markoutercomma(d, ':').split('@:@') if len(dl) == 2 and '*' in dl: # e.g. dimension(5:*) dl = ['*'] d = '*' - if len(dl) == 1 and dl[0] != star: + if len(dl) == 1 and dl[0] != dim_char: dl = ['1', dl[0]] if len(dl) == 2: d1, d2 = map(symbolic.Expr.parse, dl) @@ -2963,9 +2993,152 @@ def analyzevars(block): del vars[n] return vars + analyzeargs_re_1 = re.compile(r'\A[a-z]+[\w$]*\Z', re.I) +def param_eval(v, g_params, params, dimspec=None): + """ + Creates a dictionary of indices and values for each parameter in a + parameter array to be evaluated later. + + WARNING: It is not possible to initialize multidimensional array + parameters e.g. dimension(-3:1, 4, 3:5) at this point. This is because in + Fortran initialization through array constructor requires the RESHAPE + intrinsic function. Since the right-hand side of the parameter declaration + is not executed in f2py, but rather at the compiled c/fortran extension, + later, it is not possible to execute a reshape of a parameter array. + One issue remains: if the user wants to access the array parameter from + python, we should either + 1) allow them to access the parameter array using python standard indexing + (which is often incompatible with the original fortran indexing) + 2) allow the parameter array to be accessed in python as a dictionary with + fortran indices as keys + We are choosing 2 for now. + """ + if dimspec is None: + try: + p = eval(v, g_params, params) + except Exception as msg: + p = v + outmess(f'param_eval: got "{msg}" on {v!r}\n') + return p + + # This is an array parameter. + # First, we parse the dimension information + if len(dimspec) < 2 or dimspec[::len(dimspec)-1] != "()": + raise ValueError(f'param_eval: dimension {dimspec} can\'t be parsed') + dimrange = dimspec[1:-1].split(',') + if len(dimrange) == 1: + # e.g. dimension(2) or dimension(-1:1) + dimrange = dimrange[0].split(':') + # now, dimrange is a list of 1 or 2 elements + if len(dimrange) == 1: + bound = param_parse(dimrange[0], params) + dimrange = range(1, int(bound)+1) + else: + lbound = param_parse(dimrange[0], params) + ubound = param_parse(dimrange[1], params) + dimrange = range(int(lbound), int(ubound)+1) + else: + raise ValueError(f'param_eval: multidimensional array parameters ' + '{dimspec} not supported') + + # Parse parameter value + v = (v[2:-2] if v.startswith('(/') else v).split(',') + v_eval = [] + for item in v: + try: + item = eval(item, g_params, params) + except Exception as msg: + outmess(f'param_eval: got "{msg}" on {item!r}\n') + v_eval.append(item) + + p = dict(zip(dimrange, v_eval)) + + return p + + +def param_parse(d, params): + """Recursively parse array dimensions. + + Parses the declaration of an array variable or parameter + `dimension` keyword, and is called recursively if the + dimension for this array is a previously defined parameter + (found in `params`). + + Parameters + ---------- + d : str + Fortran expression describing the dimension of an array. + params : dict + Previously parsed parameters declared in the Fortran source file. + + Returns + ------- + out : str + Parsed dimension expression. + + Examples + -------- + + * If the line being analyzed is + + `integer, parameter, dimension(2) :: pa = (/ 3, 5 /)` + + then `d = 2` and we return immediately, with + + >>> d = '2' + >>> param_parse(d, params) + 2 + + * If the line being analyzed is + + `integer, parameter, dimension(pa) :: pb = (/1, 2, 3/)` + + then `d = 'pa'`; since `pa` is a previously parsed parameter, + and `pa = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa' + >>> params = {'pa': 3} + >>> param_parse(d, params) + 3 + + * If the line being analyzed is + + `integer, parameter, dimension(pa(1)) :: pb = (/1, 2, 3/)` + + then `d = 'pa(1)'`; since `pa` is a previously parsed parameter, + and `pa(1) = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa(1)' + >>> params = dict(pa={1: 3, 2: 5}) + >>> param_parse(d, params) + 3 + """ + if "(" in d: + # this dimension expression is an array + dname = d[:d.find("(")] + ddims = d[d.find("(")+1:d.rfind(")")] + # this dimension expression is also a parameter; + # parse it recursively + index = int(param_parse(ddims, params)) + return str(params[dname][index]) + elif d in params: + return str(params[d]) + else: + for p in params: + re_1 = re.compile( + r'(?P<before>.*?)\b' + p + r'\b(?P<after>.*)', re.I + ) + m = re_1.match(d) + while m: + d = m.group('before') + \ + str(params[p]) + m.group('after') + m = re_1.match(d) + return d + + def expr2name(a, block, args=[]): orig_a = a a_is_expr = not analyzeargs_re_1.match(a) @@ -3218,11 +3391,6 @@ def true_intent_list(var): def vars2fortran(block, vars, args, tab='', as_interface=False): - """ - TODO: - public sub - ... - """ setmesstext(block) ret = '' nout = [] diff --git a/contrib/python/numpy/py3/numpy/f2py/f2py2e.py b/contrib/python/numpy/py3/numpy/f2py/f2py2e.py index 1cfe8cddd6..ce22b2d8a9 100755 --- a/contrib/python/numpy/py3/numpy/f2py/f2py2e.py +++ b/contrib/python/numpy/py3/numpy/f2py/f2py2e.py @@ -4,15 +4,12 @@ f2py2e - Fortran to Python C/API generator. 2nd Edition. See __usage__ below. -Copyright 1999--2011 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@cens.ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 08:31:19 $ -Pearu Peterson - """ import sys import os @@ -21,6 +18,7 @@ import re from pathlib import Path from itertools import dropwhile import argparse +import copy from . import crackfortran from . import rules @@ -38,6 +36,7 @@ errmess = sys.stderr.write # outmess=sys.stdout.write show = pprint.pprint outmess = auxfuncs.outmess +MESON_ONLY_VER = (sys.version_info >= (3, 12)) __usage__ =\ f"""Usage: @@ -63,12 +62,6 @@ Description: This program generates a Python C/API file (<modulename>module.c) Options: - --2d-numpy Use numpy.f2py tool with NumPy support. [DEFAULT] - --2d-numeric Use f2py2e tool with Numeric support. - --2d-numarray Use f2py2e tool with Numarray support. - --g3-numpy Use 3rd generation f2py from the separate f2py package. - [NOT AVAILABLE YET] - -h <filename> Write signatures of the fortran routines to file <filename> and exit. You can then edit <filename> and use it instead of <fortran files>. If <filename>==stdout then the @@ -129,20 +122,22 @@ Options: -v Print f2py version ID and exit. -build backend options (only effective with -c): +build backend options (only effective with -c) +[NO_MESON] is used to indicate an option not meant to be used +with the meson backend or above Python 3.12: - --fcompiler= Specify Fortran compiler type by vendor - --compiler= Specify C compiler type (as defined by distutils) + --fcompiler= Specify Fortran compiler type by vendor [NO_MESON] + --compiler= Specify distutils C compiler type [NO_MESON] - --help-fcompiler List available Fortran compilers and exit - --f77exec= Specify the path to F77 compiler - --f90exec= Specify the path to F90 compiler + --help-fcompiler List available Fortran compilers and exit [NO_MESON] + --f77exec= Specify the path to F77 compiler [NO_MESON] + --f90exec= Specify the path to F90 compiler [NO_MESON] --f77flags= Specify F77 compiler flags --f90flags= Specify F90 compiler flags - --opt= Specify optimization flags - --arch= Specify architecture specific optimization flags - --noopt Compile without optimization - --noarch Compile without arch-dependent optimization + --opt= Specify optimization flags [NO_MESON] + --arch= Specify architecture specific optimization flags [NO_MESON] + --noopt Compile without optimization [NO_MESON] + --noarch Compile without arch-dependent optimization [NO_MESON] --debug Compile with debugging information --dep <dependency> @@ -167,7 +162,7 @@ Extra options (only effective with -c): by numpy.distutils/system_info.py. E.g. to link with optimized LAPACK libraries (vecLib on MacOSX, ATLAS elsewhere), use --link-lapack_opt. - See also --help-link switch. + See also --help-link switch. [NO_MESON] -L/path/to/lib/ -l<libname> -D<define> -U<name> @@ -189,15 +184,15 @@ Extra options (only effective with -c): Version: {f2py_version} numpy Version: {numpy_version} -Requires: Python 3.5 or higher. License: NumPy license (see LICENSE.txt in the NumPy source code) -Copyright 1999 - 2011 Pearu Peterson all rights reserved. -https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e""" +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +https://numpy.org/doc/stable/f2py/index.html\n""" def scaninputline(inputline): files, skipfuncs, onlyfuncs, debug = [], [], [], [] - f, f2, f3, f5, f6, f7, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0, 0 + f, f2, f3, f5, f6, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0 verbose = 1 emptygen = True dolc = -1 @@ -205,7 +200,7 @@ def scaninputline(inputline): dorestdoc = 0 wrapfuncs = 1 buildpath = '.' - include_paths = [] + include_paths, inputline = get_includes(inputline) signsfile, modulename = None, None options = {'buildpath': buildpath, 'coutput': None, @@ -265,14 +260,6 @@ def scaninputline(inputline): elif l[:8] == '-include': cfuncs.outneeds['userincludes'].append(l[9:-1]) cfuncs.userincludes[l[9:-1]] = '#include ' + l[8:] - elif l[:15] in '--include_paths': - outmess( - 'f2py option --include_paths is deprecated, use --include-paths instead.\n') - f7 = 1 - elif l[:15] in '--include-paths': - # Similar to using -I with -c, however this is - # also used during generation of wrappers - f7 = 1 elif l == '--skip-empty-wrappers': emptygen = False elif l[0] == '-': @@ -287,9 +274,6 @@ def scaninputline(inputline): elif f6: f6 = 0 buildpath = l - elif f7: - f7 = 0 - include_paths.extend(l.split(os.pathsep)) elif f8: f8 = 0 options["coutput"] = l @@ -456,6 +440,23 @@ def run_main(comline_list): f2pydir = os.path.dirname(os.path.abspath(cfuncs.__file__)) fobjhsrc = os.path.join(f2pydir, 'src', 'fortranobject.h') fobjcsrc = os.path.join(f2pydir, 'src', 'fortranobject.c') + # gh-22819 -- begin + parser = make_f2py_compile_parser() + args, comline_list = parser.parse_known_args(comline_list) + pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) + # Checks that no existing modulename is defined in a pyf file + # TODO: Remove all this when scaninputline is replaced + if args.module_name: + if "-h" in comline_list: + modname = ( + args.module_name + ) # Directly use from args when -h is present + else: + modname = validate_modulename( + pyf_files, args.module_name + ) # Validate modname when -h is not present + comline_list += ['-m', modname] # needed for the rest of scaninputline + # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options capi_maps.load_f2cmap_file(options['f2cmap_file']) @@ -522,24 +523,59 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p -def preparse_sysargv(): - # To keep backwards bug compatibility, newer flags are handled by argparse, - # and `sys.argv` is passed to the rest of `f2py` as is. + +class CombineIncludePaths(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + include_paths_set = set(getattr(namespace, 'include_paths', []) or []) + if option_string == "--include_paths": + outmess("Use --include-paths or -I instead of --include_paths which will be removed") + if option_string == "--include-paths" or option_string == "--include_paths": + include_paths_set.update(values.split(':')) + else: + include_paths_set.add(values) + setattr(namespace, 'include_paths', list(include_paths_set)) + +def include_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("-I", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include-paths", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include_paths", dest="include_paths", action=CombineIncludePaths) + return parser + +def get_includes(iline): + iline = (' '.join(iline)).split() + parser = include_parser() + args, remain = parser.parse_known_args(iline) + ipaths = args.include_paths + if args.include_paths is None: + ipaths = [] + return ipaths, remain + +def make_f2py_compile_parser(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--dep", action="append", dest="dependencies") parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') + parser.add_argument("-m", dest="module_name") + return parser + +def preparse_sysargv(): + # To keep backwards bug compatibility, newer flags are handled by argparse, + # and `sys.argv` is passed to the rest of `f2py` as is. + parser = make_f2py_compile_parser() args, remaining_argv = parser.parse_known_args() sys.argv = [sys.argv[0]] + remaining_argv backend_key = args.backend - if sys.version_info >= (3, 12) and backend_key == 'distutils': - outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') - backend_key = 'meson' + if MESON_ONLY_VER and backend_key == 'distutils': + outmess("Cannot use distutils backend with Python>=3.12," + " using meson backend instead.\n") + backend_key = "meson" return { "dependencies": args.dependencies or [], - "backend": backend_key + "backend": backend_key, + "modulename": args.module_name, } def run_compile(): @@ -550,11 +586,13 @@ def run_compile(): # Collect dependency flags, preprocess sys.argv argy = preparse_sysargv() + modulename = argy["modulename"] + if modulename is None: + modulename = 'untitled' dependencies = argy["dependencies"] backend_key = argy["backend"] build_backend = f2py_build_generator(backend_key) - i = sys.argv.index('-c') del sys.argv[i] @@ -607,21 +645,27 @@ def run_compile(): for s in flib_flags: v = '--fcompiler=' if s[:len(v)] == v: - from numpy.distutils import fcompiler - fcompiler.load_all_fcompiler_classes() - allowed_keys = list(fcompiler.fcompiler_class.keys()) - nv = ov = s[len(v):].lower() - if ov not in allowed_keys: - vmap = {} # XXX - try: - nv = vmap[ov] - except KeyError: - if ov not in vmap.values(): - print('Unknown vendor: "%s"' % (s[len(v):])) - nv = ov - i = flib_flags.index(s) - flib_flags[i] = '--fcompiler=' + nv - continue + if MESON_ONLY_VER or backend_key == 'meson': + outmess( + "--fcompiler cannot be used with meson," + "set compiler with the FC environment variable\n" + ) + else: + from numpy.distutils import fcompiler + fcompiler.load_all_fcompiler_classes() + allowed_keys = list(fcompiler.fcompiler_class.keys()) + nv = ov = s[len(v):].lower() + if ov not in allowed_keys: + vmap = {} # XXX + try: + nv = vmap[ov] + except KeyError: + if ov not in vmap.values(): + print('Unknown vendor: "%s"' % (s[len(v):])) + nv = ov + i = flib_flags.index(s) + flib_flags[i] = '--fcompiler=' + nv + continue for s in del_list: i = flib_flags.index(s) del flib_flags[i] @@ -634,32 +678,19 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') - modulename = 'untitled' + # Ugly filter to remove everything but sources sources = sys.argv[1:] - - for optname in ['--include_paths', '--include-paths', '--f2cmap']: - if optname in sys.argv: - i = sys.argv.index(optname) - f2py_flags.extend(sys.argv[i:i + 2]) - del sys.argv[i + 1], sys.argv[i] - sources = sys.argv[1:] - - pyf_files = [] - if '-m' in sys.argv: - i = sys.argv.index('-m') - modulename = sys.argv[i + 1] + f2cmapopt = '--f2cmap' + if f2cmapopt in sys.argv: + i = sys.argv.index(f2cmapopt) + f2py_flags.extend(sys.argv[i:i + 2]) del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] - else: - pyf_files, _sources = filter_files('', '[.]pyf([.]src|)', sources) - sources = pyf_files + _sources - for f in pyf_files: - modulename = auxfuncs.get_f2py_modulename(f) - if modulename: - break + pyf_files, _sources = filter_files("", "[.]pyf([.]src|)", sources) + sources = pyf_files + _sources + modulename = validate_modulename(pyf_files, modulename) extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) - include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) libraries, sources = filter_files('-l', '', sources, remove_prefix=1) undef_macros, sources = filter_files('-U', '', sources, remove_prefix=1) @@ -675,13 +706,15 @@ def run_compile(): # Construct wrappers / signatures / things if backend_key == 'meson': - outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') - f2py_flags.append('--lower') - if pyf_files: - run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) - else: + if not pyf_files: + outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html\n') + f2py_flags.append('--lower') run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) + else: + run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) + # Order matters here, includes are needed for run_main above + include_dirs, sources = get_includes(sources) # Now use the builder builder = build_backend( modulename, @@ -704,30 +737,31 @@ def run_compile(): builder.compile() + +def validate_modulename(pyf_files, modulename='untitled'): + if len(pyf_files) > 1: + raise ValueError("Only one .pyf file per call") + if pyf_files: + pyff = pyf_files[0] + pyf_modname = auxfuncs.get_f2py_modulename(pyff) + if modulename != pyf_modname: + outmess( + f"Ignoring -m {modulename}.\n" + f"{pyff} defines {pyf_modname} to be the modulename.\n" + ) + modulename = pyf_modname + return modulename + def main(): if '--help-link' in sys.argv[1:]: sys.argv.remove('--help-link') - from numpy.distutils.system_info import show_all - show_all() + if MESON_ONLY_VER: + outmess("Use --dep for meson builds\n") + else: + from numpy.distutils.system_info import show_all + show_all() return - # Probably outdated options that were not working before 1.16 - if '--g3-numpy' in sys.argv[1:]: - sys.stderr.write("G3 f2py support is not implemented, yet.\\n") - sys.exit(1) - elif '--2e-numeric' in sys.argv[1:]: - sys.argv.remove('--2e-numeric') - elif '--2e-numarray' in sys.argv[1:]: - # Note that this errors becaust the -DNUMARRAY argument is - # not recognized. Just here for back compatibility and the - # error message. - sys.argv.append("-DNUMARRAY") - sys.argv.remove('--2e-numarray') - elif '--2e-numpy' in sys.argv[1:]: - sys.argv.remove('--2e-numpy') - else: - pass - if '-c' in sys.argv[1:]: run_compile() else: diff --git a/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py b/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py index 1c47bee02b..2f8a8dc187 100644 --- a/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/f90mod_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build F90 module support for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/02/03 19:30:23 $ -Pearu Peterson - """ __version__ = "$Revision: 1.27 $"[10:-1] @@ -99,6 +94,8 @@ def buildhooks(pymod): def dadd(line, s=doc): s[0] = '%s\n%s' % (s[0], line) + + usenames = getuseblocks(pymod) for m in findf90modules(pymod): sargs, fargs, efargs, modobjs, notvars, onlyvars = [], [], [], [], [ m['name']], [] @@ -115,6 +112,9 @@ def buildhooks(pymod): mfargs.append(n) outmess('\t\tConstructing F90 module support for "%s"...\n' % (m['name'])) + if m['name'] in usenames and not onlyvars: + outmess(f"\t\t\tSkipping {m['name']} since it is in 'use'...\n") + continue if onlyvars: outmess('\t\t Variables: %s\n' % (' '.join(onlyvars))) chooks = [''] diff --git a/contrib/python/numpy/py3/numpy/f2py/func2subr.py b/contrib/python/numpy/py3/numpy/f2py/func2subr.py index 2eedc0ade8..b9aa9fc007 100644 --- a/contrib/python/numpy/py3/numpy/f2py/func2subr.py +++ b/contrib/python/numpy/py3/numpy/f2py/func2subr.py @@ -1,17 +1,13 @@ -#!/usr/bin/env python3 """ Rules for building C/API module with f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2004/11/26 11:13:06 $ -Pearu Peterson - """ import copy diff --git a/contrib/python/numpy/py3/numpy/f2py/rules.py b/contrib/python/numpy/py3/numpy/f2py/rules.py index 1bac871024..009365e047 100755 --- a/contrib/python/numpy/py3/numpy/f2py/rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/rules.py @@ -40,15 +40,12 @@ wrapper_function(args) return buildvalue -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/08/30 08:58:42 $ -Pearu Peterson - """ import os, sys import time diff --git a/contrib/python/numpy/py3/numpy/f2py/setup.py b/contrib/python/numpy/py3/numpy/f2py/setup.py index 98f1e9aaae..05bef30001 100644 --- a/contrib/python/numpy/py3/numpy/f2py/setup.py +++ b/contrib/python/numpy/py3/numpy/f2py/setup.py @@ -31,7 +31,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h', - 'backends/meson.build.template', + '_backends/meson.build.template', ) config.add_data_files('*.pyi') return config diff --git a/contrib/python/numpy/py3/numpy/f2py/symbolic.py b/contrib/python/numpy/py3/numpy/f2py/symbolic.py index b1b9f5b6a1..67120d79a5 100644 --- a/contrib/python/numpy/py3/numpy/f2py/symbolic.py +++ b/contrib/python/numpy/py3/numpy/f2py/symbolic.py @@ -2,6 +2,13 @@ References: - J3/21-007: Draft Fortran 202x. https://j3-fortran.org/doc/year/21/21-007.pdf + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ # To analyze Fortran expressions to solve dimensions specifications, diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f new file mode 100644 index 0000000000..ba727a10a0 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.f @@ -0,0 +1,10 @@ + SUBROUTINE FOO(FUN,R) + EXTERNAL FUN + INTEGER I + REAL*8 R, FUN +Cf2py intent(out) r + R = 0D0 + DO I=-5,5 + R = R + FUN(I) + ENDDO + END diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf new file mode 100644 index 0000000000..f120111533 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/callback/gh25211.pyf @@ -0,0 +1,18 @@ +python module __user__routines + interface + function fun(i) result (r) + integer :: i + real*8 :: r + end function fun + end interface +end python module __user__routines + +python module callback2 + interface + subroutine foo(f,r) + use __user__routines, f=>fun + external f + real*8 intent(out) :: r + end subroutine foo + end interface +end python module callback2 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf new file mode 100644 index 0000000000..8eb5bb106a --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/cli/gh_22819.pyf @@ -0,0 +1,6 @@ +python module test_22819 + interface + subroutine hello() + end subroutine hello + end interface +end python module test_22819 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 new file mode 100644 index 0000000000..a2f40735ad --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/common/gh19161.f90 @@ -0,0 +1,10 @@ +module typedefmod + use iso_fortran_env, only: real32 +end module typedefmod + +module data + use typedefmod, only: real32 + implicit none + real(kind=real32) :: x + common/test/x +end module data diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 index 42db6cccc1..765f7c1ce6 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/isocintrin/isoCtests.f90 @@ -1,5 +1,5 @@ module coddity - use iso_c_binding, only: c_double, c_int + use iso_c_binding, only: c_double, c_int, c_int64_t implicit none contains subroutine c_add(a, b, c) bind(c, name="c_add") @@ -14,4 +14,21 @@ z = x + 7 end function wat + ! gh-25207 + subroutine c_add_int64(a, b, c) bind(c) + integer(c_int64_t), intent(in) :: a, b + integer(c_int64_t), intent(out) :: c + c = a + b + end subroutine c_add_int64 + ! gh-25207 + subroutine add_arr(A, B, C) + integer(c_int64_t), intent(in) :: A(3) + integer(c_int64_t), intent(in) :: B(3) + integer(c_int64_t), intent(out) :: C(3) + integer :: j + + do j = 1, 3 + C(j) = A(j)+B(j) + end do + end subroutine end module coddity diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 new file mode 100644 index 0000000000..483d13ceb9 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/data.f90 @@ -0,0 +1,8 @@ +module data + real(8) :: shift +contains + subroutine set_shift(in_shift) + real(8), intent(in) :: in_shift + shift = in_shift + end subroutine set_shift +end module data diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 new file mode 100644 index 0000000000..b3fae8b875 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/regression/gh25337/use_data.f90 @@ -0,0 +1,6 @@ +subroutine shift_a(dim_a, a) + use data, only: shift + integer, intent(in) :: dim_a + real(8), intent(inout), dimension(dim_a) :: a + a = a + shift +end subroutine shift_a diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 new file mode 100644 index 0000000000..ca53413cc9 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh24662.f90 @@ -0,0 +1,7 @@ +subroutine string_inout_optional(output) + implicit none + character*(32), optional, intent(inout) :: output + if (present(output)) then + output="output string" + endif +end subroutine diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 new file mode 100644 index 0000000000..db1c7100d2 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.f90 @@ -0,0 +1,14 @@ +subroutine charint(trans, info) + character, intent(in) :: trans + integer, intent(out) :: info + if (trans == 'N') then + info = 1 + else if (trans == 'T') then + info = 2 + else if (trans == 'C') then + info = 3 + else + info = -1 + end if + +end subroutine charint diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf new file mode 100644 index 0000000000..7b9609071b --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(trans=='N'||trans=='T'||trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf new file mode 100644 index 0000000000..e7b10fa921 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/src/string/gh25286_bc.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(*trans=='N'||*trans=='T'||*trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py index 018cea4fd5..5b6c294d33 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_callback.py @@ -228,3 +228,16 @@ class TestGH18335(util.F2PyTest): r = self.module.gh18335(foo) assert r == 123 + 1 + + +class TestGH25211(util.F2PyTest): + sources = [util.getpath("tests", "src", "callback", "gh25211.f"), + util.getpath("tests", "src", "callback", "gh25211.pyf")] + module_name = "callback2" + + def test_gh18335(self): + def bar(x): + return x*x + + res = self.module.foo(bar) + assert res == 110 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py index 373262bf96..e55b1b6b23 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_character.py @@ -595,3 +595,42 @@ class TestStringAssumedLength(util.F2PyTest): def test_gh24008(self): self.module.greet("joe", "bob") + +class TestStringOptionalInOut(util.F2PyTest): + sources = [util.getpath("tests", "src", "string", "gh24662.f90")] + + def test_gh24662(self): + self.module.string_inout_optional() + a = np.array('hi', dtype='S32') + self.module.string_inout_optional(a) + assert "output string" in a.tobytes().decode() + with pytest.raises(Exception): + aa = "Hi" + self.module.string_inout_optional(aa) + + +@pytest.mark.slow +class TestNewCharHandling(util.F2PyTest): + # from v1.24 onwards, gh-19388 + sources = [ + util.getpath("tests", "src", "string", "gh25286.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 + +@pytest.mark.slow +class TestBCCharHandling(util.F2PyTest): + # SciPy style, "incorrect" bindings with a hook + sources = [ + util.getpath("tests", "src", "string", "gh25286_bc.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py index 8a4b221ef8..68c1b3b31c 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_common.py @@ -16,3 +16,12 @@ class TestCommonBlock(util.F2PyTest): assert self.module.block.long_bn == np.array(1.0, dtype=np.float64) assert self.module.block.string_bn == np.array("2", dtype="|S1") assert self.module.block.ok == np.array(3, dtype=np.int32) + + +class TestCommonWithUse(util.F2PyTest): + sources = [util.getpath("tests", "src", "common", "gh19161.f90")] + + @pytest.mark.skipif(sys.platform == "win32", + reason="Fails with MinGW64 Gfortran (Issue #9673)") + def test_common_gh19161(self): + assert self.module.data.x == 0 diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py index 5f7b56a68a..659e0e96bd 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_f2py2e.py @@ -1,6 +1,7 @@ import textwrap, re, sys, subprocess, shlex from pathlib import Path from collections import namedtuple +import platform import pytest @@ -72,6 +73,15 @@ def gh23598_warn(tmpdir_factory): @pytest.fixture(scope="session") +def gh22819_cli(tmpdir_factory): + """F90 file for testing disallowed CLI arguments in ghff819""" + fdat = util.getpath("tests", "src", "cli", "gh_22819.pyf").read_text() + fn = tmpdir_factory.getbasetemp() / "gh_22819.pyf" + fn.write_text(fdat, encoding="ascii") + return fn + + +@pytest.fixture(scope="session") def hello_world_f77(tmpdir_factory): """Generates a single f77 file for testing""" fdat = util.getpath("tests", "src", "cli", "hi77.f").read_text() @@ -100,6 +110,38 @@ def f2cmap_f90(tmpdir_factory): return fn +def test_gh22819_cli(capfd, gh22819_cli, monkeypatch): + """Check that module names are handled correctly + gh-22819 + Essentially, the -m name cannot be used to import the module, so the module + named in the .pyf needs to be used instead + + CLI :: -m and a .pyf file + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + gen_paths = [item.name for item in ipath.parent.rglob("*") if item.is_file()] + assert "blahmodule.c" not in gen_paths # shouldn't be generated + assert "blah-f2pywrappers.f" not in gen_paths + assert "test_22819-f2pywrappers.f" in gen_paths + assert "test_22819module.c" in gen_paths + assert "Ignoring blah" + + +def test_gh22819_many_pyf(capfd, gh22819_cli, monkeypatch): + """Only one .pyf file allowed + gh-22819 + CLI :: .pyf files + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath} hello.pyf".split()) + with util.switchdir(ipath.parent): + with pytest.raises(ValueError, match="Only one .pyf file per call"): + f2pycli() + + def test_gh23598_warn(capfd, gh23598_warn, monkeypatch): foutl = get_io_paths(gh23598_warn, mname="test") ipath = foutl.f90inp @@ -156,6 +198,53 @@ def test_gen_pyf_no_overwrite(capfd, hello_world_f90, monkeypatch): assert "Use --overwrite-signature to overwrite" in err +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), + reason='Compiler and 3.12 required') +def test_untitled_cli(capfd, hello_world_f90, monkeypatch): + """Check that modules are named correctly + + CLI :: defaults + """ + ipath = Path(hello_world_f90) + monkeypatch.setattr(sys, "argv", f"f2py --backend meson -c {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "untitledmodule.c" in out + + +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), reason='Compiler and 3.12 required') +def test_no_py312_distutils_fcompiler(capfd, hello_world_f90, monkeypatch): + """Check that no distutils imports are performed on 3.12 + CLI :: --fcompiler --help-link --backend distutils + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f90, mname=MNAME) + ipath = foutl.f90inp + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c --fcompiler=gfortran -m {MNAME}".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "--fcompiler cannot be used with meson" in out + monkeypatch.setattr( + sys, "argv", f"f2py --help-link".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Use --dep for meson builds" in out + MNAME = "hi2" # Needs to be different for a new -c + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c -m {MNAME} --backend distutils".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Cannot use distutils backend with Python>=3.12" in out + + @pytest.mark.xfail def test_f2py_skip(capfd, retreal_f77, monkeypatch): """Tests that functions can be skipped @@ -250,6 +339,22 @@ def test_mod_gen_f77(capfd, hello_world_f90, monkeypatch): assert Path.exists(foutl.wrap77) +def test_mod_gen_gh25263(capfd, hello_world_f77, monkeypatch): + """Check that pyf files are correctly generated with module structure + CLI :: -m <name> -h pyf_file + BUG: numpy-gh #20520 + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f77, mname=MNAME) + ipath = foutl.finp + monkeypatch.setattr(sys, "argv", f'f2py {ipath} -m {MNAME} -h hi.pyf'.split()) + with util.switchdir(ipath.parent): + f2pycli() + with Path('hi.pyf').open() as hipyf: + pyfdat = hipyf.read() + assert "python module hi" in pyfdat + + def test_lower_cmod(capfd, hello_world_f77, monkeypatch): """Lowers cases by flag or when -h is present diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py index 7e189bd7b8..594bd7caea 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_isoc.py @@ -1,5 +1,7 @@ from . import util import numpy as np +import pytest +from numpy.testing import assert_allclose class TestISOC(util.F2PyTest): sources = [ @@ -17,3 +19,34 @@ class TestISOC(util.F2PyTest): out = self.module.coddity.wat(1, 20) exp_out = 8 assert out == exp_out + + # gh-25207 + def test_bindc_kinds(self): + out = self.module.coddity.c_add_int64(1, 20) + exp_out = 21 + assert out == exp_out + + # gh-25207 + def test_bindc_add_arr(self): + a = np.array([1,2,3]) + b = np.array([1,2,3]) + out = self.module.coddity.add_arr(a, b) + exp_out = a*2 + assert_allclose(out, exp_out) + + +def test_process_f2cmap_dict(): + from numpy.f2py.auxfuncs import process_f2cmap_dict + + f2cmap_all = {"integer": {"8": "rubbish_type"}} + new_map = {"INTEGER": {"4": "int"}} + c2py_map = {"int": "int", "rubbish_type": "long"} + + exp_map, exp_maptyp = ({"integer": {"8": "rubbish_type", "4": "int"}}, ["int"]) + + # Call the function + res_map, res_maptyp = process_f2cmap_dict(f2cmap_all, new_map, c2py_map) + + # Assert the result is as expected + assert res_map == exp_map + assert res_maptyp == exp_maptyp diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py new file mode 100644 index 0000000000..f77ded2f31 --- /dev/null +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_pyf_src.py @@ -0,0 +1,44 @@ +# This test is ported from numpy.distutils +from numpy.f2py._src_pyf import process_str +from numpy.testing import assert_equal + + +pyf_src = """ +python module foo + <_rd=real,double precision> + interface + subroutine <s,d>foosub(tol) + <_rd>, intent(in,out) :: tol + end subroutine <s,d>foosub + end interface +end python module foo +""" + +expected_pyf = """ +python module foo + interface + subroutine sfoosub(tol) + real, intent(in,out) :: tol + end subroutine sfoosub + subroutine dfoosub(tol) + double precision, intent(in,out) :: tol + end subroutine dfoosub + end interface +end python module foo +""" + + +def normalize_whitespace(s): + """ + Remove leading and trailing whitespace, and convert internal + stretches of whitespace to a single space. + """ + return ' '.join(s.split()) + + +def test_from_template(): + """Regression test for gh-10712.""" + pyf = process_str(pyf_src) + normalized_pyf = normalize_whitespace(pyf) + normalized_expected_pyf = normalize_whitespace(expected_pyf) + assert_equal(normalized_pyf, normalized_expected_pyf) diff --git a/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py b/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py index 044f952f22..1c10978309 100644 --- a/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py +++ b/contrib/python/numpy/py3/numpy/f2py/tests/test_regression.py @@ -64,3 +64,14 @@ def test_include_path(): fnames_in_dir = os.listdir(incdir) for fname in ("fortranobject.c", "fortranobject.h"): assert fname in fnames_in_dir + + +class TestModuleAndSubroutine(util.F2PyTest): + module_name = "example" + sources = [util.getpath("tests", "src", "regression", "gh25337", "data.f90"), + util.getpath("tests", "src", "regression", "gh25337", "use_data.f90")] + + @pytest.mark.slow + def test_gh25337(self): + self.module.data.set_shift(3) + assert "data" in dir(self.module) diff --git a/contrib/python/numpy/py3/numpy/f2py/use_rules.py b/contrib/python/numpy/py3/numpy/f2py/use_rules.py index f1b71e83c2..808b3dd97e 100644 --- a/contrib/python/numpy/py3/numpy/f2py/use_rules.py +++ b/contrib/python/numpy/py3/numpy/f2py/use_rules.py @@ -1,19 +1,12 @@ -#!/usr/bin/env python3 """ - Build 'use others module data' mechanism for f2py2e. -Unfinished. - -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson <pearu@ioc.ee> +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2000/09/10 12:35:43 $ -Pearu Peterson - """ __version__ = "$Revision: 1.3 $"[10:-1] diff --git a/contrib/python/numpy/py3/numpy/random/_generator.pyx b/contrib/python/numpy/py3/numpy/random/_generator.pyx index 1bdba33565..ba3118d9c7 100644 --- a/contrib/python/numpy/py3/numpy/random/_generator.pyx +++ b/contrib/python/numpy/py3/numpy/random/_generator.pyx @@ -3757,7 +3757,7 @@ cdef class Generator: if size is None: shape = [] - elif isinstance(size, (int, long, np.integer)): + elif isinstance(size, (int, np.integer)): shape = [size] else: shape = size diff --git a/contrib/python/numpy/py3/numpy/random/mtrand.pyx b/contrib/python/numpy/py3/numpy/random/mtrand.pyx index 9ffaa572d8..690dea1485 100644 --- a/contrib/python/numpy/py3/numpy/random/mtrand.pyx +++ b/contrib/python/numpy/py3/numpy/random/mtrand.pyx @@ -367,15 +367,16 @@ cdef class RandomState: else: if not isinstance(state, (tuple, list)): raise TypeError('state must be a dict or a tuple.') - if state[0] != 'MT19937': - raise ValueError('set_state can only be used with legacy MT19937' - 'state instances.') - st = {'bit_generator': state[0], - 'state': {'key': state[1], 'pos': state[2]}} - if len(state) > 3: - st['has_gauss'] = state[3] - st['gauss'] = state[4] - value = st + with cython.boundscheck(True): + if state[0] != 'MT19937': + raise ValueError('set_state can only be used with legacy ' + 'MT19937 state instances.') + st = {'bit_generator': state[0], + 'state': {'key': state[1], 'pos': state[2]}} + if len(state) > 3: + st['has_gauss'] = state[3] + st['gauss'] = state[4] + value = st self._aug_state.gauss = st.get('gauss', 0.0) self._aug_state.has_gauss = st.get('has_gauss', 0) diff --git a/contrib/python/numpy/py3/numpy/random/tests/test_random.py b/contrib/python/numpy/py3/numpy/random/tests/test_random.py index e64ace7119..3d081fe1db 100644 --- a/contrib/python/numpy/py3/numpy/random/tests/test_random.py +++ b/contrib/python/numpy/py3/numpy/random/tests/test_random.py @@ -147,6 +147,11 @@ class TestSetState: # arguments without truncation. self.prng.negative_binomial(0.5, 0.5) + def test_set_invalid_state(self): + # gh-25402 + with pytest.raises(IndexError): + self.prng.set_state(()) + class TestRandint: diff --git a/contrib/python/numpy/py3/numpy/tests/test_warnings.py b/contrib/python/numpy/py3/numpy/tests/test_warnings.py index d7a6d880cb..ee5124c5d5 100644 --- a/contrib/python/numpy/py3/numpy/tests/test_warnings.py +++ b/contrib/python/numpy/py3/numpy/tests/test_warnings.py @@ -5,6 +5,7 @@ all of these occurrences but should catch almost all. import pytest from pathlib import Path +import sys import ast import tokenize import numpy @@ -56,6 +57,8 @@ class FindFuncs(ast.NodeVisitor): @pytest.mark.slow +@pytest.mark.skipif(sys.version_info >= (3, 12), + reason="Deprecation warning in ast") def test_warning_calls(): # combined "ignore" and stacklevel error base = Path(numpy.__file__).parent diff --git a/contrib/python/numpy/py3/numpy/version.py b/contrib/python/numpy/py3/numpy/version.py index 54fc637f26..692240a486 100644 --- a/contrib/python/numpy/py3/numpy/version.py +++ b/contrib/python/numpy/py3/numpy/version.py @@ -1,5 +1,5 @@ -version = "1.26.2" +version = "1.26.3" __version__ = version full_version = version diff --git a/contrib/python/numpy/py3/ya.make b/contrib/python/numpy/py3/ya.make index cc5ca1691b..92042220c3 100644 --- a/contrib/python/numpy/py3/ya.make +++ b/contrib/python/numpy/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() PROVIDES(numpy) -VERSION(1.26.2) +VERSION(1.26.3) LICENSE(BSD-3-Clause) @@ -441,6 +441,7 @@ PY_SRCS( numpy/f2py/_backends/_distutils.py numpy/f2py/_backends/_meson.py numpy/f2py/_isocbind.py + numpy/f2py/_src_pyf.py numpy/f2py/auxfuncs.py numpy/f2py/capi_maps.py numpy/f2py/cb_rules.py diff --git a/contrib/python/pytest/py3/.dist-info/METADATA b/contrib/python/pytest/py3/.dist-info/METADATA index 8e45f6d805..20b02c2453 100644 --- a/contrib/python/pytest/py3/.dist-info/METADATA +++ b/contrib/python/pytest/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pytest -Version: 7.4.2 +Version: 7.4.4 Summary: pytest: simple powerful testing with Python Home-page: https://docs.pytest.org/en/latest/ Author: Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others diff --git a/contrib/python/pytest/py3/AUTHORS b/contrib/python/pytest/py3/AUTHORS index e8456d92b3..950672930f 100644 --- a/contrib/python/pytest/py3/AUTHORS +++ b/contrib/python/pytest/py3/AUTHORS @@ -47,6 +47,7 @@ Ariel Pillemer Armin Rigo Aron Coyle Aron Curzon +Arthur Richard Ashish Kurmi Aviral Verma Aviv Palivoda @@ -231,6 +232,7 @@ Maho Maik Figura Mandeep Bhutani Manuel Krebber +Marc Mueller Marc Schlaich Marcelo Duarte Trevisani Marcin Bachry @@ -322,6 +324,7 @@ Ronny Pfannschmidt Ross Lawley Ruaridh Williamson Russel Winder +Ryan Puddephatt Ryan Wooden Saiprasad Kale Samuel Colvin @@ -336,6 +339,7 @@ Serhii Mozghovyi Seth Junot Shantanu Jain Shubham Adep +Simon Blanchard Simon Gomizelj Simon Holesch Simon Kerr @@ -373,6 +377,7 @@ Tomer Keren Tony Narlock Tor Colvin Trevor Bekolay +Tushar Sadhwani Tyler Goodlet Tyler Smart Tzu-ping Chung diff --git a/contrib/python/pytest/py3/_pytest/_version.py b/contrib/python/pytest/py3/_pytest/_version.py index a7f556f8a6..458d065928 100644 --- a/contrib/python/pytest/py3/_pytest/_version.py +++ b/contrib/python/pytest/py3/_pytest/_version.py @@ -1,4 +1,16 @@ # file generated by setuptools_scm # don't change, don't track in version control -__version__ = version = '7.4.2' -__version_tuple__ = version_tuple = (7, 4, 2) +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '7.4.4' +__version_tuple__ = version_tuple = (7, 4, 4) diff --git a/contrib/python/pytest/py3/_pytest/assertion/rewrite.py b/contrib/python/pytest/py3/_pytest/assertion/rewrite.py index ab83fee32b..d1974bb3b4 100644 --- a/contrib/python/pytest/py3/_pytest/assertion/rewrite.py +++ b/contrib/python/pytest/py3/_pytest/assertion/rewrite.py @@ -13,6 +13,7 @@ import struct import sys import tokenize import types +from collections import defaultdict from pathlib import Path from pathlib import PurePath from typing import Callable @@ -56,6 +57,10 @@ else: astNum = ast.Num +class Sentinel: + pass + + assertstate_key = StashKey["AssertionState"]() # pytest caches rewritten pycs in pycache dirs @@ -63,6 +68,9 @@ PYTEST_TAG = f"{sys.implementation.cache_tag}-pytest-{version}" PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT +# Special marker that denotes we have just left a scope definition +_SCOPE_END_MARKER = Sentinel() + class AssertionRewritingHook(importlib.abc.MetaPathFinder, importlib.abc.Loader): """PEP302/PEP451 import hook which rewrites asserts.""" @@ -596,6 +604,13 @@ def _get_assertion_exprs(src: bytes) -> Dict[int, str]: return ret +def _get_ast_constant_value(value: astStr) -> object: + if sys.version_info >= (3, 8): + return value.value + else: + return value.s + + class AssertionRewriter(ast.NodeVisitor): """Assertion rewriting implementation. @@ -645,6 +660,8 @@ class AssertionRewriter(ast.NodeVisitor): .push_format_context() and .pop_format_context() which allows to build another %-formatted string while already building one. + :scope: A tuple containing the current scope used for variables_overwrite. + :variables_overwrite: A dict filled with references to variables that change value within an assert. This happens when a variable is reassigned with the walrus operator @@ -666,7 +683,10 @@ class AssertionRewriter(ast.NodeVisitor): else: self.enable_assertion_pass_hook = False self.source = source - self.variables_overwrite: Dict[str, str] = {} + self.scope: tuple[ast.AST, ...] = () + self.variables_overwrite: defaultdict[ + tuple[ast.AST, ...], Dict[str, str] + ] = defaultdict(dict) def run(self, mod: ast.Module) -> None: """Find all assert statements in *mod* and rewrite them.""" @@ -687,11 +707,10 @@ class AssertionRewriter(ast.NodeVisitor): expect_docstring and isinstance(item, ast.Expr) and isinstance(item.value, astStr) + and isinstance(_get_ast_constant_value(item.value), str) ): - if sys.version_info >= (3, 8): - doc = item.value.value - else: - doc = item.value.s + doc = _get_ast_constant_value(item.value) + assert isinstance(doc, str) if self.is_rewrite_disabled(doc): return expect_docstring = False @@ -732,9 +751,17 @@ class AssertionRewriter(ast.NodeVisitor): mod.body[pos:pos] = imports # Collect asserts. - nodes: List[ast.AST] = [mod] + self.scope = (mod,) + nodes: List[Union[ast.AST, Sentinel]] = [mod] while nodes: node = nodes.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + self.scope = tuple((*self.scope, node)) + nodes.append(_SCOPE_END_MARKER) + if node == _SCOPE_END_MARKER: + self.scope = self.scope[:-1] + continue + assert isinstance(node, ast.AST) for name, field in ast.iter_fields(node): if isinstance(field, list): new: List[ast.AST] = [] @@ -1005,7 +1032,7 @@ class AssertionRewriter(ast.NodeVisitor): ] ): pytest_temp = self.variable() - self.variables_overwrite[ + self.variables_overwrite[self.scope][ v.left.target.id ] = v.left # type:ignore[assignment] v.left.target.id = pytest_temp @@ -1048,17 +1075,20 @@ class AssertionRewriter(ast.NodeVisitor): new_args = [] new_kwargs = [] for arg in call.args: - if isinstance(arg, ast.Name) and arg.id in self.variables_overwrite: - arg = self.variables_overwrite[arg.id] # type:ignore[assignment] + if isinstance(arg, ast.Name) and arg.id in self.variables_overwrite.get( + self.scope, {} + ): + arg = self.variables_overwrite[self.scope][ + arg.id + ] # type:ignore[assignment] res, expl = self.visit(arg) arg_expls.append(expl) new_args.append(res) for keyword in call.keywords: - if ( - isinstance(keyword.value, ast.Name) - and keyword.value.id in self.variables_overwrite - ): - keyword.value = self.variables_overwrite[ + if isinstance( + keyword.value, ast.Name + ) and keyword.value.id in self.variables_overwrite.get(self.scope, {}): + keyword.value = self.variables_overwrite[self.scope][ keyword.value.id ] # type:ignore[assignment] res, expl = self.visit(keyword.value) @@ -1094,12 +1124,14 @@ class AssertionRewriter(ast.NodeVisitor): def visit_Compare(self, comp: ast.Compare) -> Tuple[ast.expr, str]: self.push_format_context() # We first check if we have overwritten a variable in the previous assert - if isinstance(comp.left, ast.Name) and comp.left.id in self.variables_overwrite: - comp.left = self.variables_overwrite[ + if isinstance( + comp.left, ast.Name + ) and comp.left.id in self.variables_overwrite.get(self.scope, {}): + comp.left = self.variables_overwrite[self.scope][ comp.left.id ] # type:ignore[assignment] if isinstance(comp.left, namedExpr): - self.variables_overwrite[ + self.variables_overwrite[self.scope][ comp.left.target.id ] = comp.left # type:ignore[assignment] left_res, left_expl = self.visit(comp.left) @@ -1119,7 +1151,7 @@ class AssertionRewriter(ast.NodeVisitor): and next_operand.target.id == left_res.id ): next_operand.target.id = self.variable() - self.variables_overwrite[ + self.variables_overwrite[self.scope][ left_res.id ] = next_operand # type:ignore[assignment] next_res, next_expl = self.visit(next_operand) diff --git a/contrib/python/pytest/py3/_pytest/assertion/util.py b/contrib/python/pytest/py3/_pytest/assertion/util.py index fc5dfdbd5b..39ca5403e0 100644 --- a/contrib/python/pytest/py3/_pytest/assertion/util.py +++ b/contrib/python/pytest/py3/_pytest/assertion/util.py @@ -132,7 +132,7 @@ def isiterable(obj: Any) -> bool: try: iter(obj) return not istext(obj) - except TypeError: + except Exception: return False diff --git a/contrib/python/pytest/py3/_pytest/compat.py b/contrib/python/pytest/py3/_pytest/compat.py index a1f9d37722..6fdb5a1d8e 100644 --- a/contrib/python/pytest/py3/_pytest/compat.py +++ b/contrib/python/pytest/py3/_pytest/compat.py @@ -380,15 +380,24 @@ else: def get_user_id() -> int | None: - """Return the current user id, or None if we cannot get it reliably on the current platform.""" - # win32 does not have a getuid() function. - # On Emscripten, getuid() is a stub that always returns 0. - if sys.platform in ("win32", "emscripten"): + """Return the current process's real user id or None if it could not be + determined. + + :return: The user id or None if it could not be determined. + """ + # mypy follows the version and platform checking expectation of PEP 484: + # https://mypy.readthedocs.io/en/stable/common_issues.html?highlight=platform#python-version-and-system-platform-checks + # Containment checks are too complex for mypy v1.5.0 and cause failure. + if sys.platform == "win32" or sys.platform == "emscripten": + # win32 does not have a getuid() function. + # Emscripten has a return 0 stub. return None - # getuid shouldn't fail, but cpython defines such a case. - # Let's hope for the best. - uid = os.getuid() - return uid if uid != -1 else None + else: + # On other platforms, a return value of -1 is assumed to indicate that + # the current process's real user id could not be determined. + ERROR = -1 + uid = os.getuid() + return uid if uid != ERROR else None # Perform exhaustiveness checking. diff --git a/contrib/python/pytest/py3/_pytest/faulthandler.py b/contrib/python/pytest/py3/_pytest/faulthandler.py index af879aa44c..d8c7e9fd3b 100644 --- a/contrib/python/pytest/py3/_pytest/faulthandler.py +++ b/contrib/python/pytest/py3/_pytest/faulthandler.py @@ -1,4 +1,3 @@ -import io import os import sys from typing import Generator @@ -10,8 +9,8 @@ from _pytest.nodes import Item from _pytest.stash import StashKey +fault_handler_original_stderr_fd_key = StashKey[int]() fault_handler_stderr_fd_key = StashKey[int]() -fault_handler_originally_enabled_key = StashKey[bool]() def pytest_addoption(parser: Parser) -> None: @@ -25,8 +24,15 @@ def pytest_addoption(parser: Parser) -> None: def pytest_configure(config: Config) -> None: import faulthandler - config.stash[fault_handler_stderr_fd_key] = os.dup(get_stderr_fileno()) - config.stash[fault_handler_originally_enabled_key] = faulthandler.is_enabled() + # at teardown we want to restore the original faulthandler fileno + # but faulthandler has no api to return the original fileno + # so here we stash the stderr fileno to be used at teardown + # sys.stderr and sys.__stderr__ may be closed or patched during the session + # so we can't rely on their values being good at that point (#11572). + stderr_fileno = get_stderr_fileno() + if faulthandler.is_enabled(): + config.stash[fault_handler_original_stderr_fd_key] = stderr_fileno + config.stash[fault_handler_stderr_fd_key] = os.dup(stderr_fileno) faulthandler.enable(file=config.stash[fault_handler_stderr_fd_key]) @@ -38,9 +44,10 @@ def pytest_unconfigure(config: Config) -> None: if fault_handler_stderr_fd_key in config.stash: os.close(config.stash[fault_handler_stderr_fd_key]) del config.stash[fault_handler_stderr_fd_key] - if config.stash.get(fault_handler_originally_enabled_key, False): - # Re-enable the faulthandler if it was originally enabled. - faulthandler.enable(file=get_stderr_fileno()) + # Re-enable the faulthandler if it was originally enabled. + if fault_handler_original_stderr_fd_key in config.stash: + faulthandler.enable(config.stash[fault_handler_original_stderr_fd_key]) + del config.stash[fault_handler_original_stderr_fd_key] def get_stderr_fileno() -> int: @@ -51,7 +58,7 @@ def get_stderr_fileno() -> int: if fileno == -1: raise AttributeError() return fileno - except (AttributeError, io.UnsupportedOperation): + except (AttributeError, ValueError): # pytest-xdist monkeypatches sys.stderr with an object that is not an actual file. # https://docs.python.org/3/library/faulthandler.html#issue-with-file-descriptors # This is potentially dangerous, but the best we can do. diff --git a/contrib/python/pytest/py3/_pytest/junitxml.py b/contrib/python/pytest/py3/_pytest/junitxml.py index ed259e4c41..9ee35b84e8 100644 --- a/contrib/python/pytest/py3/_pytest/junitxml.py +++ b/contrib/python/pytest/py3/_pytest/junitxml.py @@ -369,7 +369,7 @@ def record_testsuite_property(request: FixtureRequest) -> Callable[[str, object] __tracebackhide__ = True def record_func(name: str, value: object) -> None: - """No-op function in case --junitxml was not passed in the command-line.""" + """No-op function in case --junit-xml was not passed in the command-line.""" __tracebackhide__ = True _check_record_param_type("name", name) diff --git a/contrib/python/pytest/py3/_pytest/mark/structures.py b/contrib/python/pytest/py3/_pytest/mark/structures.py index 42fb294c6d..32bdc7e38b 100644 --- a/contrib/python/pytest/py3/_pytest/mark/structures.py +++ b/contrib/python/pytest/py3/_pytest/mark/structures.py @@ -373,7 +373,9 @@ def get_unpacked_marks( if not consider_mro: mark_lists = [obj.__dict__.get("pytestmark", [])] else: - mark_lists = [x.__dict__.get("pytestmark", []) for x in obj.__mro__] + mark_lists = [ + x.__dict__.get("pytestmark", []) for x in reversed(obj.__mro__) + ] mark_list = [] for item in mark_lists: if isinstance(item, list): diff --git a/contrib/python/pytest/py3/_pytest/nodes.py b/contrib/python/pytest/py3/_pytest/nodes.py index 667a02b77a..a5313cb765 100644 --- a/contrib/python/pytest/py3/_pytest/nodes.py +++ b/contrib/python/pytest/py3/_pytest/nodes.py @@ -567,7 +567,7 @@ class Collector(Node): ntraceback = traceback.cut(path=self.path) if ntraceback == traceback: ntraceback = ntraceback.cut(excludepath=tracebackcutdir) - return excinfo.traceback.filter(excinfo) + return ntraceback.filter(excinfo) return excinfo.traceback diff --git a/contrib/python/pytest/py3/_pytest/outcomes.py b/contrib/python/pytest/py3/_pytest/outcomes.py index 1be97dda4e..53c3e1511c 100644 --- a/contrib/python/pytest/py3/_pytest/outcomes.py +++ b/contrib/python/pytest/py3/_pytest/outcomes.py @@ -123,7 +123,7 @@ def exit( only because `msg` is deprecated. :param returncode: - Return code to be used when exiting pytest. + Return code to be used when exiting pytest. None means the same as ``0`` (no error), same as :func:`sys.exit`. :param msg: Same as ``reason``, but deprecated. Will be removed in a future version, use ``reason`` instead. diff --git a/contrib/python/pytest/py3/_pytest/pathlib.py b/contrib/python/pytest/py3/_pytest/pathlib.py index 5c765c6834..c2f8535f5f 100644 --- a/contrib/python/pytest/py3/_pytest/pathlib.py +++ b/contrib/python/pytest/py3/_pytest/pathlib.py @@ -623,8 +623,9 @@ def module_name_from_path(path: Path, root: Path) -> str: # Use the parts for the relative path to the root path. path_parts = relative_path.parts - # Module name for packages do not contain the __init__ file. - if path_parts[-1] == "__init__": + # Module name for packages do not contain the __init__ file, unless + # the `__init__.py` file is at the root. + if len(path_parts) >= 2 and path_parts[-1] == "__init__": path_parts = path_parts[:-1] return ".".join(path_parts) diff --git a/contrib/python/pytest/py3/_pytest/pytester.py b/contrib/python/pytest/py3/_pytest/pytester.py index cdfc2c04ae..0771065e06 100644 --- a/contrib/python/pytest/py3/_pytest/pytester.py +++ b/contrib/python/pytest/py3/_pytest/pytester.py @@ -1074,7 +1074,7 @@ class Pytester: return self.inline_run(*values) def inline_genitems(self, *args) -> Tuple[List[Item], HookRecorder]: - """Run ``pytest.main(['--collectonly'])`` in-process. + """Run ``pytest.main(['--collect-only'])`` in-process. Runs the :py:func:`pytest.main` function to run all of pytest inside the test process itself like :py:meth:`inline_run`, but returns a diff --git a/contrib/python/pytest/py3/ya.make b/contrib/python/pytest/py3/ya.make index 8bef9919d6..a1ab790ee2 100644 --- a/contrib/python/pytest/py3/ya.make +++ b/contrib/python/pytest/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.4.2) +VERSION(7.4.4) LICENSE(MIT) diff --git a/contrib/python/setuptools/py3/.dist-info/METADATA b/contrib/python/setuptools/py3/.dist-info/METADATA index ebe368510a..a12855b091 100644 --- a/contrib/python/setuptools/py3/.dist-info/METADATA +++ b/contrib/python/setuptools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: setuptools -Version: 69.0.2 +Version: 69.0.3 Summary: Easily download, build, install, upgrade, and uninstall Python packages Home-page: https://github.com/pypa/setuptools Author: Python Packaging Authority @@ -75,7 +75,7 @@ Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'testing' .. image:: https://img.shields.io/pypi/pyversions/setuptools.svg -.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg +.. image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 :alt: tests diff --git a/contrib/python/setuptools/py3/README.rst b/contrib/python/setuptools/py3/README.rst index 699576551d..92c7dc6e87 100644 --- a/contrib/python/setuptools/py3/README.rst +++ b/contrib/python/setuptools/py3/README.rst @@ -3,7 +3,7 @@ .. image:: https://img.shields.io/pypi/pyversions/setuptools.svg -.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg +.. image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 :alt: tests diff --git a/contrib/python/setuptools/py3/setuptools/_normalization.py b/contrib/python/setuptools/py3/setuptools/_normalization.py index eee4fb7746..aa9274f093 100644 --- a/contrib/python/setuptools/py3/setuptools/_normalization.py +++ b/contrib/python/setuptools/py3/setuptools/_normalization.py @@ -12,7 +12,7 @@ _Path = Union[str, Path] # https://packaging.python.org/en/latest/specifications/core-metadata/#name _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I) -_UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9.]+", re.I) +_UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9._-]+", re.I) _NON_ALPHANUMERIC = re.compile(r"[^A-Z0-9]+", re.I) _PEP440_FALLBACK = re.compile(r"^v?(?P<safe>(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)*)", re.I) @@ -35,6 +35,8 @@ def safe_name(component: str) -> str: 'hello-world' >>> safe_name("hello?world") 'hello-world' + >>> safe_name("hello_world") + 'hello_world' """ # See pkg_resources.safe_name return _UNSAFE_NAME_CHARS.sub("-", component) diff --git a/contrib/python/setuptools/py3/ya.make b/contrib/python/setuptools/py3/ya.make index 87de186610..2cb2a8b26c 100644 --- a/contrib/python/setuptools/py3/ya.make +++ b/contrib/python/setuptools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(69.0.2) +VERSION(69.0.3) LICENSE(MIT) diff --git a/contrib/python/types-protobuf/.dist-info/METADATA b/contrib/python/types-protobuf/.dist-info/METADATA index 1eaca69326..a54cbbdacd 100644 --- a/contrib/python/types-protobuf/.dist-info/METADATA +++ b/contrib/python/types-protobuf/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: types-protobuf -Version: 3.20.4.1 +Version: 4.24.0.4 Summary: Typing stubs for protobuf Home-page: https://github.com/python/typeshed License: Apache-2.0 license @@ -11,17 +11,33 @@ Project-URL: Chat, https://gitter.im/python/typing Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 3 Classifier: Typing :: Stubs Only +Requires-Python: >=3.7 Description-Content-Type: text/markdown ## Typing stubs for protobuf -This is a PEP 561 type stub package for the `protobuf` package. -It can be used by type-checking tools like mypy, PyCharm, pytype etc. to check code -that uses `protobuf`. The source for this package can be found at -https://github.com/python/typeshed/tree/master/stubs/protobuf. All fixes for +This is a [PEP 561](https://peps.python.org/pep-0561/) +type stub package for the [`protobuf`](https://github.com/protocolbuffers/protobuf) package. +It can be used by type-checking tools like +[mypy](https://github.com/python/mypy/), +[pyright](https://github.com/microsoft/pyright), +[pytype](https://github.com/google/pytype/), +PyCharm, etc. to check code that uses +`protobuf`. + +This version of `types-protobuf` aims to provide accurate annotations +for `protobuf==4.24.*`. +The source for this package can be found at +https://github.com/python/typeshed/tree/main/stubs/protobuf. All fixes for types and metadata should be contributed there. -Generated with aid from mypy-protobuf v3.4.0 +Generated using [mypy-protobuf==3.5.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.5.0) on protobuf==4.21.8 + +This stub package is marked as [partial](https://peps.python.org/pep-0561/#partial-stub-packages). +If you find that annotations are missing, feel free to contribute and help complete them. + -See https://github.com/python/typeshed/blob/master/README.md for more details. -This package was generated from typeshed commit `f43ee406f155b563856551fb7215cb03fdef8e2d`. +See https://github.com/python/typeshed/blob/main/README.md for more details. +This package was generated from typeshed commit `9d345b4df42939b697a84ee461a8760eb674050e` and was tested +with mypy 1.6.1, pyright 1.1.332, and +pytype 2023.10.17. diff --git a/contrib/python/types-protobuf/google-stubs/METADATA.toml b/contrib/python/types-protobuf/google-stubs/METADATA.toml index 06628e4dcc..7ee1107f03 100644 --- a/contrib/python/types-protobuf/google-stubs/METADATA.toml +++ b/contrib/python/types-protobuf/google-stubs/METADATA.toml @@ -1,2 +1,7 @@ -version = "3.20.*" -extra_description = "Generated with aid from mypy-protobuf v3.4.0" +version = "4.24.*" +upstream_repository = "https://github.com/protocolbuffers/protobuf" +extra_description = "Generated using [mypy-protobuf==3.5.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.5.0) on protobuf==4.21.8" +partial_stub = true + +[tool.stubtest] +ignore_missing_stub = true diff --git a/contrib/python/types-protobuf/ya.make b/contrib/python/types-protobuf/ya.make index 5e5ba3f5c1..4fb5c85b88 100644 --- a/contrib/python/types-protobuf/ya.make +++ b/contrib/python/types-protobuf/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.20.4.1) +VERSION(4.24.0.4) LICENSE(Apache-2.0) @@ -10,7 +10,6 @@ NO_LINT() PY_SRCS( TOP_LEVEL - google-stubs/__init__.pyi google-stubs/protobuf/__init__.pyi google-stubs/protobuf/any_pb2.pyi google-stubs/protobuf/api_pb2.pyi @@ -24,6 +23,7 @@ PY_SRCS( google-stubs/protobuf/field_mask_pb2.pyi google-stubs/protobuf/internal/__init__.pyi google-stubs/protobuf/internal/api_implementation.pyi + google-stubs/protobuf/internal/builder.pyi google-stubs/protobuf/internal/containers.pyi google-stubs/protobuf/internal/decoder.pyi google-stubs/protobuf/internal/encoder.pyi |