diff options
Diffstat (limited to 'contrib')
79 files changed, 3568 insertions, 1174 deletions
diff --git a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym index 04bc79b0c7..d437597c8d 100644 --- a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym +++ b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym @@ -1,6 +1,6 @@ {% extends '//builtin/bag.ym' %} -{% block current_version %}20.1.0{% endblock %} +{% block current_version %}20.1.2{% endblock %} {% block current_url %} https://github.com/llvm/llvm-project/releases/download/llvmorg-{{self.version().strip()}}/compiler-rt-{{self.version().strip()}}.src.tar.xz diff --git a/contrib/libs/cxxsupp/builtins/ya.make b/contrib/libs/cxxsupp/builtins/ya.make index dce2f27201..c6c546c341 100644 --- a/contrib/libs/cxxsupp/builtins/ya.make +++ b/contrib/libs/cxxsupp/builtins/ya.make @@ -12,9 +12,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.0) +VERSION(20.1.2) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.0/compiler-rt-20.1.0.src.tar.xz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.2/compiler-rt-20.1.2.src.tar.xz) NO_COMPILER_WARNINGS() diff --git a/contrib/libs/libfuzzer/.yandex_meta/override.nix b/contrib/libs/libfuzzer/.yandex_meta/override.nix index 7415420b65..15a7622b27 100644 --- a/contrib/libs/libfuzzer/.yandex_meta/override.nix +++ b/contrib/libs/libfuzzer/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "20.1.0"; + version = "20.1.2"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-86Z8e4ubnHJc1cYHjYPLeQC9eoPF417HYtqg8NAzxts="; + hash = "sha256-t30Jh8ckp5qD6XDxtvnSaYiAWbEi6L6hAWh6tN8JjtY="; }; sourceRoot = "source/compiler-rt"; diff --git a/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make b/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make index 0315d60cc6..2cf1cb720d 100644 --- a/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make +++ b/contrib/libs/libfuzzer/lib/fuzzer/afl/ya.make @@ -8,7 +8,7 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:cpp-contrib) -VERSION(20.1.0) +VERSION(20.1.2) PEERDIR( contrib/libs/afl/llvm_mode diff --git a/contrib/libs/libfuzzer/ya.make b/contrib/libs/libfuzzer/ya.make index 938438e570..787eeaea14 100644 --- a/contrib/libs/libfuzzer/ya.make +++ b/contrib/libs/libfuzzer/ya.make @@ -12,9 +12,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.0) +VERSION(20.1.2) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.2.tar.gz) SET(SANITIZER_CFLAGS) diff --git a/contrib/libs/libunwind/.yandex_meta/override.nix b/contrib/libs/libunwind/.yandex_meta/override.nix index 29e81dd677..2666c93f5f 100644 --- a/contrib/libs/libunwind/.yandex_meta/override.nix +++ b/contrib/libs/libunwind/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "20.1.0"; + version = "20.1.2"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-86Z8e4ubnHJc1cYHjYPLeQC9eoPF417HYtqg8NAzxts="; + hash = "sha256-t30Jh8ckp5qD6XDxtvnSaYiAWbEi6L6hAWh6tN8JjtY="; }; patches = []; diff --git a/contrib/libs/libunwind/ya.make b/contrib/libs/libunwind/ya.make index b04a57ec9e..c2478a4035 100644 --- a/contrib/libs/libunwind/ya.make +++ b/contrib/libs/libunwind/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.0) +VERSION(20.1.2) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.2.tar.gz) PEERDIR( library/cpp/sanitizer/include diff --git a/contrib/python/argcomplete/py3/.dist-info/METADATA b/contrib/python/argcomplete/py3/.dist-info/METADATA index bf74fb4961..8eff29ade2 100644 --- a/contrib/python/argcomplete/py3/.dist-info/METADATA +++ b/contrib/python/argcomplete/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: argcomplete -Version: 3.6.0 +Version: 3.6.1 Summary: Bash tab completion for argparse Project-URL: Homepage, https://github.com/kislyuk/argcomplete Project-URL: Documentation, https://kislyuk.github.io/argcomplete diff --git a/contrib/python/argcomplete/py3/argcomplete/finders.py b/contrib/python/argcomplete/py3/argcomplete/finders.py index 793b462eed..8d248fd973 100644 --- a/contrib/python/argcomplete/py3/argcomplete/finders.py +++ b/contrib/python/argcomplete/py3/argcomplete/finders.py @@ -515,7 +515,7 @@ class CompletionFinder(object): # Bash mangles completions which contain characters in COMP_WORDBREAKS. # This workaround has the same effect as __ltrim_colon_completions in bash_completion # (extended to characters other than the colon). - if last_wordbreak_pos: + if last_wordbreak_pos is not None: completions = [c[last_wordbreak_pos + 1 :] for c in completions] special_chars += "();<>|&!`$* \t\n\"'" elif cword_prequote == '"': diff --git a/contrib/python/argcomplete/py3/argcomplete/packages/_shlex.py b/contrib/python/argcomplete/py3/argcomplete/packages/_shlex.py index ecd785b80b..890a38f43f 100644 --- a/contrib/python/argcomplete/py3/argcomplete/packages/_shlex.py +++ b/contrib/python/argcomplete/py3/argcomplete/packages/_shlex.py @@ -177,6 +177,9 @@ class shlex: elif self.whitespace_split: self.token = nextchar self.state = 'a' + # Modified by argcomplete: Record last wordbreak position + if nextchar in self.wordbreaks: + self.last_wordbreak_pos = len(self.token) - 1 else: self.token = nextchar if self.token or (self.posix and quoted): diff --git a/contrib/python/argcomplete/py3/argcomplete/scripts/activate_global_python_argcomplete.py b/contrib/python/argcomplete/py3/argcomplete/scripts/activate_global_python_argcomplete.py index 768b8aa6bf..299d081c0e 100644 --- a/contrib/python/argcomplete/py3/argcomplete/scripts/activate_global_python_argcomplete.py +++ b/contrib/python/argcomplete/py3/argcomplete/scripts/activate_global_python_argcomplete.py @@ -121,16 +121,33 @@ def append_to_config_file(path, shellcode): fh.write(shellcode) print("Added.", file=sys.stderr) - -def link_user_rcfiles(): - # TODO: warn if running as superuser +def link_zsh_user_rcfile(zsh_fpath=None): zsh_rcfile = os.path.join(os.path.expanduser(os.environ.get("ZDOTDIR", "~")), ".zshenv") - append_to_config_file(zsh_rcfile, zsh_shellcode.format(zsh_fpath=get_activator_dir())) + append_to_config_file(zsh_rcfile, zsh_shellcode.format(zsh_fpath=zsh_fpath or get_activator_dir())) +def link_bash_user_rcfile(): bash_completion_user_file = os.path.expanduser("~/.bash_completion") append_to_config_file(bash_completion_user_file, bash_shellcode.format(activator=get_activator_path())) +def link_user_rcfiles(): + # TODO: warn if running as superuser + link_zsh_user_rcfile() + link_bash_user_rcfile() + +def add_zsh_system_dir_to_fpath_for_user(): + if "zsh" not in os.environ.get("SHELL", ""): + return + try: + zsh_system_dir = get_zsh_system_dir() + fpath_output = subprocess.check_output([os.environ["SHELL"], "-c", 'printf "%s\n" "${fpath[@]}"']) + for fpath in fpath_output.decode().splitlines(): + if fpath == zsh_system_dir: + return + link_zsh_user_rcfile(zsh_fpath=zsh_system_dir) + except (FileNotFoundError, subprocess.CalledProcessError): + pass + def main(): global args args = parser.parse_args() @@ -160,6 +177,8 @@ def main(): for destination in destinations: install_to_destination(destination) + add_zsh_system_dir_to_fpath_for_user() + if args.dest is None: print("Please restart your shell or source the installed file to activate it.", file=sys.stderr) diff --git a/contrib/python/argcomplete/py3/argcomplete/shell_integration.py b/contrib/python/argcomplete/py3/argcomplete/shell_integration.py index 37b5603b11..cac48902fa 100644 --- a/contrib/python/argcomplete/py3/argcomplete/shell_integration.py +++ b/contrib/python/argcomplete/py3/argcomplete/shell_integration.py @@ -166,7 +166,8 @@ def shellcode(executables, use_defaults=True, shell="bash", complete_arguments=N executables_list = " ".join(quoted_executables) script = argcomplete_script if script: - function_suffix = "_" + script + # If the script path contain a space, this would generate an invalid function name. + function_suffix = "_" + script.replace(" ", "_SPACE_") else: script = "" function_suffix = "" diff --git a/contrib/python/argcomplete/py3/ya.make b/contrib/python/argcomplete/py3/ya.make index 74c5629658..327bc4e34e 100644 --- a/contrib/python/argcomplete/py3/ya.make +++ b/contrib/python/argcomplete/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.6.0) +VERSION(3.6.1) LICENSE(Apache-2.0) diff --git a/contrib/python/iniconfig/.dist-info/METADATA b/contrib/python/iniconfig/.dist-info/METADATA index 3ea1e01cb0..3a8ef46a3b 100644 --- a/contrib/python/iniconfig/.dist-info/METADATA +++ b/contrib/python/iniconfig/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.4 Name: iniconfig -Version: 2.0.0 +Version: 2.1.0 Summary: brain-dead simple config-ini parsing Project-URL: Homepage, https://github.com/pytest-dev/iniconfig Author-email: Ronny Pfannschmidt <opensource@ronnypfannschmidt.de>, Holger Krekel <holger.krekel@gmail.com> @@ -14,14 +14,15 @@ Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: Utilities -Requires-Python: >=3.7 +Requires-Python: >=3.8 Description-Content-Type: text/x-rst iniconfig: brain-dead simple parsing of ini files diff --git a/contrib/python/iniconfig/LICENSE b/contrib/python/iniconfig/LICENSE index 31ecdfb1db..46f4b2846f 100644 --- a/contrib/python/iniconfig/LICENSE +++ b/contrib/python/iniconfig/LICENSE @@ -1,19 +1,21 @@ +The MIT License (MIT) - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. +Copyright (c) 2010 - 2023 Holger Krekel and others +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/python/iniconfig/iniconfig/__init__.py b/contrib/python/iniconfig/iniconfig/__init__.py index c1c94f70ae..ed6499bc6c 100644 --- a/contrib/python/iniconfig/iniconfig/__init__.py +++ b/contrib/python/iniconfig/iniconfig/__init__.py @@ -20,7 +20,7 @@ from typing import ( import os if TYPE_CHECKING: - from typing_extensions import Final + from typing import Final __all__ = ["IniConfig", "ParseError", "COMMENTCHARS", "iscommentline"] diff --git a/contrib/python/iniconfig/iniconfig/_version.py b/contrib/python/iniconfig/iniconfig/_version.py index dd1883d734..e058e2c657 100644 --- a/contrib/python/iniconfig/iniconfig/_version.py +++ b/contrib/python/iniconfig/iniconfig/_version.py @@ -1,4 +1,21 @@ -# file generated by setuptools_scm +# file generated by setuptools-scm # don't change, don't track in version control -__version__ = version = '2.0.0' -__version_tuple__ = version_tuple = (2, 0, 0) + +__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '2.1.0' +__version_tuple__ = version_tuple = (2, 1, 0) diff --git a/contrib/python/iniconfig/iniconfig/exceptions.py b/contrib/python/iniconfig/iniconfig/exceptions.py index bc898e68ee..8c4dc9a8b0 100644 --- a/contrib/python/iniconfig/iniconfig/exceptions.py +++ b/contrib/python/iniconfig/iniconfig/exceptions.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing_extensions import Final + from typing import Final class ParseError(Exception): diff --git a/contrib/python/iniconfig/ya.make b/contrib/python/iniconfig/ya.make index 0121cca743..20492d75c6 100644 --- a/contrib/python/iniconfig/ya.make +++ b/contrib/python/iniconfig/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.0) +VERSION(2.1.0) LICENSE(MIT) diff --git a/contrib/python/multidict/.dist-info/METADATA b/contrib/python/multidict/.dist-info/METADATA index 93f85177b9..b5c6dad90b 100644 --- a/contrib/python/multidict/.dist-info/METADATA +++ b/contrib/python/multidict/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: multidict -Version: 6.1.0 +Version: 6.2.0 Summary: multidict implementation Home-page: https://github.com/aio-libs/multidict Author: Andrew Svetlov @@ -20,16 +20,15 @@ Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE -Requires-Dist: typing-extensions >=4.1.0 ; python_version < "3.11" +Requires-Dist: typing-extensions>=4.1.0; python_version < "3.11" ========= multidict diff --git a/contrib/python/multidict/multidict/__init__.py b/contrib/python/multidict/multidict/__init__.py index 25ddca41e9..b6b532a1f2 100644 --- a/contrib/python/multidict/multidict/__init__.py +++ b/contrib/python/multidict/multidict/__init__.py @@ -5,6 +5,8 @@ multidict. It behaves mostly like a dict but it can have several values for the same key. """ +from typing import TYPE_CHECKING + from ._abc import MultiMapping, MutableMultiMapping from ._compat import USE_EXTENSIONS @@ -20,13 +22,11 @@ __all__ = ( "getversion", ) -__version__ = "6.1.0" +__version__ = "6.2.0" -try: - if not USE_EXTENSIONS: - raise ImportError - from ._multidict import ( +if TYPE_CHECKING or not USE_EXTENSIONS: + from ._multidict_py import ( CIMultiDict, CIMultiDictProxy, MultiDict, @@ -34,8 +34,8 @@ try: getversion, istr, ) -except ImportError: # pragma: no cover - from ._multidict_py import ( +else: + from ._multidict import ( CIMultiDict, CIMultiDictProxy, MultiDict, diff --git a/contrib/python/multidict/multidict/__init__.pyi b/contrib/python/multidict/multidict/__init__.pyi deleted file mode 100644 index 0940340f81..0000000000 --- a/contrib/python/multidict/multidict/__init__.pyi +++ /dev/null @@ -1,152 +0,0 @@ -import abc -from typing import ( - Generic, - Iterable, - Iterator, - Mapping, - MutableMapping, - TypeVar, - overload, -) - -class istr(str): ... - -upstr = istr - -_S = str | istr - -_T = TypeVar("_T") - -_T_co = TypeVar("_T_co", covariant=True) - -_D = TypeVar("_D") - -class MultiMapping(Mapping[_S, _T_co]): - @overload - @abc.abstractmethod - def getall(self, key: _S) -> list[_T_co]: ... - @overload - @abc.abstractmethod - def getall(self, key: _S, default: _D) -> list[_T_co] | _D: ... - @overload - @abc.abstractmethod - def getone(self, key: _S) -> _T_co: ... - @overload - @abc.abstractmethod - def getone(self, key: _S, default: _D) -> _T_co | _D: ... - -_Arg = ( - Mapping[str, _T] - | Mapping[istr, _T] - | dict[str, _T] - | dict[istr, _T] - | MultiMapping[_T] - | Iterable[tuple[str, _T]] - | Iterable[tuple[istr, _T]] -) - -class MutableMultiMapping(MultiMapping[_T], MutableMapping[_S, _T], Generic[_T]): - @abc.abstractmethod - def add(self, key: _S, value: _T) -> None: ... - @abc.abstractmethod - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - @abc.abstractmethod - def popone(self, key: _S) -> _T: ... - @overload - @abc.abstractmethod - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - @abc.abstractmethod - def popall(self, key: _S) -> list[_T]: ... - @overload - @abc.abstractmethod - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class MultiDict(MutableMultiMapping[_T], Generic[_T]): - def __init__(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - def copy(self) -> MultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __setitem__(self, k: _S, v: _T) -> None: ... - def __delitem__(self, v: _S) -> None: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def add(self, key: _S, value: _T) -> None: ... - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - def popone(self, key: _S) -> _T: ... - @overload - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - def popall(self, key: _S) -> list[_T]: ... - @overload - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class CIMultiDict(MutableMultiMapping[_T], Generic[_T]): - def __init__(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - def copy(self) -> CIMultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __setitem__(self, k: _S, v: _T) -> None: ... - def __delitem__(self, v: _S) -> None: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def add(self, key: _S, value: _T) -> None: ... - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - def popone(self, key: _S) -> _T: ... - @overload - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - def popall(self, key: _S) -> list[_T]: ... - @overload - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class MultiDictProxy(MultiMapping[_T], Generic[_T]): - def __init__(self, arg: MultiMapping[_T] | MutableMultiMapping[_T]) -> None: ... - def copy(self) -> MultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - -class CIMultiDictProxy(MultiMapping[_T], Generic[_T]): - def __init__(self, arg: MultiMapping[_T] | MutableMultiMapping[_T]) -> None: ... - def __getitem__(self, k: _S) -> _T: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def copy(self) -> CIMultiDict[_T]: ... - -def getversion( - md: MultiDict[_T] | CIMultiDict[_T] | MultiDictProxy[_T] | CIMultiDictProxy[_T], -) -> int: ... diff --git a/contrib/python/multidict/multidict/_abc.py b/contrib/python/multidict/multidict/_abc.py index 0603cdd244..ff0e2a6976 100644 --- a/contrib/python/multidict/multidict/_abc.py +++ b/contrib/python/multidict/multidict/_abc.py @@ -1,48 +1,69 @@ import abc -import sys -import types -from collections.abc import Mapping, MutableMapping +from collections.abc import Iterable, Mapping, MutableMapping +from typing import TYPE_CHECKING, Protocol, TypeVar, Union, overload +if TYPE_CHECKING: + from ._multidict_py import istr +else: + istr = str -class _TypingMeta(abc.ABCMeta): - # A fake metaclass to satisfy typing deps in runtime - # basically MultiMapping[str] and other generic-like type instantiations - # are emulated. - # Note: real type hints are provided by __init__.pyi stub file - if sys.version_info >= (3, 9): +_V = TypeVar("_V") +_V_co = TypeVar("_V_co", covariant=True) +_T = TypeVar("_T") - def __getitem__(self, key): - return types.GenericAlias(self, key) - else: +class SupportsKeys(Protocol[_V_co]): + def keys(self) -> Iterable[str]: ... + def __getitem__(self, key: str, /) -> _V_co: ... - def __getitem__(self, key): - return self +class SupportsIKeys(Protocol[_V_co]): + def keys(self) -> Iterable[istr]: ... + def __getitem__(self, key: istr, /) -> _V_co: ... -class MultiMapping(Mapping, metaclass=_TypingMeta): + +MDArg = Union[SupportsKeys[_V], SupportsIKeys[_V], Iterable[tuple[str, _V]], None] + + +class MultiMapping(Mapping[str, _V_co]): + @overload + def getall(self, key: str) -> list[_V_co]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V_co], _T]: ... @abc.abstractmethod - def getall(self, key, default=None): - raise KeyError + def getall(self, key: str, default: _T = ...) -> Union[list[_V_co], _T]: + """Return all values for key.""" + @overload + def getone(self, key: str) -> _V_co: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V_co, _T]: ... @abc.abstractmethod - def getone(self, key, default=None): - raise KeyError + def getone(self, key: str, default: _T = ...) -> Union[_V_co, _T]: + """Return first value for key.""" -class MutableMultiMapping(MultiMapping, MutableMapping): +class MutableMultiMapping(MultiMapping[_V], MutableMapping[str, _V]): @abc.abstractmethod - def add(self, key, value): - raise NotImplementedError + def add(self, key: str, value: _V) -> None: + """Add value to list.""" @abc.abstractmethod - def extend(self, *args, **kwargs): - raise NotImplementedError + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Add everything from arg and kwargs to the mapping.""" + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... @abc.abstractmethod - def popone(self, key, default=None): - raise KeyError + def popone(self, key: str, default: _T = ...) -> Union[_V, _T]: + """Remove specified key and return the corresponding value.""" + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... @abc.abstractmethod - def popall(self, key, default=None): - raise KeyError + def popall(self, key: str, default: _T = ...) -> Union[list[_V], _T]: + """Remove all occurrences of key and return the list of corresponding values.""" diff --git a/contrib/python/multidict/multidict/_multidict.c b/contrib/python/multidict/multidict/_multidict.c index 60864953b1..ebb1949f0a 100644 --- a/contrib/python/multidict/multidict/_multidict.c +++ b/contrib/python/multidict/multidict/_multidict.c @@ -1,6 +1,8 @@ #include "Python.h" #include "structmember.h" +#include "_multilib/pythoncapi_compat.h" + // Include order important #include "_multilib/defs.h" #include "_multilib/istr.h" @@ -9,7 +11,7 @@ #include "_multilib/iter.h" #include "_multilib/views.h" -#if PY_MAJOR_VERSION < 3 || PY_MINOR_VERSION < 12 +#if PY_MINOR_VERSION < 12 #ifndef _PyArg_UnpackKeywords #define FASTCALL_OLD #endif @@ -19,14 +21,13 @@ static PyObject *collections_abc_mapping; static PyObject *collections_abc_mut_mapping; static PyObject *collections_abc_mut_multi_mapping; +static PyObject *repr_func; static PyTypeObject multidict_type; static PyTypeObject cimultidict_type; static PyTypeObject multidict_proxy_type; static PyTypeObject cimultidict_proxy_type; -static PyObject *repr_func; - #define MultiDict_CheckExact(o) (Py_TYPE(o) == &multidict_type) #define CIMultiDict_CheckExact(o) (Py_TYPE(o) == &cimultidict_type) #define MultiDictProxy_CheckExact(o) (Py_TYPE(o) == &multidict_proxy_type) @@ -155,13 +156,17 @@ _multidict_append_items_seq(MultiDictObject *self, PyObject *arg, Py_INCREF(value); } else if (PyList_CheckExact(item)) { - if (PyList_GET_SIZE(item) != 2) { + if (PyList_Size(item) != 2) { + goto invalid_type; + } + key = PyList_GetItemRef(item, 0); + if (key == NULL) { + goto invalid_type; + } + value = PyList_GetItemRef(item, 1); + if (value == NULL) { goto invalid_type; } - key = PyList_GET_ITEM(item, 0); - Py_INCREF(key); - value = PyList_GET_ITEM(item, 1); - Py_INCREF(value); } else if (PySequence_Check(item)) { if (PySequence_Size(item) != 2) { @@ -339,8 +344,8 @@ _multidict_extend(MultiDictObject *self, PyObject *args, PyObject *kwds, if (args && PyObject_Length(args) > 1) { PyErr_Format( PyExc_TypeError, - "%s takes at most 1 positional argument (%zd given)", - name, PyObject_Length(args), NULL + "%s takes from 1 to 2 positional arguments but %zd were given", + name, PyObject_Length(args) + 1, NULL ); return -1; } @@ -769,21 +774,13 @@ static inline void multidict_tp_dealloc(MultiDictObject *self) { PyObject_GC_UnTrack(self); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_TRASHCAN_BEGIN(self, multidict_tp_dealloc) -#else - Py_TRASHCAN_SAFE_BEGIN(self); -#endif if (self->weaklist != NULL) { PyObject_ClearWeakRefs((PyObject *)self); }; pair_list_dealloc(&self->pairs); Py_TYPE(self)->tp_free((PyObject *)self); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_TRASHCAN_END // there should be no code after this -#else - Py_TRASHCAN_SAFE_END(self); -#endif } static inline int @@ -1230,16 +1227,7 @@ PyDoc_STRVAR(multidict_update_doc, "Update the dictionary from *other*, overwriting existing keys."); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 #define multidict_class_getitem Py_GenericAlias -#else -static inline PyObject * -multidict_class_getitem(PyObject *self, PyObject *arg) -{ - Py_INCREF(self); - return self; -} -#endif PyDoc_STRVAR(sizeof__doc__, @@ -1941,9 +1929,7 @@ getversion(PyObject *self, PyObject *md) static inline void module_free(void *m) { -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_CLEAR(multidict_str_lower); -#endif Py_CLEAR(collections_abc_mapping); Py_CLEAR(collections_abc_mut_mapping); Py_CLEAR(collections_abc_mut_multi_mapping); @@ -1972,29 +1958,14 @@ static PyModuleDef multidict_module = { PyMODINIT_FUNC PyInit__multidict(void) { -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 multidict_str_lower = PyUnicode_InternFromString("lower"); if (multidict_str_lower == NULL) { goto fail; } -#endif PyObject *module = NULL, *reg_func_call_result = NULL; -#define WITH_MOD(NAME) \ - Py_CLEAR(module); \ - module = PyImport_ImportModule(NAME); \ - if (module == NULL) { \ - goto fail; \ - } - -#define GET_MOD_ATTR(VAR, NAME) \ - VAR = PyObject_GetAttrString(module, NAME); \ - if (VAR == NULL) { \ - goto fail; \ - } - if (multidict_views_init() < 0) { goto fail; } @@ -2015,18 +1986,31 @@ PyInit__multidict(void) goto fail; } +#define WITH_MOD(NAME) \ + Py_CLEAR(module); \ + module = PyImport_ImportModule(NAME); \ + if (module == NULL) { \ + goto fail; \ + } + +#define GET_MOD_ATTR(VAR, NAME) \ + VAR = PyObject_GetAttrString(module, NAME); \ + if (VAR == NULL) { \ + goto fail; \ + } + WITH_MOD("collections.abc"); GET_MOD_ATTR(collections_abc_mapping, "Mapping"); WITH_MOD("multidict._abc"); GET_MOD_ATTR(collections_abc_mut_mapping, "MultiMapping"); - - WITH_MOD("multidict._abc"); GET_MOD_ATTR(collections_abc_mut_multi_mapping, "MutableMultiMapping"); WITH_MOD("multidict._multidict_base"); GET_MOD_ATTR(repr_func, "_mdrepr"); + Py_CLEAR(module); \ + /* Register in _abc mappings (CI)MultiDict and (CI)MultiDictProxy */ reg_func_call_result = PyObject_CallMethod( collections_abc_mut_mapping, @@ -2070,6 +2054,13 @@ PyInit__multidict(void) /* Instantiate this module */ module = PyModule_Create(&multidict_module); + if (module == NULL) { + goto fail; + } + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif Py_INCREF(&istr_type); if (PyModule_AddObject( @@ -2109,9 +2100,7 @@ PyInit__multidict(void) return module; fail: -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_XDECREF(multidict_str_lower); -#endif Py_XDECREF(collections_abc_mapping); Py_XDECREF(collections_abc_mut_mapping); Py_XDECREF(collections_abc_mut_multi_mapping); diff --git a/contrib/python/multidict/multidict/_multidict_base.py b/contrib/python/multidict/multidict/_multidict_base.py index de2f762a5c..df0d70097a 100644 --- a/contrib/python/multidict/multidict/_multidict_base.py +++ b/contrib/python/multidict/multidict/_multidict_base.py @@ -1,5 +1,19 @@ import sys -from collections.abc import ItemsView, Iterable, KeysView, Set, ValuesView +from collections.abc import ( + Container, + ItemsView, + Iterable, + KeysView, + Mapping, + Set, + ValuesView, +) +from typing import Literal, Union + +if sys.version_info >= (3, 10): + from types import NotImplementedType +else: + from typing import Any as NotImplementedType if sys.version_info >= (3, 11): from typing import assert_never @@ -7,26 +21,28 @@ else: from typing_extensions import assert_never -def _abc_itemsview_register(view_cls): +def _abc_itemsview_register(view_cls: type[object]) -> None: ItemsView.register(view_cls) -def _abc_keysview_register(view_cls): +def _abc_keysview_register(view_cls: type[object]) -> None: KeysView.register(view_cls) -def _abc_valuesview_register(view_cls): +def _abc_valuesview_register(view_cls: type[object]) -> None: ValuesView.register(view_cls) -def _viewbaseset_richcmp(view, other, op): +def _viewbaseset_richcmp( + view: set[object], other: object, op: Literal[0, 1, 2, 3, 4, 5] +) -> Union[bool, NotImplementedType]: if op == 0: # < if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) < len(other) and view <= other elif op == 1: # <= if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if len(view) > len(other): return False for elem in view: @@ -35,17 +51,17 @@ def _viewbaseset_richcmp(view, other, op): return True elif op == 2: # == if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) == len(other) and view <= other elif op == 3: # != return not view == other elif op == 4: # > if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) > len(other) and view >= other elif op == 5: # >= if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if len(view) < len(other): return False for elem in other: @@ -56,9 +72,11 @@ def _viewbaseset_richcmp(view, other, op): assert_never(op) -def _viewbaseset_and(view, other): +def _viewbaseset_and( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -68,9 +86,11 @@ def _viewbaseset_and(view, other): return view & other -def _viewbaseset_or(view, other): +def _viewbaseset_or( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -80,9 +100,11 @@ def _viewbaseset_or(view, other): return view | other -def _viewbaseset_sub(view, other): +def _viewbaseset_sub( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -92,9 +114,11 @@ def _viewbaseset_sub(view, other): return view - other -def _viewbaseset_xor(view, other): +def _viewbaseset_xor( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -104,7 +128,7 @@ def _viewbaseset_xor(view, other): return view ^ other -def _itemsview_isdisjoint(view, other): +def _itemsview_isdisjoint(view: Container[object], other: Iterable[object]) -> bool: "Return True if two sets have a null intersection." for v in other: if v in view: @@ -112,7 +136,7 @@ def _itemsview_isdisjoint(view, other): return True -def _itemsview_repr(view): +def _itemsview_repr(view: Iterable[tuple[object, object]]) -> str: lst = [] for k, v in view: lst.append("{!r}: {!r}".format(k, v)) @@ -120,7 +144,7 @@ def _itemsview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _keysview_isdisjoint(view, other): +def _keysview_isdisjoint(view: Container[object], other: Iterable[object]) -> bool: "Return True if two sets have a null intersection." for k in other: if k in view: @@ -128,7 +152,7 @@ def _keysview_isdisjoint(view, other): return True -def _keysview_repr(view): +def _keysview_repr(view: Iterable[object]) -> str: lst = [] for k in view: lst.append("{!r}".format(k)) @@ -136,7 +160,7 @@ def _keysview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _valuesview_repr(view): +def _valuesview_repr(view: Iterable[object]) -> str: lst = [] for v in view: lst.append("{!r}".format(v)) @@ -144,7 +168,7 @@ def _valuesview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _mdrepr(md): +def _mdrepr(md: Mapping[object, object]) -> str: lst = [] for k, v in md.items(): lst.append("'{}': {!r}".format(k, v)) diff --git a/contrib/python/multidict/multidict/_multidict_py.py b/contrib/python/multidict/multidict/_multidict_py.py index 79c45aa19c..b8ecb8b962 100644 --- a/contrib/python/multidict/multidict/_multidict_py.py +++ b/contrib/python/multidict/multidict/_multidict_py.py @@ -1,47 +1,56 @@ +import enum import sys -import types from array import array -from collections import abc - -from ._abc import MultiMapping, MutableMultiMapping - -_marker = object() - -if sys.version_info >= (3, 9): - GenericAlias = types.GenericAlias +from collections.abc import ( + Callable, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + ValuesView, +) +from typing import ( + TYPE_CHECKING, + Generic, + NoReturn, + TypeVar, + Union, + cast, + overload, +) + +from ._abc import MDArg, MultiMapping, MutableMultiMapping, SupportsKeys + +if sys.version_info >= (3, 11): + from typing import Self else: - - def GenericAlias(cls): - return cls + from typing_extensions import Self class istr(str): - """Case insensitive str.""" __is_istr__ = True -upstr = istr # for relaxing backward compatibility problems - - -def getversion(md): - if not isinstance(md, _Base): - raise TypeError("Parameter should be multidict or proxy") - return md._impl._version +_V = TypeVar("_V") +_T = TypeVar("_T") +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel _version = array("Q", [0]) -class _Impl: +class _Impl(Generic[_V]): __slots__ = ("_items", "_version") - def __init__(self): - self._items = [] + def __init__(self) -> None: + self._items: list[tuple[str, str, _V]] = [] self.incr_version() - def incr_version(self): + def incr_version(self) -> None: global _version v = _version v[0] += 1 @@ -49,25 +58,138 @@ class _Impl: if sys.implementation.name != "pypy": - def __sizeof__(self): + def __sizeof__(self) -> int: return object.__sizeof__(self) + sys.getsizeof(self._items) -class _Base: - def _title(self, key): +class _Iter(Generic[_T]): + __slots__ = ("_size", "_iter") + + def __init__(self, size: int, iterator: Iterator[_T]): + self._size = size + self._iter = iterator + + def __iter__(self) -> Self: + return self + + def __next__(self) -> _T: + return next(self._iter) + + def __length_hint__(self) -> int: + return self._size + + +class _ViewBase(Generic[_V]): + def __init__(self, impl: _Impl[_V]): + self._impl = impl + + def __len__(self) -> int: + return len(self._impl._items) + + +class _ItemsView(_ViewBase[_V], ItemsView[str, _V]): + def __contains__(self, item: object) -> bool: + if not isinstance(item, (tuple, list)) or len(item) != 2: + return False + for i, k, v in self._impl._items: + if item[0] == k and item[1] == v: + return True + return False + + def __iter__(self) -> _Iter[tuple[str, _V]]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[tuple[str, _V]]: + for i, k, v in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield k, v + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}: {!r}".format(item[1], item[2])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _ValuesView(_ViewBase[_V], ValuesView[_V]): + def __contains__(self, value: object) -> bool: + for item in self._impl._items: + if item[2] == value: + return True + return False + + def __iter__(self) -> _Iter[_V]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[_V]: + for item in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield item[2] + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}".format(item[2])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _KeysView(_ViewBase[_V], KeysView[str]): + def __contains__(self, key: object) -> bool: + for item in self._impl._items: + if item[1] == key: + return True + return False + + def __iter__(self) -> _Iter[str]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[str]: + for item in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield item[1] + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}".format(item[1])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _Base(MultiMapping[_V]): + _impl: _Impl[_V] + + def _title(self, key: str) -> str: return key - def getall(self, key, default=_marker): + @overload + def getall(self, key: str) -> list[_V]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def getall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: """Return a list of all values matching the key.""" identity = self._title(key) res = [v for i, k, v in self._impl._items if i == identity] if res: return res - if not res and default is not _marker: + if not res and default is not sentinel: return default raise KeyError("Key not found: %r" % key) - def getone(self, key, default=_marker): + @overload + def getone(self, key: str) -> _V: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V, _T]: ... + def getone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: """Get first value matching the key. Raises KeyError if the key is not found and no default is provided. @@ -76,42 +198,46 @@ class _Base: for i, k, v in self._impl._items: if i == identity: return v - if default is not _marker: + if default is not sentinel: return default raise KeyError("Key not found: %r" % key) # Mapping interface # - def __getitem__(self, key): + def __getitem__(self, key: str) -> _V: return self.getone(key) - def get(self, key, default=None): + @overload + def get(self, key: str, /) -> Union[_V, None]: ... + @overload + def get(self, key: str, /, default: _T) -> Union[_V, _T]: ... + def get(self, key: str, default: Union[_T, None] = None) -> Union[_V, _T, None]: """Get first value matching the key. If the key is not found, returns the default (or None if no default is provided) """ return self.getone(key, default) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.keys()) - def __len__(self): + def __len__(self) -> int: return len(self._impl._items) - def keys(self): + def keys(self) -> KeysView[str]: """Return a new view of the dictionary's keys.""" return _KeysView(self._impl) - def items(self): + def items(self) -> ItemsView[str, _V]: """Return a new view of the dictionary's items *(key, value) pairs).""" return _ItemsView(self._impl) - def values(self): + def values(self) -> _ValuesView[_V]: """Return a new view of the dictionary's values.""" return _ValuesView(self._impl) - def __eq__(self, other): - if not isinstance(other, abc.Mapping): + def __eq__(self, other: object) -> bool: + if not isinstance(other, Mapping): return NotImplemented if isinstance(other, _Base): lft = self._impl._items @@ -125,124 +251,83 @@ class _Base: if len(self._impl._items) != len(other): return False for k, v in self.items(): - nv = other.get(k, _marker) + nv = other.get(k, sentinel) if v != nv: return False return True - def __contains__(self, key): + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False identity = self._title(key) for i, k, v in self._impl._items: if i == identity: return True return False - def __repr__(self): + def __repr__(self) -> str: body = ", ".join("'{}': {!r}".format(k, v) for k, v in self.items()) return "<{}({})>".format(self.__class__.__name__, body) - __class_getitem__ = classmethod(GenericAlias) - - -class MultiDictProxy(_Base, MultiMapping): - """Read-only proxy for MultiDict instance.""" - - def __init__(self, arg): - if not isinstance(arg, (MultiDict, MultiDictProxy)): - raise TypeError( - "ctor requires MultiDict or MultiDictProxy instance" - ", not {}".format(type(arg)) - ) - - self._impl = arg._impl - def __reduce__(self): - raise TypeError("can't pickle {} objects".format(self.__class__.__name__)) - - def copy(self): - """Return a copy of itself.""" - return MultiDict(self.items()) - - -class CIMultiDictProxy(MultiDictProxy): - """Read-only proxy for CIMultiDict instance.""" - - def __init__(self, arg): - if not isinstance(arg, (CIMultiDict, CIMultiDictProxy)): - raise TypeError( - "ctor requires CIMultiDict or CIMultiDictProxy instance" - ", not {}".format(type(arg)) - ) - - self._impl = arg._impl - - def _title(self, key): - return key.title() - - def copy(self): - """Return a copy of itself.""" - return CIMultiDict(self.items()) - - -class MultiDict(_Base, MutableMultiMapping): +class MultiDict(_Base[_V], MutableMultiMapping[_V]): """Dictionary with the support for duplicate keys.""" - def __init__(self, *args, **kwargs): + def __init__(self, arg: MDArg[_V] = None, /, **kwargs: _V): self._impl = _Impl() - self._extend(args, kwargs, self.__class__.__name__, self._extend_items) + self._extend(arg, kwargs, self.__class__.__name__, self._extend_items) if sys.implementation.name != "pypy": - def __sizeof__(self): + def __sizeof__(self) -> int: return object.__sizeof__(self) + sys.getsizeof(self._impl) - def __reduce__(self): + def __reduce__(self) -> tuple[type[Self], tuple[list[tuple[str, _V]]]]: return (self.__class__, (list(self.items()),)) - def _title(self, key): + def _title(self, key: str) -> str: return key - def _key(self, key): + def _key(self, key: str) -> str: if isinstance(key, str): return key else: - raise TypeError( - "MultiDict keys should be either str " "or subclasses of str" - ) + raise TypeError("MultiDict keys should be either str or subclasses of str") - def add(self, key, value): + def add(self, key: str, value: _V) -> None: identity = self._title(key) self._impl._items.append((identity, self._key(key), value)) self._impl.incr_version() - def copy(self): + def copy(self) -> Self: """Return a copy of itself.""" cls = self.__class__ return cls(self.items()) __copy__ = copy - def extend(self, *args, **kwargs): + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: """Extend current MultiDict with more values. This method must be used instead of update. """ - self._extend(args, kwargs, "extend", self._extend_items) - - def _extend(self, args, kwargs, name, method): - if len(args) > 1: - raise TypeError( - "{} takes at most 1 positional argument" - " ({} given)".format(name, len(args)) - ) - if args: - arg = args[0] - if isinstance(args[0], (MultiDict, MultiDictProxy)) and not kwargs: + self._extend(arg, kwargs, "extend", self._extend_items) + + def _extend( + self, + arg: MDArg[_V], + kwargs: Mapping[str, _V], + name: str, + method: Callable[[list[tuple[str, str, _V]]], None], + ) -> None: + if arg: + if isinstance(arg, (MultiDict, MultiDictProxy)) and not kwargs: items = arg._impl._items else: - if hasattr(arg, "items"): - arg = arg.items() + if hasattr(arg, "keys"): + arg = cast(SupportsKeys[_V], arg) + arg = [(k, arg[k]) for k in arg.keys()] if kwargs: arg = list(arg) arg.extend(list(kwargs.items())) @@ -264,21 +349,21 @@ class MultiDict(_Base, MutableMultiMapping): ] ) - def _extend_items(self, items): + def _extend_items(self, items: Iterable[tuple[str, str, _V]]) -> None: for identity, key, value in items: self.add(key, value) - def clear(self): + def clear(self) -> None: """Remove all items from MultiDict.""" self._impl._items.clear() self._impl.incr_version() # Mapping interface # - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: _V) -> None: self._replace(key, value) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: identity = self._title(key) items = self._impl._items found = False @@ -291,16 +376,28 @@ class MultiDict(_Base, MutableMultiMapping): else: self._impl.incr_version() - def setdefault(self, key, default=None): + @overload + def setdefault( + self: "MultiDict[Union[_T, None]]", key: str, default: None = None + ) -> Union[_T, None]: ... + @overload + def setdefault(self, key: str, default: _V) -> _V: ... + def setdefault(self, key: str, default: Union[_V, None] = None) -> Union[_V, None]: # type: ignore[misc] """Return value for key, set value to default if key is not present.""" identity = self._title(key) for i, k, v in self._impl._items: if i == identity: return v - self.add(key, default) + self.add(key, default) # type: ignore[arg-type] return default - def popone(self, key, default=_marker): + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... + def popone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: """Remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise @@ -314,14 +411,22 @@ class MultiDict(_Base, MutableMultiMapping): del self._impl._items[i] self._impl.incr_version() return value - if default is _marker: + if default is sentinel: raise KeyError(key) else: return default - pop = popone # type: ignore - - def popall(self, key, default=_marker): + # Type checking will inherit signature for pop() if we don't confuse it here. + if not TYPE_CHECKING: + pop = popone + + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def popall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: """Remove all occurrences of key and return the list of corresponding values. @@ -340,7 +445,7 @@ class MultiDict(_Base, MutableMultiMapping): self._impl.incr_version() found = True if not found: - if default is _marker: + if default is sentinel: raise KeyError(key) else: return default @@ -348,7 +453,7 @@ class MultiDict(_Base, MutableMultiMapping): ret.reverse() return ret - def popitem(self): + def popitem(self) -> tuple[str, _V]: """Remove and return an arbitrary (key, value) pair.""" if self._impl._items: i = self._impl._items.pop(0) @@ -357,14 +462,14 @@ class MultiDict(_Base, MutableMultiMapping): else: raise KeyError("empty multidict") - def update(self, *args, **kwargs): + def update(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: """Update the dictionary from *other*, overwriting existing keys.""" - self._extend(args, kwargs, "update", self._update_items) + self._extend(arg, kwargs, "update", self._update_items) - def _update_items(self, items): + def _update_items(self, items: list[tuple[str, str, _V]]) -> None: if not items: return - used_keys = {} + used_keys: dict[str, int] = {} for identity, key, value in items: start = used_keys.get(identity, 0) for i in range(start, len(self._impl._items)): @@ -393,7 +498,7 @@ class MultiDict(_Base, MutableMultiMapping): self._impl.incr_version() - def _replace(self, key, value): + def _replace(self, key: str, value: _V) -> None: key = self._key(key) identity = self._title(key) items = self._impl._items @@ -412,7 +517,8 @@ class MultiDict(_Base, MutableMultiMapping): return # remove all tail items - i = rgt + 1 + # Mypy bug: https://github.com/python/mypy/issues/14209 + i = rgt + 1 # type: ignore[possibly-undefined] while i < len(items): item = items[i] if item[0] == identity: @@ -421,107 +527,54 @@ class MultiDict(_Base, MutableMultiMapping): i += 1 -class CIMultiDict(MultiDict): +class CIMultiDict(MultiDict[_V]): """Dictionary with the support for duplicate case-insensitive keys.""" - def _title(self, key): + def _title(self, key: str) -> str: return key.title() -class _Iter: - __slots__ = ("_size", "_iter") - - def __init__(self, size, iterator): - self._size = size - self._iter = iterator - - def __iter__(self): - return self - - def __next__(self): - return next(self._iter) - - def __length_hint__(self): - return self._size - - -class _ViewBase: - def __init__(self, impl): - self._impl = impl - - def __len__(self): - return len(self._impl._items) - - -class _ItemsView(_ViewBase, abc.ItemsView): - def __contains__(self, item): - assert isinstance(item, tuple) or isinstance(item, list) - assert len(item) == 2 - for i, k, v in self._impl._items: - if item[0] == k and item[1] == v: - return True - return False - - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) +class MultiDictProxy(_Base[_V]): + """Read-only proxy for MultiDict instance.""" - def _iter(self, version): - for i, k, v in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield k, v + def __init__(self, arg: Union[MultiDict[_V], "MultiDictProxy[_V]"]): + if not isinstance(arg, (MultiDict, MultiDictProxy)): + raise TypeError( + "ctor requires MultiDict or MultiDictProxy instance" + ", not {}".format(type(arg)) + ) - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}: {!r}".format(item[1], item[2])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) + self._impl = arg._impl + def __reduce__(self) -> NoReturn: + raise TypeError("can't pickle {} objects".format(self.__class__.__name__)) -class _ValuesView(_ViewBase, abc.ValuesView): - def __contains__(self, value): - for item in self._impl._items: - if item[2] == value: - return True - return False + def copy(self) -> MultiDict[_V]: + """Return a copy of itself.""" + return MultiDict(self.items()) - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) - def _iter(self, version): - for item in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield item[2] +class CIMultiDictProxy(MultiDictProxy[_V]): + """Read-only proxy for CIMultiDict instance.""" - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}".format(item[2])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) + def __init__(self, arg: Union[MultiDict[_V], MultiDictProxy[_V]]): + if not isinstance(arg, (CIMultiDict, CIMultiDictProxy)): + raise TypeError( + "ctor requires CIMultiDict or CIMultiDictProxy instance" + ", not {}".format(type(arg)) + ) + self._impl = arg._impl -class _KeysView(_ViewBase, abc.KeysView): - def __contains__(self, key): - for item in self._impl._items: - if item[1] == key: - return True - return False + def _title(self, key: str) -> str: + return key.title() - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) + def copy(self) -> CIMultiDict[_V]: + """Return a copy of itself.""" + return CIMultiDict(self.items()) - def _iter(self, version): - for item in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield item[1] - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}".format(item[1])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) +def getversion(md: Union[MultiDict[object], MultiDictProxy[object]]) -> int: + if not isinstance(md, _Base): + raise TypeError("Parameter should be multidict or proxy") + return md._impl._version diff --git a/contrib/python/multidict/multidict/_multilib/defs.h b/contrib/python/multidict/multidict/_multilib/defs.h index 55c21074dd..51a6639c42 100644 --- a/contrib/python/multidict/multidict/_multilib/defs.h +++ b/contrib/python/multidict/multidict/_multilib/defs.h @@ -5,11 +5,7 @@ extern "C" { #endif -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 static PyObject *multidict_str_lower = NULL; -#else -_Py_IDENTIFIER(lower); -#endif /* We link this module statically for convenience. If compiled as a shared library instead, some compilers don't allow addresses of Python objects diff --git a/contrib/python/multidict/multidict/_multilib/istr.h b/contrib/python/multidict/multidict/_multilib/istr.h index 61dc61aec6..8454f78b88 100644 --- a/contrib/python/multidict/multidict/_multilib/istr.h +++ b/contrib/python/multidict/multidict/_multilib/istr.h @@ -43,11 +43,7 @@ istr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!ret) { goto fail; } -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 s = PyObject_CallMethodNoArgs(ret, multidict_str_lower); -#else - s =_PyObject_CallMethodId(ret, &PyId_lower, NULL); -#endif if (!s) { goto fail; } diff --git a/contrib/python/multidict/multidict/_multilib/pair_list.h b/contrib/python/multidict/multidict/_multilib/pair_list.h index 15291d46a8..b23150dfad 100644 --- a/contrib/python/multidict/multidict/_multilib/pair_list.h +++ b/contrib/python/multidict/multidict/_multilib/pair_list.h @@ -31,11 +31,7 @@ The embedded buffer intention is to fit the vast majority of possible HTTP headers into the buffer without allocating an extra memory block. */ -#if (PY_VERSION_HEX < 0x03080000) -#define EMBEDDED_CAPACITY 28 -#else #define EMBEDDED_CAPACITY 29 -#endif typedef struct pair_list { Py_ssize_t capacity; @@ -110,11 +106,7 @@ ci_key_to_str(PyObject *key) return ret; } if (PyUnicode_Check(key)) { -#if PY_VERSION_HEX < 0x03090000 - return _PyObject_CallMethodId(key, &PyId_lower, NULL); -#else return PyObject_CallMethodNoArgs(key, multidict_str_lower); -#endif } PyErr_SetString(PyExc_TypeError, "CIMultiDict keys should be either str " @@ -497,6 +489,10 @@ pair_list_contains(pair_list_t *list, PyObject *key) PyObject *identity = NULL; int tmp; + if (!PyUnicode_Check(key)) { + return 0; + } + ident = pair_list_calc_identity(list, key); if (ident == NULL) { goto fail; @@ -916,13 +912,18 @@ _pair_list_post_update(pair_list_t *list, PyObject* used_keys, Py_ssize_t pos) for (; pos < list->size; pos++) { pair = pair_list_get(list, pos); - tmp = PyDict_GetItem(used_keys, pair->identity); - if (tmp == NULL) { + int status = PyDict_GetItemRef(used_keys, pair->identity, &tmp); + if (status == -1) { + // exception set + return -1; + } + else if (status == 0) { // not found continue; } num = PyLong_AsSsize_t(tmp); + Py_DECREF(tmp); if (num == -1) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "invalid internal state"); @@ -955,12 +956,18 @@ _pair_list_update(pair_list_t *list, PyObject *key, int found; int ident_cmp_res; - item = PyDict_GetItem(used_keys, identity); - if (item == NULL) { + int status = PyDict_GetItemRef(used_keys, identity, &item); + if (status == -1) { + // exception set + return -1; + } + else if (status == 0) { + // not found pos = 0; } else { pos = PyLong_AsSsize_t(item); + Py_DECREF(item); if (pos == -1) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "invalid internal state"); @@ -1087,18 +1094,28 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) } // Convert item to sequence, and verify length 2. +#ifdef Py_GIL_DISABLED + if (!PySequence_Check(item)) { +#else fast = PySequence_Fast(item, ""); if (fast == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { +#endif PyErr_Format(PyExc_TypeError, "multidict cannot convert sequence element #%zd" " to a sequence", i); +#ifndef Py_GIL_DISABLED } +#endif goto fail_1; } +#ifdef Py_GIL_DISABLED + n = PySequence_Size(item); +#else n = PySequence_Fast_GET_SIZE(fast); +#endif if (n != 2) { PyErr_Format(PyExc_ValueError, "multidict update sequence element #%zd " @@ -1107,10 +1124,27 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) goto fail_1; } +#ifdef Py_GIL_DISABLED + key = PySequence_ITEM(item, 0); + if (key == NULL) { + PyErr_Format(PyExc_ValueError, + "multidict update sequence element #%zd's " + "key could not be fetched", i); + goto fail_1; + } + value = PySequence_ITEM(item, 1); + if (value == NULL) { + PyErr_Format(PyExc_ValueError, + "multidict update sequence element #%zd's " + "value could not be fetched", i); + goto fail_1; + } +#else key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); Py_INCREF(key); Py_INCREF(value); +#endif identity = pair_list_calc_identity(list, key); if (identity == NULL) { @@ -1128,7 +1162,9 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) Py_DECREF(key); Py_DECREF(value); +#ifndef Py_GIL_DISABLED Py_DECREF(fast); +#endif Py_DECREF(item); Py_DECREF(identity); } diff --git a/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h b/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h new file mode 100644 index 0000000000..971981993b --- /dev/null +++ b/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h @@ -0,0 +1,1142 @@ +// Header file providing new C API functions to old Python versions. +// +// File distributed under the Zero Clause BSD (0BSD) license. +// Copyright Contributors to the pythoncapi_compat project. +// +// Homepage: +// https://github.com/python/pythoncapi_compat +// +// Latest version: +// https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h +// +// The vendored version comes from commit: +// https://raw.githubusercontent.com/python/pythoncapi-compat/2d18aecd7b2f549d38a13e27b682ea4966f37bd8/pythoncapi_compat.h +// +// SPDX-License-Identifier: 0BSD + +#ifndef PYTHONCAPI_COMPAT +#define PYTHONCAPI_COMPAT + +#ifdef __cplusplus +extern "C" { +#endif + +#include <Python.h> + +// Python 3.11.0b4 added PyFrame_Back() to Python.h +#if PY_VERSION_HEX < 0x030b00B4 && !defined(PYPY_VERSION) +# include "frameobject.h" // PyFrameObject, PyFrame_GetBack() +#endif + + +#ifndef _Py_CAST +# define _Py_CAST(type, expr) ((type)(expr)) +#endif + +// Static inline functions should use _Py_NULL rather than using directly NULL +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +#else +# define _Py_NULL NULL +#endif + +// Cast argument to PyObject* type. +#ifndef _PyObject_CAST +# define _PyObject_CAST(op) _Py_CAST(PyObject*, op) +#endif + + +// bpo-42262 added Py_NewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} +#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-42262 added Py_XNewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} +#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-43753 added Py_Is(), Py_IsNone(), Py_IsTrue() and Py_IsFalse() +// to Python 3.10.0b1. +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_Is) +# define Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) +# define Py_IsNone(x) Py_Is(x, Py_None) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsTrue) +# define Py_IsTrue(x) Py_Is(x, Py_True) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsFalse) +# define Py_IsFalse(x) Py_Is(x, Py_False) +#endif + + +#if defined(PYPY_VERSION) +static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) +{ + assert(frame != _Py_NULL); + assert(frame->f_code != _Py_NULL); + return _Py_CAST(PyCodeObject*, Py_NewRef(frame->f_code)); +} +#endif + +static inline PyCodeObject* _PyFrame_GetCodeBorrow(PyFrameObject *frame) +{ + PyCodeObject *code = PyFrame_GetCode(frame); + Py_DECREF(code); + return code; +} + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* _PyFrame_GetBackBorrow(PyFrameObject *frame) +{ + PyFrameObject *back = PyFrame_GetBack(frame); + Py_XDECREF(back); + return back; +} +#endif + + +// bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetLocals(PyFrameObject *frame) +{ + if (PyFrame_FastToLocalsWithError(frame) < 0) { + return NULL; + } + return Py_NewRef(frame->f_locals); +} +#endif + + +// bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetGlobals(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_globals); +} +#endif + + +// bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_builtins); +} +#endif + + +// bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline int PyFrame_GetLasti(PyFrameObject *frame) +{ +#if PY_VERSION_HEX >= 0x030A00A7 + // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, + // not a bytes offset anymore. Python uses 16-bit "wordcode" (2 bytes) + // instructions. + if (frame->f_lasti < 0) { + return -1; + } + return frame->f_lasti * 2; +#else + return frame->f_lasti; +#endif +} +#endif + + +// gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name) +{ + PyObject *locals, *value; + + locals = PyFrame_GetLocals(frame); + if (locals == NULL) { + return NULL; + } + value = PyDict_GetItemWithError(locals, name); + Py_DECREF(locals); + + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + PyErr_Format(PyExc_NameError, "variable %R does not exist", name); + return NULL; + } + return Py_NewRef(value); +} +#endif + + +// gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* +PyFrame_GetVarString(PyFrameObject *frame, const char *name) +{ + PyObject *name_obj, *value; + name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { + return NULL; + } + value = PyFrame_GetVar(frame, name_obj); + Py_DECREF(name_obj); + return value; +} +#endif + + +#if defined(PYPY_VERSION) +static inline PyInterpreterState * +PyThreadState_GetInterpreter(PyThreadState *tstate) +{ + assert(tstate != _Py_NULL); + return tstate->interp; +} +#endif + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* +_PyThreadState_GetFrameBorrow(PyThreadState *tstate) +{ + PyFrameObject *frame = PyThreadState_GetFrame(tstate); + Py_XDECREF(frame); + return frame; +} +#endif + + +#if defined(PYPY_VERSION) +static inline PyInterpreterState* PyInterpreterState_Get(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_GET(); + if (tstate == _Py_NULL) { + Py_FatalError("GIL released (tstate is NULL)"); + } + interp = tstate->interp; + if (interp == _Py_NULL) { + Py_FatalError("no current interpreter"); + } + return interp; +} +#endif + +// bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_EnterTracing(PyThreadState *tstate) +{ + tstate->tracing++; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = 0; +#else + tstate->use_tracing = 0; +#endif +} +#endif + +// bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + int use_tracing = (tstate->c_tracefunc != _Py_NULL + || tstate->c_profilefunc != _Py_NULL); + tstate->tracing--; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = use_tracing; +#else + tstate->use_tracing = use_tracing; +#endif +} +#endif + + +// bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 +static inline int +PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) +{ + int res; + + if (!value && !PyErr_Occurred()) { + // PyModule_AddObject() raises TypeError in this case + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + return -1; + } + + Py_XINCREF(value); + res = PyModule_AddObject(module, name, value); + if (res < 0) { + Py_XDECREF(value); + } + return res; +} +#endif + + +// bpo-46906 added PyFloat_Pack2() and PyFloat_Unpack2() to Python 3.11a7. +// Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal +// C API: Python 3.11a2-3.11a6 versions are not supported. +#if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack2(double x, char *p, int le) +{ return _PyFloat_Pack2(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack2(const char *p, int le) +{ return _PyFloat_Unpack2((const unsigned char *)p, le); } +#endif + + +// bpo-46906 added PyFloat_Pack4(), PyFloat_Pack8(), PyFloat_Unpack4() and +// PyFloat_Unpack8() to Python 3.11a7. +// Python 3.11a2 moved _PyFloat_Pack4(), _PyFloat_Pack8(), _PyFloat_Unpack4() +// and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions +// are not supported. +#if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack4(double x, char *p, int le) +{ return _PyFloat_Pack4(x, (unsigned char*)p, le); } + +static inline int PyFloat_Pack8(double x, char *p, int le) +{ return _PyFloat_Pack8(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack4(const char *p, int le) +{ return _PyFloat_Unpack4((const unsigned char *)p, le); } + +static inline double PyFloat_Unpack8(const char *p, int le) +{ return _PyFloat_Unpack8((const unsigned char *)p, le); } +#endif + + +// gh-92154 added PyCode_GetCode() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCode(PyCodeObject *code) +{ + return Py_NewRef(code->co_code); +} +#endif + + +// gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetVarnames(PyCodeObject *code) +{ + return Py_NewRef(code->co_varnames); +} +#endif + +// gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetFreevars(PyCodeObject *code) +{ + return Py_NewRef(code->co_freevars); +} +#endif + +// gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCellvars(PyCodeObject *code) +{ + return Py_NewRef(code->co_cellvars); +} +#endif + + +// gh-105922 added PyImport_AddModuleRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A0 +static inline PyObject* PyImport_AddModuleRef(const char *name) +{ + return Py_XNewRef(PyImport_AddModule(name)); +} +#endif + + +// gh-105927 added PyWeakref_GetRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D0000 +static inline int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj; + if (ref != NULL && !PyWeakref_Check(ref)) { + *pobj = NULL; + PyErr_SetString(PyExc_TypeError, "expected a weakref"); + return -1; + } + obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + *pobj = Py_NewRef(obj); + return (*pobj != NULL); +} +#endif + + +// gh-106521 added PyObject_GetOptionalAttr() and +// PyObject_GetOptionalAttrString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result) +{ + return _PyObject_LookupAttr(obj, attr_name, result); +} + +static inline int +PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result) +{ + PyObject *name_obj; + int rc; + name_obj = PyUnicode_FromString(attr_name); + if (name_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyObject_GetOptionalAttr(obj, name_obj, result); + Py_DECREF(name_obj); + return rc; +} +#endif + + +// gh-106307 added PyObject_GetOptionalAttr() and +// PyMapping_GetOptionalItemString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) +{ + *result = PyObject_GetItem(obj, key); + if (*result) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; +} + +static inline int +PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) +{ + PyObject *key_obj; + int rc; + key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyMapping_GetOptionalItem(obj, key_obj, result); + Py_DECREF(key_obj); + return rc; +} +#endif + +// gh-108511 added PyMapping_HasKeyWithError() and +// PyMapping_HasKeyStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItem(obj, key, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyMapping_HasKeyStringWithError(PyObject *obj, const char *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItemString(obj, key, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-108511 added PyObject_HasAttrWithError() and +// PyObject_HasAttrStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_HasAttrWithError(PyObject *obj, PyObject *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttr(obj, attr, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyObject_HasAttrStringWithError(PyObject *obj, const char *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, attr, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-106004 added PyDict_GetItemRef() and PyDict_GetItemStringRef() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ + PyObject *item = PyDict_GetItemWithError(mp, key); + if (item != NULL) { + *result = Py_NewRef(item); + return 1; // found + } + if (!PyErr_Occurred()) { + *result = NULL; + return 0; // not found + } + *result = NULL; + return -1; +} + +static inline int +PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result) +{ + int res; + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + *result = NULL; + return -1; + } + res = PyDict_GetItemRef(mp, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-106307 added PyModule_Add() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} +#endif + + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +// _Py_IsFinalizing() was added to PyPy 7.3.0. +#if (PY_VERSION_HEX < 0x030D00A1) \ + && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x7030000) +static inline int Py_IsFinalizing(void) +{ + return _Py_IsFinalizing(); +} +#endif + + +// gh-108323 added PyDict_ContainsString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-108445 added PyLong_AsInt() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyLong_AsInt(PyObject *obj) +{ +#ifdef PYPY_VERSION + long value = PyLong_AsLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + if (value < (long)INT_MIN || (long)INT_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)value; +#else + return _PyLong_AsInt(obj); +#endif +} +#endif + + +// gh-107073 added PyObject_VisitManagedDict() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return -1; + } + Py_VISIT(*dict); + return 0; +} + +static inline void +PyObject_ClearManagedDict(PyObject *obj) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return; + } + Py_CLEAR(*dict); +} +#endif + +// gh-108867 added PyThreadState_GetUnchecked() to Python 3.13.0a1. +#if PY_VERSION_HEX < 0x030D00A1 +static inline PyThreadState* +PyThreadState_GetUnchecked(void) +{ + return _PyThreadState_UncheckedGet(); +} +#endif + +// gh-110289 added PyUnicode_EqualToUTF8() and PyUnicode_EqualToUTF8AndSize() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *str, Py_ssize_t str_len) +{ + Py_ssize_t len; + const void *utf8; + PyObject *exc_type, *exc_value, *exc_tb; + int res; + + // API cannot report errors so save/restore the exception + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + if (PyUnicode_IS_ASCII(unicode)) { + utf8 = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + } + else { + utf8 = PyUnicode_AsUTF8AndSize(unicode, &len); + if (utf8 == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + } + + if (len != str_len) { + res = 0; + goto done; + } + res = (memcmp(utf8, str, (size_t)len) == 0); + +done: + PyErr_Restore(exc_type, exc_value, exc_tb); + return res; +} + +static inline int +PyUnicode_EqualToUTF8(PyObject *unicode, const char *str) +{ + return PyUnicode_EqualToUTF8AndSize(unicode, str, (Py_ssize_t)strlen(str)); +} +#endif + + +// gh-111138 added PyList_Extend() and PyList_Clear() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyList_Extend(PyObject *list, PyObject *iterable) +{ + return PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable); +} + +static inline int +PyList_Clear(PyObject *list) +{ + return PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL); +} +#endif + +// gh-111262 added PyDict_Pop() and PyDict_PopString() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result) +{ + PyObject *value; + + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + if (result) { + *result = NULL; + } + return -1; + } + + // Python 3.13.0a1 removed _PyDict_Pop(). +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x030D0000 + value = PyObject_CallMethod(dict, "pop", "O", key); +#else + value = _PyDict_Pop(dict, key, NULL); +#endif + if (value == NULL) { + if (result) { + *result = NULL; + } + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; +} + +static inline int +PyDict_PopString(PyObject *dict, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(dict, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-111545 added Py_HashPointer() to Python 3.13.0a3 +#if PY_VERSION_HEX < 0x030D00A3 +static inline Py_hash_t Py_HashPointer(const void *ptr) +{ +#if !defined(PYPY_VERSION) + return _Py_HashPointer(ptr); +#else + return _Py_HashPointer(_Py_CAST(void*, ptr)); +#endif +} +#endif + + +// Python 3.13a4 added a PyTime API. +#if PY_VERSION_HEX < 0x030D00A4 +typedef _PyTime_t PyTime_t; +#define PyTime_MIN _PyTime_MIN +#define PyTime_MAX _PyTime_MAX + +static inline double PyTime_AsSecondsDouble(PyTime_t t) +{ return _PyTime_AsSecondsDouble(t); } + +static inline int PyTime_Monotonic(PyTime_t *result) +{ return _PyTime_GetMonotonicClockWithInfo(result, NULL); } + +static inline int PyTime_Time(PyTime_t *result) +{ return _PyTime_GetSystemClockWithInfo(result, NULL); } + +static inline int PyTime_PerfCounter(PyTime_t *result) +{ +#if !defined(PYPY_VERSION) + return _PyTime_GetPerfCounterWithInfo(result, NULL); +#else + // Call time.perf_counter_ns() and convert Python int object to PyTime_t. + // Cache time.perf_counter_ns() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter_ns"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + long long value = PyLong_AsLongLong(res); + Py_DECREF(res); + + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(value) >= sizeof(PyTime_t)); + *result = (PyTime_t)value; + return 0; +#endif +} + +#endif + +// gh-111389 added hash constants to Python 3.13.0a5. These constants were +// added first as private macros to Python 3.4.0b1 and PyPy 7.3.9. +#if (!defined(PyHASH_BITS) \ + && (!defined(PYPY_VERSION) \ + || (defined(PYPY_VERSION) && PYPY_VERSION_NUM >= 0x07090000))) +# define PyHASH_BITS _PyHASH_BITS +# define PyHASH_MODULUS _PyHASH_MODULUS +# define PyHASH_INF _PyHASH_INF +# define PyHASH_IMAG _PyHASH_IMAG +#endif + + +// gh-111545 added Py_GetConstant() and Py_GetConstantBorrowed() +// to Python 3.13.0a6 +#if PY_VERSION_HEX < 0x030D00A6 && !defined(Py_CONSTANT_NONE) + +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +static inline PyObject* Py_GetConstant(unsigned int constant_id) +{ + static PyObject* constants[Py_CONSTANT_EMPTY_TUPLE + 1] = {NULL}; + + if (constants[Py_CONSTANT_NONE] == NULL) { + constants[Py_CONSTANT_NONE] = Py_None; + constants[Py_CONSTANT_FALSE] = Py_False; + constants[Py_CONSTANT_TRUE] = Py_True; + constants[Py_CONSTANT_ELLIPSIS] = Py_Ellipsis; + constants[Py_CONSTANT_NOT_IMPLEMENTED] = Py_NotImplemented; + + constants[Py_CONSTANT_ZERO] = PyLong_FromLong(0); + if (constants[Py_CONSTANT_ZERO] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_ONE] = PyLong_FromLong(1); + if (constants[Py_CONSTANT_ONE] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_STR] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_BYTES] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); + if (constants[Py_CONSTANT_EMPTY_TUPLE] == NULL) { + goto fatal_error; + } + // goto dance to avoid compiler warnings about Py_FatalError() + goto init_done; + +fatal_error: + // This case should never happen + Py_FatalError("Py_GetConstant() failed to get constants"); + } + +init_done: + if (constant_id <= Py_CONSTANT_EMPTY_TUPLE) { + return Py_NewRef(constants[constant_id]); + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + +static inline PyObject* Py_GetConstantBorrowed(unsigned int constant_id) +{ + PyObject *obj = Py_GetConstant(constant_id); + Py_XDECREF(obj); + return obj; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t index) +{ + PyObject *item = PyList_GetItem(op, index); + Py_XINCREF(item); + return item; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + PyObject *value; + if (PyDict_GetItemRef(d, key, &value) < 0) { + // get error + if (result) { + *result = NULL; + } + return -1; + } + if (value != NULL) { + // present + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; + } + + // missing: set the item + if (PyDict_SetItem(d, key, default_value) < 0) { + // set error + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = Py_NewRef(default_value); + } + return 0; +} +#endif + +#if PY_VERSION_HEX < 0x030D00B3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) { +# define Py_END_CRITICAL_SECTION2() } +#endif + +#if PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) +typedef struct PyUnicodeWriter PyUnicodeWriter; + +static inline void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + +static inline PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == _Py_NULL) { + PyErr_NoMemory(); + return _Py_NULL; + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + return pub_writer; +} + +static inline PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + +static inline int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > 0x10ffff) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + +static inline int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Repr(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + PyObject *str_obj = PyUnicode_FromStringAndSize(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, + const wchar_t *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)wcslen(str); + } + + PyObject *str_obj = PyUnicode_FromWideChar(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + +static inline int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (str == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030E0000 + +// gh-116560 added PyLong_GetSign() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyLong_GetSign(PyObject *obj, int *sign) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + + *sign = _PyLong_Sign(obj); + return 0; +} +#endif + + +#ifdef __cplusplus +} +#endif +#endif // PYTHONCAPI_COMPAT diff --git a/contrib/python/multidict/tests/conftest.py b/contrib/python/multidict/tests/conftest.py index 0d003950cd..a37f58f2d1 100644 --- a/contrib/python/multidict/tests/conftest.py +++ b/contrib/python/multidict/tests/conftest.py @@ -3,26 +3,22 @@ from __future__ import annotations import argparse import pickle from dataclasses import dataclass +from functools import cached_property from importlib import import_module -from sys import version_info as _version_info from types import ModuleType -from typing import Callable, Type - -try: - from functools import cached_property # Python 3.8+ -except ImportError: - from functools import lru_cache as _lru_cache - - def cached_property(func): - return property(_lru_cache()(func)) - +from typing import Callable, Type, Union import pytest -from multidict import MultiMapping, MutableMultiMapping +from multidict import ( + CIMultiDict, + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) C_EXT_MARK = pytest.mark.c_extension -PY_38_AND_BELOW = _version_info < (3, 9) @dataclass(frozen=True) @@ -51,7 +47,7 @@ class MultidictImplementation: importable_module = "_multidict_py" if self.is_pure_python else "_multidict" return import_module(f"multidict.{importable_module}") - def __str__(self): + def __str__(self) -> str: """Render the implementation facade instance as a string.""" return f"{self.tag}-module" @@ -69,7 +65,7 @@ class MultidictImplementation: ) def multidict_implementation(request: pytest.FixtureRequest) -> MultidictImplementation: """Return a multidict variant facade.""" - return request.param + return request.param # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -87,7 +83,7 @@ def multidict_module( ) def any_multidict_class_name(request: pytest.FixtureRequest) -> str: """Return a class name of a mutable multidict implementation.""" - return request.param + return request.param # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -96,29 +92,29 @@ def any_multidict_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a class object of a mutable multidict implementation.""" - return getattr(multidict_module, any_multidict_class_name) + return getattr(multidict_module, any_multidict_class_name) # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_sensitive_multidict_class( multidict_module: ModuleType, -) -> Type[MutableMultiMapping[str]]: +) -> Type[MultiDict[str]]: """Return a case-sensitive mutable multidict class.""" - return multidict_module.MultiDict + return multidict_module.MultiDict # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_insensitive_multidict_class( multidict_module: ModuleType, -) -> Type[MutableMultiMapping[str]]: +) -> Type[CIMultiDict[str]]: """Return a case-insensitive mutable multidict class.""" - return multidict_module.CIMultiDict + return multidict_module.CIMultiDict # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_insensitive_str_class(multidict_module: ModuleType) -> Type[str]: """Return a case-insensitive string class.""" - return multidict_module.istr + return multidict_module.istr # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -133,7 +129,7 @@ def any_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MultiMapping[str]]: """Return an immutable multidict implementation class object.""" - return getattr(multidict_module, any_multidict_proxy_class_name) + return getattr(multidict_module, any_multidict_proxy_class_name) # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -141,7 +137,7 @@ def case_sensitive_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a case-sensitive immutable multidict class.""" - return multidict_module.MultiDictProxy + return multidict_module.MultiDictProxy # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -149,13 +145,15 @@ def case_insensitive_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a case-insensitive immutable multidict class.""" - return multidict_module.CIMultiDictProxy + return multidict_module.CIMultiDictProxy # type: ignore[no-any-return] @pytest.fixture(scope="session") -def multidict_getversion_callable(multidict_module: ModuleType) -> Callable: +def multidict_getversion_callable( + multidict_module: ModuleType, +) -> Callable[[Union[MultiDict[object], MultiDictProxy[object]]], int]: """Return a ``getversion()`` function for current implementation.""" - return multidict_module.getversion + return multidict_module.getversion # type: ignore[no-any-return] def pytest_addoption( @@ -171,20 +169,12 @@ def pytest_addoption( parser.addoption( "--c-extensions", # disabled with `--no-c-extensions` - action="store_true" if PY_38_AND_BELOW else argparse.BooleanOptionalAction, + action=argparse.BooleanOptionalAction, default=True, dest="c_extensions", help="Test C-extensions (on by default)", ) - if PY_38_AND_BELOW: - parser.addoption( - "--no-c-extensions", - action="store_false", - dest="c_extensions", - help="Skip testing C-extensions (on by default)", - ) - def pytest_collection_modifyitems( session: pytest.Session, @@ -197,8 +187,8 @@ def pytest_collection_modifyitems( if test_c_extensions: return - selected_tests = [] - deselected_tests = [] + selected_tests: list[pytest.Item] = [] + deselected_tests: list[pytest.Item] = [] for item in items: c_ext = item.get_closest_marker(C_EXT_MARK.name) is not None @@ -218,7 +208,7 @@ def pytest_configure(config: pytest.Config) -> None: ) -def pytest_generate_tests(metafunc): +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "pickle_protocol" in metafunc.fixturenames: metafunc.parametrize( "pickle_protocol", list(range(pickle.HIGHEST_PROTOCOL + 1)), scope="session" diff --git a/contrib/python/multidict/tests/gen_pickles.py b/contrib/python/multidict/tests/gen_pickles.py index 4e0d268bed..72f41b7565 100644 --- a/contrib/python/multidict/tests/gen_pickles.py +++ b/contrib/python/multidict/tests/gen_pickles.py @@ -1,18 +1,22 @@ import pickle from importlib import import_module from pathlib import Path +from typing import Union + +from multidict import CIMultiDict, MultiDict TESTS_DIR = Path(__file__).parent.resolve() +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] -def write(tag, cls, proto): +def write(tag: str, cls: _MD_Classes, proto: int) -> None: d = cls([("a", 1), ("a", 2)]) file_basename = f"{cls.__name__.lower()}-{tag}" with (TESTS_DIR / f"{file_basename}.pickle.{proto}").open("wb") as f: pickle.dump(d, f, proto) -def generate(): +def generate() -> None: _impl_map = { "c-extension": "_multidict", "pure-python": "_multidict_py", diff --git a/contrib/python/multidict/tests/test_abc.py b/contrib/python/multidict/tests/test_abc.py index e18ad83f82..611d0fa8c3 100644 --- a/contrib/python/multidict/tests/test_abc.py +++ b/contrib/python/multidict/tests/test_abc.py @@ -1,94 +1,32 @@ from collections.abc import Mapping, MutableMapping -import pytest +from multidict import ( + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) -from multidict import MultiMapping, MutableMultiMapping - -def test_abc_inheritance(): +def test_abc_inheritance() -> None: assert issubclass(MultiMapping, Mapping) assert not issubclass(MultiMapping, MutableMapping) assert issubclass(MutableMultiMapping, Mapping) assert issubclass(MutableMultiMapping, MutableMapping) -class A(MultiMapping): - def __getitem__(self, key): - pass - - def __iter__(self): - pass - - def __len__(self): - pass - - def getall(self, key, default=None): - super().getall(key, default) - - def getone(self, key, default=None): - super().getone(key, default) - - -def test_abc_getall(): - with pytest.raises(KeyError): - A().getall("key") - - -def test_abc_getone(): - with pytest.raises(KeyError): - A().getone("key") - - -class B(A, MutableMultiMapping): - def __setitem__(self, key, value): - pass - - def __delitem__(self, key): - pass - - def add(self, key, value): - super().add(key, value) - - def extend(self, *args, **kwargs): - super().extend(*args, **kwargs) - - def popall(self, key, default=None): - super().popall(key, default) - - def popone(self, key, default=None): - super().popone(key, default) - - -def test_abc_add(): - with pytest.raises(NotImplementedError): - B().add("key", "val") - - -def test_abc_extend(): - with pytest.raises(NotImplementedError): - B().extend() - - -def test_abc_popone(): - with pytest.raises(KeyError): - B().popone("key") - - -def test_abc_popall(): - with pytest.raises(KeyError): - B().popall("key") - - -def test_multidict_inheritance(any_multidict_class): +def test_multidict_inheritance(any_multidict_class: type[MultiDict[str]]) -> None: assert issubclass(any_multidict_class, MultiMapping) assert issubclass(any_multidict_class, MutableMultiMapping) -def test_proxy_inheritance(any_multidict_proxy_class): +def test_proxy_inheritance( + any_multidict_proxy_class: type[MultiDictProxy[str]], +) -> None: assert issubclass(any_multidict_proxy_class, MultiMapping) assert not issubclass(any_multidict_proxy_class, MutableMultiMapping) -def test_generic_type_in_runtime(): +def test_generic_type_in_runtime() -> None: MultiMapping[str] MutableMultiMapping[str] diff --git a/contrib/python/multidict/tests/test_copy.py b/contrib/python/multidict/tests/test_copy.py index cd926cdc1d..deff64db37 100644 --- a/contrib/python/multidict/tests/test_copy.py +++ b/contrib/python/multidict/tests/test_copy.py @@ -1,7 +1,13 @@ import copy +from typing import Union +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy -def test_copy(any_multidict_class): +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] +_MDP_Classes = Union[type[MultiDictProxy[int]], type[CIMultiDictProxy[int]]] + + +def test_copy(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class() d["foo"] = 6 d2 = d.copy() @@ -10,7 +16,9 @@ def test_copy(any_multidict_class): assert d2["foo"] == 7 -def test_copy_proxy(any_multidict_class, any_multidict_proxy_class): +def test_copy_proxy( + any_multidict_class: _MD_Classes, any_multidict_proxy_class: _MDP_Classes +) -> None: d = any_multidict_class() d["foo"] = 6 p = any_multidict_proxy_class(d) @@ -21,7 +29,7 @@ def test_copy_proxy(any_multidict_class, any_multidict_proxy_class): assert d2["foo"] == 7 -def test_copy_std_copy(any_multidict_class): +def test_copy_std_copy(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class() d["foo"] = 6 d2 = copy.copy(d) @@ -30,7 +38,7 @@ def test_copy_std_copy(any_multidict_class): assert d2["foo"] == 7 -def test_ci_multidict_clone(any_multidict_class): +def test_ci_multidict_clone(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class(foo=6) d2 = any_multidict_class(d) d2["foo"] = 7 diff --git a/contrib/python/multidict/tests/test_guard.py b/contrib/python/multidict/tests/test_guard.py index 225da67c8d..c877fbf803 100644 --- a/contrib/python/multidict/tests/test_guard.py +++ b/contrib/python/multidict/tests/test_guard.py @@ -1,12 +1,10 @@ -from typing import Type - import pytest -from multidict import MultiMapping +from multidict import MultiDict def test_guard_items( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.items()) @@ -16,7 +14,7 @@ def test_guard_items( def test_guard_keys( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.keys()) @@ -26,7 +24,7 @@ def test_guard_keys( def test_guard_values( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.values()) diff --git a/contrib/python/multidict/tests/test_istr.py b/contrib/python/multidict/tests/test_istr.py index 1918153532..101f5fe8e5 100644 --- a/contrib/python/multidict/tests/test_istr.py +++ b/contrib/python/multidict/tests/test_istr.py @@ -71,4 +71,4 @@ def test_leak(create_istrs: Callable[[], None]) -> None: gc.collect() cnt2 = len(gc.get_objects()) - assert abs(cnt - cnt2) < 10 # on PyPy these numbers are not equal + assert abs(cnt - cnt2) < 50 # on PyPy these numbers are not equal diff --git a/contrib/python/multidict/tests/test_multidict.py b/contrib/python/multidict/tests/test_multidict.py index bcfa699c15..d144130a41 100644 --- a/contrib/python/multidict/tests/test_multidict.py +++ b/contrib/python/multidict/tests/test_multidict.py @@ -5,27 +5,20 @@ import operator import sys import weakref from collections import deque -from collections.abc import Mapping +from collections.abc import Callable, Iterable, Iterator, KeysView, Mapping from types import ModuleType -from typing import ( - Callable, - Dict, - Iterable, - Iterator, - KeysView, - List, - Mapping, - Set, - Tuple, - Type, - Union, - cast, -) +from typing import Union, cast import pytest import multidict -from multidict import CIMultiDict, MultiDict, MultiMapping, MutableMultiMapping +from multidict import ( + CIMultiDict, + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) def chained_callable( @@ -71,7 +64,7 @@ def cls( # type: ignore[misc] def test_exposed_names(any_multidict_class_name: str) -> None: - assert any_multidict_class_name in multidict.__all__ # type: ignore[attr-defined] + assert any_multidict_class_name in multidict.__all__ @pytest.mark.parametrize( @@ -86,8 +79,8 @@ def test_exposed_names(any_multidict_class_name: str) -> None: indirect=["cls"], ) def test__iter__types( - cls: Type[MultiDict[Union[str, int]]], - key_cls: Type[object], + cls: type[MultiDict[Union[str, int]]], + key_cls: type[str], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) for i in d: @@ -95,26 +88,26 @@ def test__iter__types( def test_proxy_copy( - any_multidict_class: Type[MutableMultiMapping[str]], - any_multidict_proxy_class: Type[MultiMapping[str]], + any_multidict_class: type[MultiDict[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: d1 = any_multidict_class(key="value", a="b") p1 = any_multidict_proxy_class(d1) - d2 = p1.copy() # type: ignore[attr-defined] + d2 = p1.copy() assert d1 == d2 assert d1 is not d2 def test_multidict_subclassing( - any_multidict_class: Type[MutableMultiMapping[str]], + any_multidict_class: type[MultiDict[str]], ) -> None: class DummyMultidict(any_multidict_class): # type: ignore[valid-type,misc] pass def test_multidict_proxy_subclassing( - any_multidict_proxy_class: Type[MultiMapping[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: class DummyMultidictProxy( any_multidict_proxy_class, # type: ignore[valid-type,misc] @@ -123,7 +116,7 @@ def test_multidict_proxy_subclassing( class BaseMultiDictTest: - def test_instantiate__empty(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_instantiate__empty(self, cls: type[MutableMultiMapping[str]]) -> None: d = cls() empty: Mapping[str, str] = {} assert d == empty @@ -133,14 +126,14 @@ class BaseMultiDictTest: assert list(d.items()) == [] assert cls() != list() # type: ignore[comparison-overlap] - with pytest.raises(TypeError, match=r"(2 given)"): + with pytest.raises(TypeError, match=r"3 were given"): cls(("key1", "value1"), ("key2", "value2")) # type: ignore[call-arg] # noqa: E501 @pytest.mark.parametrize("arg0", ([("key", "value1")], {"key": "value1"})) def test_instantiate__from_arg0( self, - cls: Type[MutableMultiMapping[str]], - arg0: Union[List[Tuple[str, str]], Dict[str, str]], + cls: type[MultiDict[str]], + arg0: Union[list[tuple[str, str]], dict[str, str]], ) -> None: d = cls(arg0) @@ -152,7 +145,7 @@ class BaseMultiDictTest: def test_instantiate__with_kwargs( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: d = cls([("key", "value1")], key2="value2") @@ -163,7 +156,7 @@ class BaseMultiDictTest: assert sorted(d.items()) == [("key", "value1"), ("key2", "value2")] def test_instantiate__from_generator( - self, cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]] + self, cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]] ) -> None: d = cls((str(i), i) for i in range(2)) @@ -175,7 +168,7 @@ class BaseMultiDictTest: def test_instantiate__from_list_of_lists( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MutableMultiMapping[str]], ) -> None: # Should work at runtime, but won't type check. d = cls([["key", "value1"]]) # type: ignore[call-arg] @@ -183,7 +176,7 @@ class BaseMultiDictTest: def test_instantiate__from_list_of_custom_pairs( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: class Pair: def __len__(self) -> int: @@ -193,10 +186,10 @@ class BaseMultiDictTest: return ("key", "value1")[pos] # Works at runtime, but won't type check. - d = cls([Pair()]) + d = cls([Pair()]) # type: ignore[list-item] assert d == {"key": "value1"} - def test_getone(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_getone(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert d.getone("key") == "value1" @@ -210,25 +203,42 @@ class BaseMultiDictTest: assert d.getone("key2", "default") == "default" - def test_call_with_kwargs(self, cls: Type[MultiDict[str]]) -> None: + def test_call_with_kwargs(self, cls: type[MultiDict[str]]) -> None: d = cls([("present", "value")]) assert d.getall(default="missing", key="notfound") == "missing" def test__iter__( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) assert list(d) == ["key", "key2", "key"] + def test__contains( + self, + cls: Union[ + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], + ], + ) -> None: + d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) + + assert list(d) == ["key", "key2", "key"] + + assert "key" in d + assert "key2" in d + + assert "foo" not in d + assert 42 not in d # type: ignore[comparison-overlap] + def test_keys__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) @@ -239,12 +249,13 @@ class BaseMultiDictTest: assert "key2" in d.keys() assert "foo" not in d.keys() + assert 42 not in d.keys() # type: ignore[comparison-overlap] def test_values__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key", "two"), ("key", 3)]) @@ -260,8 +271,8 @@ class BaseMultiDictTest: def test_items__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key", "two"), ("key", 3)]) @@ -273,15 +284,17 @@ class BaseMultiDictTest: assert ("key", 3) in d.items() assert ("foo", "bar") not in d.items() + assert (42, 3) not in d.items() # type: ignore[comparison-overlap] + assert 42 not in d.items() # type: ignore[comparison-overlap] def test_cannot_create_from_unaccepted( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MutableMultiMapping[str]], ) -> None: with pytest.raises(TypeError): cls([(1, 2, 3)]) # type: ignore[call-arg] - def test_keys_is_set_less(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_less(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() < {"key", "key2"} @@ -297,8 +310,8 @@ class BaseMultiDictTest: ) def test_keys_is_set_less_equal( self, - cls: Type[MutableMultiMapping[str]], - contents: List[Tuple[str, str]], + cls: type[MultiDict[str]], + contents: list[tuple[str, str]], expected: bool, ) -> None: d = cls(contents) @@ -306,12 +319,17 @@ class BaseMultiDictTest: result = d.keys() <= {"key", "key2"} assert result is expected - def test_keys_is_set_equal(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_equal(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() == {"key"} - def test_keys_is_set_greater(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_items_is_set_equal(self, cls: type[MultiDict[str]]) -> None: + d = cls([("key", "value1")]) + + assert d.items() == {("key", "value1")} + + def test_keys_is_set_greater(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert d.keys() > {"key"} @@ -326,16 +344,14 @@ class BaseMultiDictTest: ), ) def test_keys_is_set_greater_equal( - self, cls: Type[MutableMultiMapping[str]], set_: Set[str], expected: bool + self, cls: type[MultiDict[str]], set_: set[str], expected: bool ) -> None: d = cls([("key", "value1"), ("key2", "value2")]) result = d.keys() >= set_ assert result is expected - def test_keys_less_than_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_less_than_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -348,7 +364,7 @@ class BaseMultiDictTest: assert (d.keys() < RightOperand()) is sentinel_operation_result def test_keys_less_than_or_equal_not_implemented( - self, cls: Type[MutableMultiMapping[str]] + self, cls: type[MultiDict[str]] ) -> None: d = cls([("key", "value1")]) @@ -361,9 +377,7 @@ class BaseMultiDictTest: assert (d.keys() <= RightOperand()) is sentinel_operation_result - def test_keys_greater_than_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_greater_than_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -376,7 +390,7 @@ class BaseMultiDictTest: assert (d.keys() > RightOperand()) is sentinel_operation_result def test_keys_greater_than_or_equal_not_implemented( - self, cls: Type[MutableMultiMapping[str]] + self, cls: type[MultiDict[str]] ) -> None: d = cls([("key", "value1")]) @@ -389,30 +403,28 @@ class BaseMultiDictTest: assert (d.keys() >= RightOperand()) is sentinel_operation_result - def test_keys_is_set_not_equal(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_not_equal(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() != {"key2"} - def test_keys_not_equal_unrelated_type( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_not_equal_unrelated_type(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) - assert d.keys() != "other" + assert d.keys() != "other" # type: ignore[comparison-overlap] - def test_eq(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key": "value1"} == d - def test_eq2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq2(self, cls: type[MultiDict[str]]) -> None: d1 = cls([("key", "value1")]) d2 = cls([("key2", "value1")]) assert d1 != d2 - def test_eq3(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq3(self, cls: type[MultiDict[str]]) -> None: d1 = cls([("key", "value1")]) d2 = cls() @@ -420,7 +432,7 @@ class BaseMultiDictTest: def test_eq_other_mapping_contains_more_keys( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: d1 = cls(foo="bar") d2 = dict(foo="bar", bar="baz") @@ -428,7 +440,7 @@ class BaseMultiDictTest: assert d1 != d2 def test_eq_bad_mapping_len( - self, cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]] + self, cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]] ) -> None: class BadMapping(Mapping[str, int]): def __getitem__(self, key: str) -> int: @@ -437,8 +449,8 @@ class BaseMultiDictTest: def __iter__(self) -> Iterator[str]: yield "a" # pragma: no cover # `len()` fails earlier - def __len__(self) -> int: # type: ignore[return] - 1 / 0 + def __len__(self) -> int: + return 1 // 0 d1 = cls(a=1) d2 = BadMapping() @@ -447,11 +459,11 @@ class BaseMultiDictTest: def test_eq_bad_mapping_getitem( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: class BadMapping(Mapping[str, int]): - def __getitem__(self, key: str) -> int: # type: ignore[return] - 1 / 0 + def __getitem__(self, key: str) -> int: + return 1 // 0 def __iter__(self) -> Iterator[str]: yield "a" # pragma: no cover # foreign objects no iterated @@ -464,24 +476,22 @@ class BaseMultiDictTest: with pytest.raises(ZeroDivisionError): d1 == d2 - def test_ne(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_ne(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d != {"key": "another_value"} - def test_and(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_and(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == d.keys() & {"key", "key2"} - def test_and2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_and2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == {"key", "key2"} & d.keys() - def test_bitwise_and_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_and_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -493,26 +503,22 @@ class BaseMultiDictTest: assert d.keys() & RightOperand() is sentinel_operation_result - def test_bitwise_and_iterable_not_set( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_and_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == d.keys() & ["key", "key2"] - def test_or(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_or(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == d.keys() | {"key2"} - def test_or2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_or2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == {"key2"} | d.keys() - def test_bitwise_or_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_or_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -524,24 +530,22 @@ class BaseMultiDictTest: assert d.keys() | RightOperand() is sentinel_operation_result - def test_bitwise_or_iterable_not_set( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_or_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == d.keys() | ["key2"] - def test_sub(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key"} == d.keys() - {"key2"} - def test_sub2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key3"} == {"key", "key2", "key3"} - d.keys() - def test_sub_not_implemented(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) sentinel_operation_result = object() @@ -553,22 +557,22 @@ class BaseMultiDictTest: assert d.keys() - RightOperand() is sentinel_operation_result - def test_sub_iterable_not_set(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key"} == d.keys() - ["key2"] - def test_xor(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == d.keys() ^ {"key2", "key3"} - def test_xor2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == {"key2", "key3"} ^ d.keys() - def test_xor_not_implemented(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) sentinel_operation_result = object() @@ -580,7 +584,7 @@ class BaseMultiDictTest: assert d.keys() ^ RightOperand() is sentinel_operation_result - def test_xor_iterable_not_set(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == d.keys() ^ ["key2", "key3"] @@ -590,13 +594,13 @@ class BaseMultiDictTest: (("key2", "v", True), ("key", "value1", False)), ) def test_isdisjoint( - self, cls: Type[MutableMultiMapping[str]], key: str, value: str, expected: bool + self, cls: type[MultiDict[str]], key: str, value: str, expected: bool ) -> None: d = cls([("key", "value1")]) assert d.items().isdisjoint({(key, value)}) is expected assert d.keys().isdisjoint({key}) is expected - def test_repr_aiohttp_issue_410(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_repr_aiohttp_issue_410(self, cls: type[MutableMultiMapping[str]]) -> None: d = cls() try: @@ -614,9 +618,9 @@ class BaseMultiDictTest: @pytest.mark.parametrize("other", ({"other"},)) def test_op_issue_aiohttp_issue_410( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], op: Callable[[object, object], object], - other: Set[str], + other: set[str], ) -> None: d = cls([("key", "value")]) @@ -628,7 +632,7 @@ class BaseMultiDictTest: assert sys.exc_info()[1] == e # noqa: PT017 - def test_weakref(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_weakref(self, cls: type[MutableMultiMapping[str]]) -> None: called = False def cb(wr: object) -> None: @@ -644,7 +648,7 @@ class BaseMultiDictTest: def test_iter_length_hint_keys( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.keys()) @@ -652,7 +656,7 @@ class BaseMultiDictTest: def test_iter_length_hint_items( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.items()) @@ -660,15 +664,15 @@ class BaseMultiDictTest: def test_iter_length_hint_values( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.values()) - assert it.__length_hint__() == 2 # type: ignore[attr-defined] + assert it.__length_hint__() == 2 def test_ctor_list_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = [("a", 1)] obj = cls(arg, b=2) @@ -677,7 +681,7 @@ class BaseMultiDictTest: def test_ctor_tuple_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = (("a", 1),) obj = cls(arg, b=2) @@ -686,7 +690,7 @@ class BaseMultiDictTest: def test_ctor_deque_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = deque([("a", 1)]) obj = cls(arg, b=2) @@ -709,7 +713,7 @@ class TestMultiDict(BaseMultiDictTest): """Make a case-sensitive multidict class/proxy constructor.""" return chained_callable(multidict_module, request.param) - def test__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls() _cls = type(d) @@ -719,7 +723,7 @@ class TestMultiDict(BaseMultiDictTest): assert str(d) == "<%s('key': 'one', 'key': 'two')>" % _cls.__name__ - def test_getall(self, cls: Type[MultiDict[str]]) -> None: + def test_getall(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert d != {"key": "value1"} @@ -735,27 +739,27 @@ class TestMultiDict(BaseMultiDictTest): def test_preserve_stable_ordering( self, - cls: Type[MultiDict[Union[str, int]]], + cls: type[MultiDict[Union[str, int]]], ) -> None: d = cls([("a", 1), ("b", "2"), ("a", 3)]) s = "&".join("{}={}".format(k, v) for k, v in d.items()) assert s == "a=1&b=2&a=3" - def test_get(self, cls: Type[MultiDict[int]]) -> None: + def test_get(self, cls: type[MultiDict[int]]) -> None: d = cls([("a", 1), ("a", 2)]) assert d["a"] == 1 - def test_items__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_items__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") expected = "_ItemsView('key': 'value1', 'key': 'value2')" assert repr(d.items()) == expected - def test_keys__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_keys__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert repr(d.keys()) == "_KeysView('key', 'key')" - def test_values__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_values__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert repr(d.values()) == "_ValuesView('value1', 'value2')" @@ -775,7 +779,7 @@ class TestCIMultiDict(BaseMultiDictTest): """Make a case-insensitive multidict class/proxy constructor.""" return chained_callable(multidict_module, request.param) - def test_basics(self, cls: Type[CIMultiDict[str]]) -> None: + def test_basics(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], KEY="value2") assert d.getone("key") == "value1" @@ -789,7 +793,7 @@ class TestCIMultiDict(BaseMultiDictTest): with pytest.raises(KeyError, match="key2"): d.getone("key2") - def test_getall(self, cls: Type[CIMultiDict[str]]) -> None: + def test_getall(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], KEY="value2") assert not d == {"KEY": "value1"} @@ -800,26 +804,26 @@ class TestCIMultiDict(BaseMultiDictTest): with pytest.raises(KeyError, match="some_key"): d.getall("some_key") - def test_get(self, cls: Type[CIMultiDict[int]]) -> None: + def test_get(self, cls: type[CIMultiDict[int]]) -> None: d = cls([("A", 1), ("a", 2)]) assert 1 == d["a"] - def test__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") _cls = type(d) expected = "<%s('KEY': 'value1', 'key': 'value2')>" % _cls.__name__ assert str(d) == expected - def test_items__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_items__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") expected = "_ItemsView('KEY': 'value1', 'key': 'value2')" assert repr(d.items()) == expected - def test_keys__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_keys__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") assert repr(d.keys()) == "_KeysView('KEY', 'key')" - def test_values__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_values__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") assert repr(d.values()) == "_ValuesView('value1', 'value2')" diff --git a/contrib/python/multidict/tests/test_multidict_benchmarks.py b/contrib/python/multidict/tests/test_multidict_benchmarks.py new file mode 100644 index 0000000000..e6a538f3cc --- /dev/null +++ b/contrib/python/multidict/tests/test_multidict_benchmarks.py @@ -0,0 +1,391 @@ +"""codspeed benchmarks for multidict.""" + +from typing import Dict, Union + +from pytest_codspeed import BenchmarkFixture + +from multidict import CIMultiDict, MultiDict, istr + +# Note that this benchmark should not be refactored to use pytest.mark.parametrize +# since each benchmark name should be unique. + +_SENTINEL = object() + + +def test_multidict_insert_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_cimultidict_insert_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_cimultidict_insert_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict() + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_multidict_add_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_cimultidict_add_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_cimultidict_add_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict() + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_multidict_pop_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_cimultidict_pop_str(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_cimultidict_pop_istr(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_multidict_popitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md = md_base.copy() + for _ in range(100): + md.popitem() + + +def test_cimultidict_popitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md = md_base.copy() + for _ in range(100): + md.popitem() + + +def test_multidict_clear_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.clear() + + +def test_cimultidict_clear_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.clear() + + +def test_multidict_update_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_cimultidict_update_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_cimultidict_update_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items: Dict[Union[str, istr], istr] = {istr(i): istr(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_multidict_extend_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_cimultidict_extend_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_cimultidict_extend_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = {istr(i): istr(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_multidict_delitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_cimultidict_delitem_str(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_cimultidict_delitem_istr(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_multidict_getall_str_hit(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict(("all", str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall("all") + + +def test_cimultidict_getall_str_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict(("all", str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall("all") + + +def test_cimultidict_getall_istr_hit(benchmark: BenchmarkFixture) -> None: + all_istr = istr("all") + md: CIMultiDict[istr] = CIMultiDict((all_istr, istr(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall(all_istr) + + +def test_multidict_fetch(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_cimultidict_fetch_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_cimultidict_fetch_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_multidict_get_hit(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_multidict_get_miss(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_miss(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_istr_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_istr_miss(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_hit_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_miss_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_istr_hit_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_istr_with_default_miss( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) diff --git a/contrib/python/multidict/tests/test_mutable_multidict.py b/contrib/python/multidict/tests/test_mutable_multidict.py index 3cacec25af..45f1cdf5f6 100644 --- a/contrib/python/multidict/tests/test_mutable_multidict.py +++ b/contrib/python/multidict/tests/test_mutable_multidict.py @@ -1,16 +1,16 @@ import string import sys -from typing import Type +from typing import Union import pytest -from multidict import MultiMapping, MutableMultiMapping +from multidict import CIMultiDict, CIMultiDictProxy, MultiDictProxy, istr class TestMutableMultiDict: def test_copy( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_sensitive_multidict_class(key="value", a="b") @@ -20,7 +20,7 @@ class TestMutableMultiDict: def test__repr__( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() assert str(d) == "<%s()>" % case_sensitive_multidict_class.__name__ @@ -35,7 +35,7 @@ class TestMutableMultiDict: def test_getall( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "value1")], key="value2") assert len(d) == 2 @@ -50,7 +50,7 @@ class TestMutableMultiDict: def test_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() @@ -73,7 +73,7 @@ class TestMutableMultiDict: def test_extend( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[Union[str, int]]], ) -> None: d = case_sensitive_multidict_class() assert d == {} @@ -101,12 +101,12 @@ class TestMutableMultiDict: assert 6 == len(d) with pytest.raises(TypeError): - d.extend("foo", "bar") + d.extend("foo", "bar") # type: ignore[arg-type, call-arg] def test_extend_from_proxy( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_sensitive_multidict_proxy_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_sensitive_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: d = case_sensitive_multidict_class([("a", "a"), ("b", "b")]) proxy = case_sensitive_multidict_proxy_class(d) @@ -118,7 +118,7 @@ class TestMutableMultiDict: def test_clear( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one")], key="two", foo="bar") @@ -128,7 +128,7 @@ class TestMutableMultiDict: def test_del( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one"), ("key", "two")], foo="bar") assert list(d.keys()) == ["key", "key", "foo"] @@ -142,7 +142,7 @@ class TestMutableMultiDict: def test_set_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one"), ("key", "two")], foo="bar") assert "one" == d.setdefault("key", "three") @@ -152,7 +152,7 @@ class TestMutableMultiDict: def test_popitem( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -163,7 +163,7 @@ class TestMutableMultiDict: def test_popitem_empty_multidict( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() @@ -172,7 +172,7 @@ class TestMutableMultiDict: def test_pop( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -183,7 +183,7 @@ class TestMutableMultiDict: def test_pop2( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -195,7 +195,7 @@ class TestMutableMultiDict: def test_pop_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class(other="val") @@ -204,7 +204,7 @@ class TestMutableMultiDict: def test_pop_raises( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class(other="val") @@ -215,7 +215,7 @@ class TestMutableMultiDict: def test_replacement_order( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key1", "val1") @@ -231,16 +231,16 @@ class TestMutableMultiDict: def test_nonstr_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() with pytest.raises(TypeError): - d[1] = "val" + d[1] = "val" # type: ignore[index] def test_istr_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_sensitive_multidict_class() d[case_insensitive_str_class("1")] = "val" @@ -248,7 +248,7 @@ class TestMutableMultiDict: def test_str_derived_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: class A(str): pass @@ -259,8 +259,8 @@ class TestMutableMultiDict: def test_istr_key_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_sensitive_multidict_class() d.add(case_insensitive_str_class("1"), "val") @@ -268,7 +268,7 @@ class TestMutableMultiDict: def test_str_derived_key_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: class A(str): pass @@ -279,7 +279,7 @@ class TestMutableMultiDict: def test_popall( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key1", "val1") @@ -291,14 +291,14 @@ class TestMutableMultiDict: def test_popall_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() assert "val" == d.popall("key", "val") def test_popall_key_error( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() with pytest.raises(KeyError, match="key"): @@ -306,7 +306,7 @@ class TestMutableMultiDict: def test_large_multidict_resizing( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[int]], ) -> None: SIZE = 1024 d = case_sensitive_multidict_class() @@ -322,7 +322,7 @@ class TestMutableMultiDict: class TestCIMutableMultiDict: def test_getall( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class([("KEY", "value1")], KEY="value2") @@ -336,7 +336,7 @@ class TestCIMutableMultiDict: def test_ctor( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(k1="v1") assert "v1" == d["K1"] @@ -344,7 +344,7 @@ class TestCIMutableMultiDict: def test_setitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d["k1"] = "v1" @@ -353,7 +353,7 @@ class TestCIMutableMultiDict: def test_delitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d["k1"] = "v1" @@ -363,7 +363,7 @@ class TestCIMutableMultiDict: def test_copy( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_insensitive_multidict_class(key="KEY", a="b") @@ -374,7 +374,7 @@ class TestCIMutableMultiDict: def test__repr__( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() assert str(d) == "<%s()>" % case_insensitive_multidict_class.__name__ @@ -389,7 +389,7 @@ class TestCIMutableMultiDict: def test_add( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() @@ -421,7 +421,7 @@ class TestCIMutableMultiDict: def test_extend( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[Union[str, int]]], ) -> None: d = case_insensitive_multidict_class() assert d == {} @@ -450,12 +450,12 @@ class TestCIMutableMultiDict: assert 6 == len(d) with pytest.raises(TypeError): - d.extend("foo", "bar") + d.extend("foo", "bar") # type: ignore[arg-type, call-arg] def test_extend_from_proxy( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_multidict_proxy_class: Type[MultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_multidict_proxy_class: type[CIMultiDictProxy[str]], ) -> None: d = case_insensitive_multidict_class([("a", "a"), ("b", "b")]) proxy = case_insensitive_multidict_proxy_class(d) @@ -467,7 +467,7 @@ class TestCIMutableMultiDict: def test_clear( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class([("KEY", "one")], key="two", foo="bar") @@ -477,7 +477,7 @@ class TestCIMutableMultiDict: def test_del( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class( [("KEY", "one"), ("key", "two")], @@ -493,7 +493,7 @@ class TestCIMutableMultiDict: def test_set_default( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class( [("KEY", "one"), ("key", "two")], @@ -507,7 +507,7 @@ class TestCIMutableMultiDict: def test_popitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -520,7 +520,7 @@ class TestCIMutableMultiDict: def test_popitem_empty_multidict( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() @@ -529,7 +529,7 @@ class TestCIMutableMultiDict: def test_pop( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -540,7 +540,7 @@ class TestCIMutableMultiDict: def test_pop_lowercase( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -551,7 +551,7 @@ class TestCIMutableMultiDict: def test_pop_default( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(OTHER="val") @@ -560,7 +560,7 @@ class TestCIMutableMultiDict: def test_pop_raises( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(OTHER="val") @@ -571,8 +571,8 @@ class TestCIMutableMultiDict: def test_extend_with_istr( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[istr], ) -> None: us = case_insensitive_str_class("aBc") d = case_insensitive_multidict_class() @@ -582,8 +582,8 @@ class TestCIMutableMultiDict: def test_copy_istr( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[istr], ) -> None: d = case_insensitive_multidict_class({case_insensitive_str_class("Foo"): "bar"}) d2 = d.copy() @@ -591,7 +591,7 @@ class TestCIMutableMultiDict: def test_eq( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_insensitive_multidict_class(Key="val") d2 = case_insensitive_multidict_class(KEY="val") @@ -604,7 +604,7 @@ class TestCIMutableMultiDict: ) def test_sizeof( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: md = case_insensitive_multidict_class() s1 = sys.getsizeof(md) @@ -621,14 +621,14 @@ class TestCIMutableMultiDict: ) def test_min_sizeof( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: md = case_insensitive_multidict_class() assert sys.getsizeof(md) < 1024 def test_issue_620_items( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) @@ -639,7 +639,7 @@ class TestCIMutableMultiDict: def test_issue_620_keys( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) @@ -650,7 +650,7 @@ class TestCIMutableMultiDict: def test_issue_620_values( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) diff --git a/contrib/python/multidict/tests/test_pickle.py b/contrib/python/multidict/tests/test_pickle.py index 48adea13f0..3159ea45c6 100644 --- a/contrib/python/multidict/tests/test_pickle.py +++ b/contrib/python/multidict/tests/test_pickle.py @@ -1,13 +1,21 @@ import pickle from pathlib import Path +from typing import TYPE_CHECKING import pytest +from multidict import MultiDict, MultiDictProxy + +if TYPE_CHECKING: + from conftest import MultidictImplementation + import yatest.common as yc here = Path(yc.source_path(__file__)).resolve().parent -def test_pickle(any_multidict_class, pickle_protocol): +def test_pickle( + any_multidict_class: type[MultiDict[int]], pickle_protocol: int +) -> None: d = any_multidict_class([("a", 1), ("a", 2)]) pbytes = pickle.dumps(d, pickle_protocol) obj = pickle.loads(pbytes) @@ -15,14 +23,21 @@ def test_pickle(any_multidict_class, pickle_protocol): assert isinstance(obj, any_multidict_class) -def test_pickle_proxy(any_multidict_class, any_multidict_proxy_class): +def test_pickle_proxy( + any_multidict_class: type[MultiDict[int]], + any_multidict_proxy_class: type[MultiDictProxy[int]], +) -> None: d = any_multidict_class([("a", 1), ("a", 2)]) proxy = any_multidict_proxy_class(d) with pytest.raises(TypeError): pickle.dumps(proxy) -def test_load_from_file(any_multidict_class, multidict_implementation, pickle_protocol): +def test_load_from_file( + any_multidict_class: type[MultiDict[int]], + multidict_implementation: "MultidictImplementation", + pickle_protocol: int, +) -> None: multidict_class_name = any_multidict_class.__name__ pickle_file_basename = "-".join( ( diff --git a/contrib/python/multidict/tests/test_types.py b/contrib/python/multidict/tests/test_types.py index ceaa391e37..6339006b68 100644 --- a/contrib/python/multidict/tests/test_types.py +++ b/contrib/python/multidict/tests/test_types.py @@ -1,52 +1,57 @@ -import sys import types import pytest -def test_proxies(multidict_module): +def test_proxies(multidict_module: types.ModuleType) -> None: assert issubclass( multidict_module.CIMultiDictProxy, multidict_module.MultiDictProxy, ) -def test_dicts(multidict_module): +def test_dicts(multidict_module: types.ModuleType) -> None: assert issubclass(multidict_module.CIMultiDict, multidict_module.MultiDict) -def test_proxy_not_inherited_from_dict(multidict_module): +def test_proxy_not_inherited_from_dict(multidict_module: types.ModuleType) -> None: assert not issubclass(multidict_module.MultiDictProxy, multidict_module.MultiDict) -def test_dict_not_inherited_from_proxy(multidict_module): +def test_dict_not_inherited_from_proxy(multidict_module: types.ModuleType) -> None: assert not issubclass(multidict_module.MultiDict, multidict_module.MultiDictProxy) -def test_multidict_proxy_copy_type(multidict_module): +def test_multidict_proxy_copy_type(multidict_module: types.ModuleType) -> None: d = multidict_module.MultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert isinstance(p.copy(), multidict_module.MultiDict) -def test_cimultidict_proxy_copy_type(multidict_module): +def test_cimultidict_proxy_copy_type(multidict_module: types.ModuleType) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.CIMultiDictProxy(d) assert isinstance(p.copy(), multidict_module.CIMultiDict) -def test_create_multidict_proxy_from_nonmultidict(multidict_module): +def test_create_multidict_proxy_from_nonmultidict( + multidict_module: types.ModuleType, +) -> None: with pytest.raises(TypeError): multidict_module.MultiDictProxy({}) -def test_create_multidict_proxy_from_cimultidict(multidict_module): +def test_create_multidict_proxy_from_cimultidict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert p == d -def test_create_multidict_proxy_from_multidict_proxy_from_mdict(multidict_module): +def test_create_multidict_proxy_from_multidict_proxy_from_mdict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.MultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert p == d @@ -54,7 +59,9 @@ def test_create_multidict_proxy_from_multidict_proxy_from_mdict(multidict_module assert p2 == p -def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci(multidict_module): +def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.CIMultiDictProxy(d) assert p == d @@ -62,7 +69,9 @@ def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci(multidict_modul assert p2 == p -def test_create_cimultidict_proxy_from_nonmultidict(multidict_module): +def test_create_cimultidict_proxy_from_nonmultidict( + multidict_module: types.ModuleType, +) -> None: with pytest.raises( TypeError, match=( @@ -73,7 +82,9 @@ def test_create_cimultidict_proxy_from_nonmultidict(multidict_module): multidict_module.CIMultiDictProxy({}) -def test_create_ci_multidict_proxy_from_multidict(multidict_module): +def test_create_ci_multidict_proxy_from_multidict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.MultiDict(key="val") with pytest.raises( TypeError, @@ -85,20 +96,7 @@ def test_create_ci_multidict_proxy_from_multidict(multidict_module): multidict_module.CIMultiDictProxy(d) -@pytest.mark.skipif( - sys.version_info >= (3, 9), reason="Python 3.9 uses GenericAlias which is different" -) -def test_generic_exists(multidict_module) -> None: - assert multidict_module.MultiDict[int] is multidict_module.MultiDict - assert multidict_module.MultiDictProxy[int] is multidict_module.MultiDictProxy - assert multidict_module.CIMultiDict[int] is multidict_module.CIMultiDict - assert multidict_module.CIMultiDictProxy[int] is multidict_module.CIMultiDictProxy - - -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="Python 3.9 is required for GenericAlias" -) -def test_generic_alias(multidict_module) -> None: +def test_generic_alias(multidict_module: types.ModuleType) -> None: assert multidict_module.MultiDict[int] == types.GenericAlias( multidict_module.MultiDict, (int,) ) diff --git a/contrib/python/multidict/tests/test_update.py b/contrib/python/multidict/tests/test_update.py index f455327857..46ab30a08b 100644 --- a/contrib/python/multidict/tests/test_update.py +++ b/contrib/python/multidict/tests/test_update.py @@ -1,10 +1,12 @@ from collections import deque -from typing import Type +from typing import Union -from multidict import MultiMapping +from multidict import CIMultiDict, MultiDict +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] -def test_update_replace(any_multidict_class: Type[MultiMapping[str]]) -> None: + +def test_update_replace(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4), ("b", 5), ("a", 6)]) obj1.update(obj2) @@ -12,7 +14,7 @@ def test_update_replace(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_append(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_append(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4), ("a", 5), ("a", 6)]) obj1.update(obj2) @@ -20,7 +22,7 @@ def test_update_append(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_remove(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_remove(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4)]) obj1.update(obj2) @@ -28,7 +30,7 @@ def test_update_remove(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_replace_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_replace_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4), ("b", 5), ("a", 6)] obj1.update(obj2) @@ -36,14 +38,14 @@ def test_update_replace_seq(any_multidict_class: Type[MultiMapping[str]]) -> Non assert list(obj1.items()) == expected -def test_update_replace_seq2(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_replace_seq2(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj1.update([("a", 4)], b=5, a=6) expected = [("a", 4), ("b", 5), ("a", 6), ("c", 10)] assert list(obj1.items()) == expected -def test_update_append_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_append_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4), ("a", 5), ("a", 6)] obj1.update(obj2) @@ -51,7 +53,7 @@ def test_update_append_seq(any_multidict_class: Type[MultiMapping[str]]) -> None assert list(obj1.items()) == expected -def test_update_remove_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_remove_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4)] obj1.update(obj2) @@ -59,9 +61,7 @@ def test_update_remove_seq(any_multidict_class: Type[MultiMapping[str]]) -> None assert list(obj1.items()) == expected -def test_update_md( - case_sensitive_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_md(case_sensitive_multidict_class: type[CIMultiDict[str]]) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") d.add("key", "val2") @@ -73,8 +73,8 @@ def test_update_md( def test_update_istr_ci_md( - case_insensitive_multidict_class: Type[MultiMapping[str]], - case_insensitive_str_class: str, + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_insensitive_multidict_class() d.add(case_insensitive_str_class("KEY"), "val1") @@ -86,9 +86,7 @@ def test_update_istr_ci_md( assert [("key", "val"), ("key2", "val3")] == list(d.items()) -def test_update_ci_md( - case_insensitive_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_ci_md(case_insensitive_multidict_class: type[CIMultiDict[str]]) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") d.add("key", "val2") @@ -99,9 +97,7 @@ def test_update_ci_md( assert [("Key", "val"), ("key2", "val3")] == list(d.items()) -def test_update_list_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_list_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = [("a", 1)] obj.update(arg, b=2) @@ -109,9 +105,7 @@ def test_update_list_arg_and_kwds( assert arg == [("a", 1)] -def test_update_tuple_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_tuple_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = (("a", 1),) obj.update(arg, b=2) @@ -119,9 +113,7 @@ def test_update_tuple_arg_and_kwds( assert arg == (("a", 1),) -def test_update_deque_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_deque_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = deque([("a", 1)]) obj.update(arg, b=2) diff --git a/contrib/python/multidict/tests/test_version.py b/contrib/python/multidict/tests/test_version.py index e004afa112..4fe209c678 100644 --- a/contrib/python/multidict/tests/test_version.py +++ b/contrib/python/multidict/tests/test_version.py @@ -1,18 +1,25 @@ -from typing import Callable, Type +from collections.abc import Callable +from typing import TypeVar, Union import pytest -from multidict import MultiMapping +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +_T = TypeVar("_T") +_MD_Types = Union[ + MultiDict[_T], CIMultiDict[_T], MultiDictProxy[_T], CIMultiDictProxy[_T] +] +GetVersion = Callable[[_MD_Types[_T]], int] -def test_getversion_bad_param(multidict_getversion_callable): + +def test_getversion_bad_param(multidict_getversion_callable: GetVersion[str]) -> None: with pytest.raises(TypeError): - multidict_getversion_callable(1) + multidict_getversion_callable(1) # type: ignore[arg-type] def test_ctor( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m1 = any_multidict_class() v1 = multidict_getversion_callable(m1) @@ -22,8 +29,8 @@ def test_ctor( def test_add( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() v = multidict_getversion_callable(m) @@ -32,8 +39,8 @@ def test_add( def test_delitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -43,8 +50,8 @@ def test_delitem( def test_delitem_not_found( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -55,8 +62,8 @@ def test_delitem_not_found( def test_setitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -66,8 +73,8 @@ def test_setitem( def test_setitem_not_found( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -77,8 +84,8 @@ def test_setitem_not_found( def test_clear( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -88,8 +95,8 @@ def test_clear( def test_setdefault( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -99,8 +106,8 @@ def test_setdefault( def test_popone( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -110,8 +117,8 @@ def test_popone( def test_popone_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -121,8 +128,8 @@ def test_popone_default( def test_popone_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -133,8 +140,8 @@ def test_popone_key_error( def test_pop( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -144,8 +151,8 @@ def test_pop( def test_pop_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -155,8 +162,8 @@ def test_pop_default( def test_pop_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -167,8 +174,8 @@ def test_pop_key_error( def test_popall( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -178,8 +185,8 @@ def test_popall( def test_popall_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -189,8 +196,8 @@ def test_popall_default( def test_popall_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -201,8 +208,8 @@ def test_popall_key_error( def test_popitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -212,8 +219,8 @@ def test_popitem( def test_popitem_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() v = multidict_getversion_callable(m) diff --git a/contrib/python/multidict/ya.make b/contrib/python/multidict/ya.make index 8a2950eae9..626036249b 100644 --- a/contrib/python/multidict/ya.make +++ b/contrib/python/multidict/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.1.0) +VERSION(6.2.0) LICENSE(Apache-2.0) @@ -25,7 +25,6 @@ PY_REGISTER( PY_SRCS( TOP_LEVEL multidict/__init__.py - multidict/__init__.pyi multidict/_abc.py multidict/_compat.py multidict/_multidict_base.py diff --git a/contrib/python/pyparsing/py3/.dist-info/METADATA b/contrib/python/pyparsing/py3/.dist-info/METADATA index 6b5fbefef6..ed52278486 100644 --- a/contrib/python/pyparsing/py3/.dist-info/METADATA +++ b/contrib/python/pyparsing/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pyparsing -Version: 3.2.1 +Version: 3.2.2 Summary: pyparsing module - Classes and methods to define and execute parsing grammars Author-email: Paul McGuire <ptmcg.gm+pyparsing@gmail.com> Requires-Python: >=3.9 @@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy @@ -56,7 +57,7 @@ Here is a program to parse ``"Hello, World!"`` (or any greeting of the form from pyparsing import Word, alphas greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" - print(hello, "->", greet.parseString(hello)) + print(hello, "->", greet.parse_string(hello)) The program outputs the following:: @@ -66,7 +67,7 @@ The Python representation of the grammar is quite readable, owing to the self-explanatory class names, and the use of '+', '|' and '^' operator definitions. -The parsed results returned from ``parseString()`` is a collection of type +The parsed results returned from ``parse_string()`` is a collection of type ``ParseResults``, which can be accessed as a nested list, a dictionary, or an object with named attributes. diff --git a/contrib/python/pyparsing/py3/README.rst b/contrib/python/pyparsing/py3/README.rst index 24d603c7bc..cfb9889f85 100644 --- a/contrib/python/pyparsing/py3/README.rst +++ b/contrib/python/pyparsing/py3/README.rst @@ -26,7 +26,7 @@ Here is a program to parse ``"Hello, World!"`` (or any greeting of the form from pyparsing import Word, alphas greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" - print(hello, "->", greet.parseString(hello)) + print(hello, "->", greet.parse_string(hello)) The program outputs the following:: @@ -36,7 +36,7 @@ The Python representation of the grammar is quite readable, owing to the self-explanatory class names, and the use of '+', '|' and '^' operator definitions. -The parsed results returned from ``parseString()`` is a collection of type +The parsed results returned from ``parse_string()`` is a collection of type ``ParseResults``, which can be accessed as a nested list, a dictionary, or an object with named attributes. diff --git a/contrib/python/pyparsing/py3/pyparsing/__init__.py b/contrib/python/pyparsing/py3/pyparsing/__init__.py index 726c76cb24..fa1f2abe67 100644 --- a/contrib/python/pyparsing/py3/pyparsing/__init__.py +++ b/contrib/python/pyparsing/py3/pyparsing/__init__.py @@ -120,8 +120,8 @@ class version_info(NamedTuple): return f"{__name__}.{type(self).__name__}({', '.join('{}={!r}'.format(*nv) for nv in zip(self._fields, self))})" -__version_info__ = version_info(3, 2, 1, "final", 1) -__version_time__ = "31 Dec 2024 20:41 UTC" +__version_info__ = version_info(3, 2, 2, "final", 1) +__version_time__ = "22 Mar 2025 22:09 UTC" __version__ = __version_info__.__version__ __versionTime__ = __version_time__ __author__ = "Paul McGuire <ptmcg.gm+pyparsing@gmail.com>" diff --git a/contrib/python/pyparsing/py3/pyparsing/actions.py b/contrib/python/pyparsing/py3/pyparsing/actions.py index f491aab986..0153cc7132 100644 --- a/contrib/python/pyparsing/py3/pyparsing/actions.py +++ b/contrib/python/pyparsing/py3/pyparsing/actions.py @@ -22,7 +22,7 @@ class OnlyOnce: Note: parse action signature must include all 3 arguments. """ - def __init__(self, method_call: Callable[[str, int, ParseResults], Any]): + def __init__(self, method_call: Callable[[str, int, ParseResults], Any]) -> None: from .core import _trim_arity self.callable = _trim_arity(method_call) diff --git a/contrib/python/pyparsing/py3/pyparsing/core.py b/contrib/python/pyparsing/py3/pyparsing/core.py index b884e2d4a4..86be949ad4 100644 --- a/contrib/python/pyparsing/py3/pyparsing/core.py +++ b/contrib/python/pyparsing/py3/pyparsing/core.py @@ -38,7 +38,6 @@ from .util import ( __config_flags, _collapse_string_to_ranges, _escape_regex_range_chars, - _bslash, _flatten, LRUMemo as _LRUMemo, UnboundedMemo as _UnboundedMemo, @@ -246,7 +245,7 @@ class _ParseActionIndexError(Exception): ParserElement parseImpl methods. """ - def __init__(self, msg: str, exc: BaseException): + def __init__(self, msg: str, exc: BaseException) -> None: self.msg: str = msg self.exc: BaseException = exc @@ -355,7 +354,7 @@ def _default_start_debug_action( ( f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{col(loc, instring)})\n" f" {line(loc, instring)}\n" - f" {' ' * (col(loc, instring) - 1)}^" + f" {'^':>{col(loc, instring)}}" ) ) @@ -454,7 +453,7 @@ class ParserElement(ABC): debug_match: typing.Optional[DebugSuccessAction] debug_fail: typing.Optional[DebugExceptionAction] - def __init__(self, savelist: bool = False): + def __init__(self, savelist: bool = False) -> None: self.parseAction: list[ParseAction] = list() self.failAction: typing.Optional[ParseFailAction] = None self.customName: str = None # type: ignore[assignment] @@ -465,7 +464,7 @@ class ParserElement(ABC): self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True # used when checking for left-recursion - self.mayReturnEmpty = False + self._may_return_empty = False self.keepTabs = False self.ignoreExprs: list[ParserElement] = list() self.debug = False @@ -483,6 +482,14 @@ class ParserElement(ABC): self.suppress_warnings_: list[Diagnostics] = [] self.show_in_diagram = True + @property + def mayReturnEmpty(self): + return self._may_return_empty + + @mayReturnEmpty.setter + def mayReturnEmpty(self, value): + self._may_return_empty = value + def suppress_warning(self, warning_type: Diagnostics) -> ParserElement: """ Suppress warnings emitted for a particular diagnostic on this expression. @@ -2264,6 +2271,7 @@ class ParserElement(ABC): show_results_names: bool = False, show_groups: bool = False, embed: bool = False, + show_hidden: bool = False, **kwargs, ) -> None: """ @@ -2278,6 +2286,7 @@ class ParserElement(ABC): - ``show_results_names`` - bool flag whether diagram should show annotations for defined results names - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``show_hidden`` - bool flag to show diagram elements for internal elements that are usually hidden - ``embed`` - bool flag whether generated HTML should omit <HEAD>, <BODY>, and <DOCTYPE> tags to embed the resulting HTML in an enclosing HTML source - ``head`` - str containing additional HTML to insert into the <HEAD> section of the generated code; @@ -2303,6 +2312,7 @@ class ParserElement(ABC): vertical=vertical, show_results_names=show_results_names, show_groups=show_groups, + show_hidden=show_hidden, diagram_kwargs=kwargs, ) if not isinstance(output_html, (str, Path)): @@ -2352,7 +2362,7 @@ class ParserElement(ABC): class _PendingSkip(ParserElement): # internal placeholder class to hold a place were '...' is added to a parser element, # once another ParserElement is added, this placeholder will be replaced with a SkipTo - def __init__(self, expr: ParserElement, must_skip: bool = False): + def __init__(self, expr: ParserElement, must_skip: bool = False) -> None: super().__init__() self.anchor = expr self.must_skip = must_skip @@ -2395,7 +2405,7 @@ class Token(ParserElement): matching patterns. """ - def __init__(self): + def __init__(self) -> None: super().__init__(savelist=False) def _generateDefaultName(self) -> str: @@ -2407,9 +2417,9 @@ class NoMatch(Token): A token that will never match. """ - def __init__(self): + def __init__(self) -> None: super().__init__() - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False self.errmsg = "Unmatchable token" @@ -2449,14 +2459,14 @@ class Literal(Token): def __getnewargs__(self): return (self.match,) - def __init__(self, match_string: str = "", *, matchString: str = ""): + def __init__(self, match_string: str = "", *, matchString: str = "") -> None: super().__init__() match_string = matchString or match_string self.match = match_string self.matchLen = len(match_string) self.firstMatchChar = match_string[:1] self.errmsg = f"Expected {self.name}" - self.mayReturnEmpty = False + self._may_return_empty = False self.mayIndexError = False def _generateDefaultName(self) -> str: @@ -2475,9 +2485,9 @@ class Empty(Literal): An empty token, will always match. """ - def __init__(self, match_string="", *, matchString=""): + def __init__(self, match_string="", *, matchString="") -> None: super().__init__("") - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False def _generateDefaultName(self) -> str: @@ -2534,7 +2544,7 @@ class Keyword(Token): *, matchString: str = "", identChars: typing.Optional[str] = None, - ): + ) -> None: super().__init__() identChars = identChars or ident_chars if identChars is None: @@ -2546,7 +2556,7 @@ class Keyword(Token): if not self.firstMatchChar: raise ValueError("null string passed to Keyword; use Empty() instead") self.errmsg = f"Expected {type(self).__name__} {self.name}" - self.mayReturnEmpty = False + self._may_return_empty = False self.mayIndexError = False self.caseless = caseless if caseless: @@ -2628,7 +2638,7 @@ class CaselessLiteral(Literal): (Contrast with example for :class:`CaselessKeyword`.) """ - def __init__(self, match_string: str = "", *, matchString: str = ""): + def __init__(self, match_string: str = "", *, matchString: str = "") -> None: match_string = matchString or match_string super().__init__(match_string.upper()) # Preserve the defining literal. @@ -2660,7 +2670,7 @@ class CaselessKeyword(Keyword): *, matchString: str = "", identChars: typing.Optional[str] = None, - ): + ) -> None: identChars = identChars or ident_chars match_string = matchString or match_string super().__init__(match_string, identChars, caseless=True) @@ -2708,7 +2718,7 @@ class CloseMatch(Token): *, maxMismatches: int = 1, caseless=False, - ): + ) -> None: maxMismatches = max_mismatches if max_mismatches is not None else maxMismatches super().__init__() self.match_string = match_string @@ -2716,7 +2726,7 @@ class CloseMatch(Token): self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)" self.caseless = caseless self.mayIndexError = False - self.mayReturnEmpty = False + self._may_return_empty = False def _generateDefaultName(self) -> str: return f"{type(self).__name__}:{self.match_string!r}" @@ -2834,7 +2844,7 @@ class Word(Token): bodyChars: typing.Optional[str] = None, asKeyword: bool = False, excludeChars: typing.Optional[str] = None, - ): + ) -> None: initChars = initChars or init_chars bodyChars = bodyChars or body_chars asKeyword = asKeyword or as_keyword @@ -3018,7 +3028,7 @@ class Char(Word): *, asKeyword: bool = False, excludeChars: typing.Optional[str] = None, - ): + ) -> None: asKeyword = asKeyword or as_keyword excludeChars = excludeChars or exclude_chars super().__init__( @@ -3060,7 +3070,7 @@ class Regex(Token): *, asGroupList: bool = False, asMatch: bool = False, - ): + ) -> None: """The parameters ``pattern`` and ``flags`` are passed to the ``re.compile()`` function as-is. See the Python `re module <https://docs.python.org/3/library/re.html>`_ module for an @@ -3075,15 +3085,18 @@ class Regex(Token): raise ValueError("null string passed to Regex; use Empty() instead") self._re = None + self._may_return_empty = None # type: ignore [assignment] self.reString = self.pattern = pattern elif hasattr(pattern, "pattern") and hasattr(pattern, "match"): self._re = pattern + self._may_return_empty = None # type: ignore [assignment] self.pattern = self.reString = pattern.pattern elif callable(pattern): # defer creating this pattern until we really need it self.pattern = pattern + self._may_return_empty = None # type: ignore [assignment] self._re = None else: @@ -3120,23 +3133,38 @@ class Regex(Token): try: self._re = re.compile(self.pattern, self.flags) - return self._re except re.error: raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex") + else: + self._may_return_empty = self.re.match("", pos=0) is not None + return self._re @cached_property def re_match(self) -> Callable[[str, int], Any]: return self.re.match - @cached_property - def mayReturnEmpty(self) -> bool: # type: ignore[override] - return self.re_match("", 0) is not None + @property + def mayReturnEmpty(self): + if self._may_return_empty is None: + # force compile of regex pattern, to set may_return_empty flag + self.re # noqa + return self._may_return_empty + + @mayReturnEmpty.setter + def mayReturnEmpty(self, value): + self._may_return_empty = value def _generateDefaultName(self) -> str: unescaped = repr(self.pattern).replace("\\\\", "\\") return f"Re:({unescaped})" def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # explicit check for matching past the length of the string; + # this is done because the re module will not complain about + # a match with `pos > len(instring)`, it will just return "" + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -3151,6 +3179,9 @@ class Regex(Token): return loc, ret def parseImplAsGroupList(self, instring, loc, do_actions=True): + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -3160,6 +3191,9 @@ class Regex(Token): return loc, ret def parseImplAsMatch(self, instring, loc, do_actions=True): + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -3258,7 +3292,7 @@ class QuotedString(Token): unquoteResults: bool = True, endQuoteChar: typing.Optional[str] = None, convertWhitespaceEscapes: bool = True, - ): + ) -> None: super().__init__() esc_char = escChar or esc_char esc_quote = escQuote or esc_quote @@ -3362,7 +3396,7 @@ class QuotedString(Token): self.errmsg = f"Expected {self.name}" self.mayIndexError = False - self.mayReturnEmpty = True + self._may_return_empty = True def _generateDefaultName(self) -> str: if self.quote_char == self.end_quote_char and isinstance( @@ -3465,7 +3499,7 @@ class CharsNotIn(Token): exact: int = 0, *, notChars: str = "", - ): + ) -> None: super().__init__() self.skipWhitespace = False self.notChars = not_chars or notChars @@ -3489,7 +3523,7 @@ class CharsNotIn(Token): self.minLen = exact self.errmsg = f"Expected {self.name}" - self.mayReturnEmpty = self.minLen == 0 + self._may_return_empty = self.minLen == 0 self.mayIndexError = False def _generateDefaultName(self) -> str: @@ -3552,7 +3586,9 @@ class White(Token): "\u3000": "<IDEOGRAPHIC_SPACE>", } - def __init__(self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0): + def __init__( + self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0 + ) -> None: super().__init__() self.matchWhite = ws self.set_whitespace_chars( @@ -3560,7 +3596,7 @@ class White(Token): copy_defaults=True, ) # self.leave_whitespace() - self.mayReturnEmpty = True + self._may_return_empty = True self.errmsg = f"Expected {self.name}" self.minLen = min @@ -3594,9 +3630,9 @@ class White(Token): class PositionToken(Token): - def __init__(self): + def __init__(self) -> None: super().__init__() - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False @@ -3605,7 +3641,7 @@ class GoToColumn(PositionToken): tabular report scraping. """ - def __init__(self, colno: int): + def __init__(self, colno: int) -> None: super().__init__() self.col = colno @@ -3657,7 +3693,7 @@ class LineStart(PositionToken): """ - def __init__(self): + def __init__(self) -> None: super().__init__() self.leave_whitespace() self.orig_whiteChars = set() | self.whiteChars @@ -3688,7 +3724,7 @@ class LineEnd(PositionToken): parse string """ - def __init__(self): + def __init__(self) -> None: super().__init__() self.whiteChars.discard("\n") self.set_whitespace_chars(self.whiteChars, copy_defaults=False) @@ -3711,7 +3747,7 @@ class StringStart(PositionToken): string """ - def __init__(self): + def __init__(self) -> None: super().__init__() self.set_name("start of text") @@ -3728,7 +3764,7 @@ class StringEnd(PositionToken): Matches if current position is at the end of the parse string """ - def __init__(self): + def __init__(self) -> None: super().__init__() self.set_name("end of text") @@ -3753,7 +3789,9 @@ class WordStart(PositionToken): a line. """ - def __init__(self, word_chars: str = printables, *, wordChars: str = printables): + def __init__( + self, word_chars: str = printables, *, wordChars: str = printables + ) -> None: wordChars = word_chars if wordChars == printables else wordChars super().__init__() self.wordChars = set(wordChars) @@ -3778,7 +3816,9 @@ class WordEnd(PositionToken): of a line. """ - def __init__(self, word_chars: str = printables, *, wordChars: str = printables): + def __init__( + self, word_chars: str = printables, *, wordChars: str = printables + ) -> None: wordChars = word_chars if wordChars == printables else wordChars super().__init__() self.wordChars = set(wordChars) @@ -3822,14 +3862,15 @@ class Tag(Token): - enthusiastic: True """ - def __init__(self, tag_name: str, value: Any = True): + def __init__(self, tag_name: str, value: Any = True) -> None: super().__init__() - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False self.leave_whitespace() self.tag_name = tag_name self.tag_value = value self.add_parse_action(self._add_tag) + self.show_in_diagram = False def _add_tag(self, tokens: ParseResults): tokens[self.tag_name] = self.tag_value @@ -3843,7 +3884,9 @@ class ParseExpression(ParserElement): post-processing parsed tokens. """ - def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False): + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: super().__init__(savelist) self.exprs: list[ParserElement] if isinstance(exprs, _generatorType): @@ -3939,7 +3982,7 @@ class ParseExpression(ParserElement): ): self.exprs = other.exprs[:] + [self.exprs[1]] self._defaultName = None - self.mayReturnEmpty |= other.mayReturnEmpty + self._may_return_empty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError other = self.exprs[-1] @@ -3951,7 +3994,7 @@ class ParseExpression(ParserElement): ): self.exprs = self.exprs[:-1] + other.exprs[:] self._defaultName = None - self.mayReturnEmpty |= other.mayReturnEmpty + self._may_return_empty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError self.errmsg = f"Expected {self}" @@ -4028,7 +4071,7 @@ class And(ParseExpression): """ class _ErrorStop(Empty): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.leave_whitespace() @@ -4036,28 +4079,34 @@ class And(ParseExpression): return "-" def __init__( - self, exprs_arg: typing.Iterable[ParserElement], savelist: bool = True - ): - exprs: list[ParserElement] = list(exprs_arg) - if exprs and Ellipsis in exprs: - tmp: list[ParserElement] = [] - for i, expr in enumerate(exprs): - if expr is not Ellipsis: - tmp.append(expr) - continue + self, + exprs_arg: typing.Iterable[Union[ParserElement, str]], + savelist: bool = True, + ) -> None: + # instantiate exprs as a list, converting strs to ParserElements + exprs: list[ParserElement] = [ + self._literalStringClass(e) if isinstance(e, str) else e for e in exprs_arg + ] - if i < len(exprs) - 1: - skipto_arg: ParserElement = typing.cast( - ParseExpression, (Empty() + exprs[i + 1]) - ).exprs[-1] - tmp.append(SkipTo(skipto_arg)("_skipped*")) - continue + # convert any Ellipsis elements to SkipTo + if Ellipsis in exprs: + # Ellipsis cannot be the last element + if exprs[-1] is Ellipsis: raise Exception("cannot construct And with sequence ending in ...") - exprs[:] = tmp + + tmp: list[ParserElement] = [] + for cur_expr, next_expr in zip(exprs, exprs[1:]): + if cur_expr is Ellipsis: + tmp.append(SkipTo(next_expr)("_skipped*")) + else: + tmp.append(cur_expr) + + exprs[:-1] = tmp + super().__init__(exprs, savelist) if self.exprs: - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) if not isinstance(self.exprs[0], White): self.set_whitespace_chars( self.exprs[0].whiteChars, @@ -4067,7 +4116,7 @@ class And(ParseExpression): else: self.skipWhitespace = False else: - self.mayReturnEmpty = True + self._may_return_empty = True self.callPreparse = True def streamline(self) -> ParserElement: @@ -4117,7 +4166,7 @@ class And(ParseExpression): break cur = typing.cast(ParserElement, next_first) - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) return self def parseImpl(self, instring, loc, do_actions=True): @@ -4189,18 +4238,20 @@ class Or(ParseExpression): [['123'], ['3.1416'], ['789']] """ - def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False): + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: super().__init__(exprs, savelist) if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = all(e.skipWhitespace for e in self.exprs) else: - self.mayReturnEmpty = True + self._may_return_empty = True def streamline(self) -> ParserElement: super().streamline() if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) self.saveAsList = any(e.saveAsList for e in self.exprs) self.skipWhitespace = all( e.skipWhitespace and not isinstance(e, White) for e in self.exprs @@ -4286,7 +4337,8 @@ class Or(ParseExpression): if maxException is not None: # infer from this check that all alternatives failed at the current position # so emit this collective error message instead of any single error message - if maxExcLoc == loc: + parse_start_loc = self.preParse(instring, loc) + if maxExcLoc == parse_start_loc: maxException.msg = self.errmsg or "" raise maxException @@ -4344,13 +4396,15 @@ class MatchFirst(ParseExpression): print(number.search_string("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] """ - def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False): + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: super().__init__(exprs, savelist) if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = all(e.skipWhitespace for e in self.exprs) else: - self.mayReturnEmpty = True + self._may_return_empty = True def streamline(self) -> ParserElement: if self.streamlined: @@ -4359,13 +4413,13 @@ class MatchFirst(ParseExpression): super().streamline() if self.exprs: self.saveAsList = any(e.saveAsList for e in self.exprs) - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = all( e.skipWhitespace and not isinstance(e, White) for e in self.exprs ) else: self.saveAsList = False - self.mayReturnEmpty = True + self._may_return_empty = True return self def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: @@ -4393,7 +4447,8 @@ class MatchFirst(ParseExpression): if maxException is not None: # infer from this check that all alternatives failed at the current position # so emit this collective error message instead of any individual error message - if maxExcLoc == loc: + parse_start_loc = self.preParse(instring, loc) + if maxExcLoc == parse_start_loc: maxException.msg = self.errmsg or "" raise maxException @@ -4491,12 +4546,14 @@ class Each(ParseExpression): - size: 20 """ - def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = True): + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = True + ) -> None: super().__init__(exprs, savelist) if self.exprs: - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) else: - self.mayReturnEmpty = True + self._may_return_empty = True self.skipWhitespace = True self.initExprGroups = True self.saveAsList = True @@ -4511,9 +4568,9 @@ class Each(ParseExpression): def streamline(self) -> ParserElement: super().streamline() if self.exprs: - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) else: - self.mayReturnEmpty = True + self._may_return_empty = True return self def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: @@ -4612,7 +4669,7 @@ class ParseElementEnhance(ParserElement): post-processing parsed tokens. """ - def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): + def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None: super().__init__(savelist) if isinstance(expr, str_type): expr_str = typing.cast(str, expr) @@ -4626,7 +4683,7 @@ class ParseElementEnhance(ParserElement): self.expr = expr if expr is not None: self.mayIndexError = expr.mayIndexError - self.mayReturnEmpty = expr.mayReturnEmpty + self._may_return_empty = expr.mayReturnEmpty self.set_whitespace_chars( expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars ) @@ -4724,20 +4781,20 @@ class IndentedBlock(ParseElementEnhance): """ class _Indent(Empty): - def __init__(self, ref_col: int): + def __init__(self, ref_col: int) -> None: super().__init__() self.errmsg = f"expected indent at column {ref_col}" self.add_condition(lambda s, l, t: col(l, s) == ref_col) class _IndentGreater(Empty): - def __init__(self, ref_col: int): + def __init__(self, ref_col: int) -> None: super().__init__() self.errmsg = f"expected indent at column greater than {ref_col}" self.add_condition(lambda s, l, t: col(l, s) > ref_col) def __init__( self, expr: ParserElement, *, recursive: bool = False, grouped: bool = True - ): + ) -> None: super().__init__(expr, savelist=True) # if recursive: # raise NotImplementedError("IndentedBlock with recursive is not implemented") @@ -4792,7 +4849,7 @@ class AtStringStart(ParseElementEnhance): # raises ParseException """ - def __init__(self, expr: Union[ParserElement, str]): + def __init__(self, expr: Union[ParserElement, str]) -> None: super().__init__(expr) self.callPreparse = False @@ -4825,7 +4882,7 @@ class AtLineStart(ParseElementEnhance): """ - def __init__(self, expr: Union[ParserElement, str]): + def __init__(self, expr: Union[ParserElement, str]) -> None: super().__init__(expr) self.callPreparse = False @@ -4858,9 +4915,9 @@ class FollowedBy(ParseElementEnhance): [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ - def __init__(self, expr: Union[ParserElement, str]): + def __init__(self, expr: Union[ParserElement, str]) -> None: super().__init__(expr) - self.mayReturnEmpty = True + self._may_return_empty = True def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: # by using self._expr.parse and deleting the contents of the returned ParseResults list @@ -4901,10 +4958,10 @@ class PrecededBy(ParseElementEnhance): """ - def __init__(self, expr: Union[ParserElement, str], retreat: int = 0): + def __init__(self, expr: Union[ParserElement, str], retreat: int = 0) -> None: super().__init__(expr) self.expr = self.expr().leave_whitespace() - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False self.exact = False if isinstance(expr, str_type): @@ -5019,13 +5076,13 @@ class NotAny(ParseElementEnhance): integer = Word(nums) + ~Char(".") """ - def __init__(self, expr: Union[ParserElement, str]): + def __init__(self, expr: Union[ParserElement, str]) -> None: super().__init__(expr) # do NOT use self.leave_whitespace(), don't want to propagate to exprs # self.leave_whitespace() self.skipWhitespace = False - self.mayReturnEmpty = True + self._may_return_empty = True self.errmsg = f"Found unwanted token, {self.expr}" def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: @@ -5044,7 +5101,7 @@ class _MultipleMatch(ParseElementEnhance): stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, - ): + ) -> None: super().__init__(expr) stopOn = stopOn or stop_on self.saveAsList = True @@ -5062,9 +5119,10 @@ class _MultipleMatch(ParseElementEnhance): def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: self_expr_parse = self.expr._parse self_skip_ignorables = self._skipIgnorables - check_ender = self.not_ender is not None - if check_ender: + check_ender = False + if self.not_ender is not None: try_not_ender = self.not_ender.try_parse + check_ender = True # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) @@ -5165,9 +5223,9 @@ class ZeroOrMore(_MultipleMatch): stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, - ): + ) -> None: super().__init__(expr, stopOn=stopOn or stop_on) - self.mayReturnEmpty = True + self._may_return_empty = True def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: try: @@ -5189,7 +5247,7 @@ class DelimitedList(ParseElementEnhance): max: typing.Optional[int] = None, *, allow_trailing_delim: bool = False, - ): + ) -> None: """Helper to define a delimited list of expressions - the delimiter defaults to ','. By default, the list elements and delimiters can have intervening whitespace, and comments, but this can be @@ -5296,11 +5354,11 @@ class Opt(ParseElementEnhance): def __init__( self, expr: Union[ParserElement, str], default: Any = __optionalNotMatched - ): + ) -> None: super().__init__(expr, savelist=False) self.saveAsList = self.expr.saveAsList self.defaultValue = default - self.mayReturnEmpty = True + self._may_return_empty = True def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: self_expr = self.expr @@ -5401,11 +5459,11 @@ class SkipTo(ParseElementEnhance): fail_on: typing.Optional[Union[ParserElement, str]] = None, *, failOn: typing.Optional[Union[ParserElement, str]] = None, - ): + ) -> None: super().__init__(other) failOn = failOn or fail_on self.ignoreExpr = ignore - self.mayReturnEmpty = True + self._may_return_empty = True self.mayIndexError = False self.includeMatch = include self.saveAsList = False @@ -5512,7 +5570,9 @@ class Forward(ParseElementEnhance): parser created using ``Forward``. """ - def __init__(self, other: typing.Optional[Union[ParserElement, str]] = None): + def __init__( + self, other: typing.Optional[Union[ParserElement, str]] = None + ) -> None: self.caller_frame = traceback.extract_stack(limit=2)[0] super().__init__(other, savelist=False) # type: ignore[arg-type] self.lshift_line = None @@ -5529,7 +5589,7 @@ class Forward(ParseElementEnhance): self.expr = other self.streamlined = other.streamlined self.mayIndexError = self.expr.mayIndexError - self.mayReturnEmpty = self.expr.mayReturnEmpty + self._may_return_empty = self.expr.mayReturnEmpty self.set_whitespace_chars( self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars ) @@ -5648,7 +5708,7 @@ class Forward(ParseElementEnhance): try: new_loc, new_peek = super().parseImpl(instring, loc, False) except ParseException: - # we failed before getting any match – do not hide the error + # we failed before getting any match - do not hide the error if isinstance(prev_peek, Exception): raise new_loc, new_peek = prev_loc, prev_peek @@ -5703,17 +5763,20 @@ class Forward(ParseElementEnhance): def _generateDefaultName(self) -> str: # Avoid infinite recursion by setting a temporary _defaultName + save_default_name = self._defaultName self._defaultName = ": ..." # Use the string representation of main expression. - retString = "..." try: if self.expr is not None: - retString = str(self.expr)[:1000] + ret_string = str(self.expr)[:1000] else: - retString = "None" - finally: - return f"{type(self).__name__}: {retString}" + ret_string = "None" + except Exception: + ret_string = "..." + + self._defaultName = save_default_name + return f"{type(self).__name__}: {ret_string}" def copy(self) -> ParserElement: if self.expr is not None: @@ -5752,7 +5815,7 @@ class TokenConverter(ParseElementEnhance): Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results. """ - def __init__(self, expr: Union[ParserElement, str], savelist=False): + def __init__(self, expr: Union[ParserElement, str], savelist=False) -> None: super().__init__(expr) # , savelist) self.saveAsList = False @@ -5783,7 +5846,7 @@ class Combine(TokenConverter): adjacent: bool = True, *, joinString: typing.Optional[str] = None, - ): + ) -> None: super().__init__(expr) joinString = joinString if joinString is not None else join_string # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself @@ -5835,7 +5898,7 @@ class Group(TokenConverter): # -> ['fn', ['a', 'b', '100']] """ - def __init__(self, expr: ParserElement, aslist: bool = False): + def __init__(self, expr: ParserElement, aslist: bool = False) -> None: super().__init__(expr) self.saveAsList = True self._asPythonList = aslist @@ -5893,7 +5956,7 @@ class Dict(TokenConverter): See more examples at :class:`ParseResults` of accessing fields by results name. """ - def __init__(self, expr: ParserElement, asdict: bool = False): + def __init__(self, expr: ParserElement, asdict: bool = False) -> None: super().__init__(expr) self.saveAsList = True self._asPythonDict = asdict @@ -5969,7 +6032,7 @@ class Suppress(TokenConverter): (See also :class:`DelimitedList`.) """ - def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): + def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None: if expr is ...: expr = _PendingSkip(NoMatch()) super().__init__(expr) @@ -6094,13 +6157,17 @@ def srange(s: str) -> str: - any combination of the above (``'aeiouy'``, ``'a-zA-Z0-9_$'``, etc.) """ - _expanded = lambda p: ( - p - if not isinstance(p, ParseResults) - else "".join(chr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) - ) + + def _expanded(p): + if isinstance(p, ParseResults): + yield from (chr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) + else: + yield p + try: - return "".join(_expanded(part) for part in _reBracketExpr.parse_string(s).body) + return "".join( + [c for part in _reBracketExpr.parse_string(s).body for c in _expanded(part)] + ) except Exception as e: return "" @@ -6156,11 +6223,17 @@ def autoname_elements() -> None: Utility to simplify mass-naming of parser elements, for generating railroad diagram with named subdiagrams. """ - calling_frame = sys._getframe(1) + + # guard against _getframe not being implemented in the current Python + getframe_fn = getattr(sys, "_getframe", lambda _: None) + calling_frame = getframe_fn(1) if calling_frame is None: return + + # find all locals in the calling frame that are ParserElements calling_frame = typing.cast(types.FrameType, calling_frame) for name, var in calling_frame.f_locals.items(): + # if no custom name defined, set the name to the var name if isinstance(var, ParserElement) and not var.customName: var.set_name(name) diff --git a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py index 56526b741b..526cf3862a 100644 --- a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py +++ b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py @@ -120,7 +120,7 @@ class EachItem(railroad.Group): all_label = "[ALL]" - def __init__(self, *items): + def __init__(self, *items) -> None: choice_item = railroad.Choice(len(items) - 1, *items) one_or_more_item = railroad.OneOrMore(item=choice_item) super().__init__(one_or_more_item, label=self.all_label) @@ -131,7 +131,7 @@ class AnnotatedItem(railroad.Group): Simple subclass of Group that creates an annotation label """ - def __init__(self, label: str, item): + def __init__(self, label: str, item) -> None: super().__init__(item=item, label=f"[{label}]" if label else "") @@ -144,7 +144,7 @@ class EditablePartial(Generic[T]): # We need this here because the railroad constructors actually transform the data, so can't be called until the # entire tree is assembled - def __init__(self, func: Callable[..., T], args: list, kwargs: dict): + def __init__(self, func: Callable[..., T], args: list, kwargs: dict) -> None: self.func = func self.args = args self.kwargs = kwargs @@ -226,6 +226,7 @@ def to_railroad( vertical: int = 3, show_results_names: bool = False, show_groups: bool = False, + show_hidden: bool = False, ) -> list[NamedDiagram]: """ Convert a pyparsing element tree into a list of diagrams. This is the recommended entrypoint to diagram @@ -238,6 +239,8 @@ def to_railroad( included in the diagram :param show_groups - bool to indicate whether groups should be highlighted with an unlabeled surrounding box + :param show_hidden - bool to indicate whether internal elements that are typically hidden + should be shown """ # Convert the whole tree underneath the root lookup = ConverterState(diagram_kwargs=diagram_kwargs or {}) @@ -248,6 +251,7 @@ def to_railroad( vertical=vertical, show_results_names=show_results_names, show_groups=show_groups, + show_hidden=show_hidden, ) root_id = id(element) @@ -348,7 +352,7 @@ class ConverterState: Stores some state that persists between recursions into the element tree """ - def __init__(self, diagram_kwargs: typing.Optional[dict] = None): + def __init__(self, diagram_kwargs: typing.Optional[dict] = None) -> None: #: A dictionary mapping ParserElements to state relating to them self._element_diagram_states: dict[int, ElementState] = {} #: A dictionary mapping ParserElement IDs to subdiagrams generated from them @@ -453,6 +457,7 @@ def _apply_diagram_item_enhancements(fn): name_hint: str = None, show_results_names: bool = False, show_groups: bool = False, + show_hidden: bool = False, ) -> typing.Optional[EditablePartial]: ret = fn( element, @@ -463,6 +468,7 @@ def _apply_diagram_item_enhancements(fn): name_hint, show_results_names, show_groups, + show_hidden, ) # apply annotation for results name, if present @@ -555,6 +561,7 @@ def _to_diagram_element( name_hint=propagated_name, show_results_names=show_results_names, show_groups=show_groups, + show_hidden=show_hidden, ) # If the element isn't worth extracting, we always treat it as the first time we say it @@ -641,6 +648,7 @@ def _to_diagram_element( name_hint, show_results_names, show_groups, + show_hidden, ] return _to_diagram_element( (~element.not_ender.expr + element.expr)[1, ...].set_name(element.name), @@ -657,6 +665,7 @@ def _to_diagram_element( name_hint, show_results_names, show_groups, + show_hidden, ] return _to_diagram_element( (~element.not_ender.expr + element.expr)[...].set_name(element.name), @@ -707,6 +716,7 @@ def _to_diagram_element( index=i, show_results_names=show_results_names, show_groups=show_groups, + show_hidden=show_hidden, ) # Some elements don't need to be shown in the diagram diff --git a/contrib/python/pyparsing/py3/pyparsing/exceptions.py b/contrib/python/pyparsing/py3/pyparsing/exceptions.py index 57a1579d12..fe07a85585 100644 --- a/contrib/python/pyparsing/py3/pyparsing/exceptions.py +++ b/contrib/python/pyparsing/py3/pyparsing/exceptions.py @@ -52,7 +52,7 @@ class ParseBaseException(Exception): loc: int = 0, msg: typing.Optional[str] = None, elem=None, - ): + ) -> None: if msg is None: msg, pstr = pstr, "" @@ -87,7 +87,7 @@ class ParseBaseException(Exception): ret: list[str] = [] if isinstance(exc, ParseBaseException): ret.append(exc.line) - ret.append(f"{' ' * (exc.column - 1)}^") + ret.append(f"{'^':>{exc.column}}") ret.append(f"{type(exc).__name__}: {exc}") if depth <= 0 or exc.__traceback__ is None: @@ -272,12 +272,11 @@ class ParseException(ParseBaseException): try: integer.parse_string("ABC") except ParseException as pe: - print(pe) - print(f"column: {pe.column}") + print(pe, f"column: {pe.column}") prints:: - Expected integer (at char 0), (line:1, col:1) column: 1 + Expected integer, found 'ABC' (at char 0), (line:1, col:1) column: 1 """ @@ -307,7 +306,7 @@ class RecursiveGrammarException(Exception): Deprecated: only used by deprecated method ParserElement.validate. """ - def __init__(self, parseElementList): + def __init__(self, parseElementList) -> None: self.parseElementTrace = parseElementList def __str__(self) -> str: diff --git a/contrib/python/pyparsing/py3/pyparsing/helpers.py b/contrib/python/pyparsing/py3/pyparsing/helpers.py index f781e87132..7f62df8637 100644 --- a/contrib/python/pyparsing/py3/pyparsing/helpers.py +++ b/contrib/python/pyparsing/py3/pyparsing/helpers.py @@ -208,11 +208,9 @@ def one_of( if caseless: is_equal = lambda a, b: a.upper() == b.upper() masks = lambda a, b: b.upper().startswith(a.upper()) - parse_element_class = CaselessKeyword if asKeyword else CaselessLiteral else: is_equal = operator.eq masks = lambda a, b: b.startswith(a) - parse_element_class = Keyword if asKeyword else Literal symbols: list[str] if isinstance(strs, str_type): @@ -255,7 +253,8 @@ def one_of( if asKeyword: patt = rf"\b(?:{patt})\b" - ret = Regex(patt, flags=re_flags).set_name(" | ".join(symbols)) + ret = Regex(patt, flags=re_flags) + ret.set_name(" | ".join(re.escape(s) for s in symbols)) if caseless: # add parse action to return symbols as specified, not in random @@ -270,13 +269,21 @@ def one_of( "Exception creating Regex for one_of, building MatchFirst", stacklevel=2 ) - # last resort, just use MatchFirst + # last resort, just use MatchFirst of Token class corresponding to caseless + # and asKeyword settings + CASELESS = KEYWORD = True + parse_element_class = { + (CASELESS, KEYWORD): CaselessKeyword, + (CASELESS, not KEYWORD): CaselessLiteral, + (not CASELESS, KEYWORD): Keyword, + (not CASELESS, not KEYWORD): Literal, + }[(caseless, asKeyword)] return MatchFirst(parse_element_class(sym) for sym in symbols).set_name( " | ".join(symbols) ) -def dict_of(key: ParserElement, value: ParserElement) -> ParserElement: +def dict_of(key: ParserElement, value: ParserElement) -> Dict: """Helper to easily and clearly define a dictionary by specifying the respective patterns for the key and value. Takes care of defining the :class:`Dict`, :class:`ZeroOrMore`, and @@ -411,13 +418,16 @@ def locatedExpr(expr: ParserElement) -> ParserElement: ) +_NO_IGNORE_EXPR_GIVEN = NoMatch() + + def nested_expr( opener: Union[str, ParserElement] = "(", closer: Union[str, ParserElement] = ")", content: typing.Optional[ParserElement] = None, - ignore_expr: ParserElement = quoted_string(), + ignore_expr: ParserElement = _NO_IGNORE_EXPR_GIVEN, *, - ignoreExpr: ParserElement = quoted_string(), + ignoreExpr: ParserElement = _NO_IGNORE_EXPR_GIVEN, ) -> ParserElement: """Helper method for defining nested lists enclosed in opening and closing delimiters (``"("`` and ``")"`` are the default). @@ -487,7 +497,10 @@ def nested_expr( dec_to_hex (int) args: [['char', 'hchar']] """ if ignoreExpr != ignore_expr: - ignoreExpr = ignore_expr if ignoreExpr == quoted_string() else ignoreExpr + ignoreExpr = ignore_expr if ignoreExpr is _NO_IGNORE_EXPR_GIVEN else ignoreExpr + if ignoreExpr is _NO_IGNORE_EXPR_GIVEN: + ignoreExpr = quoted_string() + if opener == closer: raise ValueError("opening and closing strings cannot be the same") if content is None: @@ -504,11 +517,11 @@ def nested_expr( exact=1, ) ) - ).set_parse_action(lambda t: t[0].strip()) + ) else: content = empty.copy() + CharsNotIn( opener + closer + ParserElement.DEFAULT_WHITE_CHARS - ).set_parse_action(lambda t: t[0].strip()) + ) else: if ignoreExpr is not None: content = Combine( @@ -518,7 +531,7 @@ def nested_expr( + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) ) - ).set_parse_action(lambda t: t[0].strip()) + ) else: content = Combine( OneOrMore( @@ -526,11 +539,16 @@ def nested_expr( + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) ) - ).set_parse_action(lambda t: t[0].strip()) + ) else: raise ValueError( "opening and closing arguments must be strings if no content expression is given" ) + if ParserElement.DEFAULT_WHITE_CHARS: + content.set_parse_action( + lambda t: t[0].strip(ParserElement.DEFAULT_WHITE_CHARS) + ) + ret = Forward() if ignoreExpr is not None: ret <<= Group( @@ -691,7 +709,7 @@ def infix_notation( op_list: list[InfixNotationOperatorSpec], lpar: Union[str, ParserElement] = Suppress("("), rpar: Union[str, ParserElement] = Suppress(")"), -) -> ParserElement: +) -> Forward: """Helper method for constructing grammars of expressions made up of operators working in a precedence hierarchy. Operators may be unary or binary, left- or right-associative. Parse actions can also be diff --git a/contrib/python/pyparsing/py3/pyparsing/results.py b/contrib/python/pyparsing/py3/pyparsing/results.py index be834b7e60..956230352c 100644 --- a/contrib/python/pyparsing/py3/pyparsing/results.py +++ b/contrib/python/pyparsing/py3/pyparsing/results.py @@ -23,7 +23,7 @@ class _ParseResultsWithOffset: tup: tuple[ParseResults, int] __slots__ = ["tup"] - def __init__(self, p1: ParseResults, p2: int): + def __init__(self, p1: ParseResults, p2: int) -> None: self.tup: tuple[ParseResults, int] = (p1, p2) def __getitem__(self, i): diff --git a/contrib/python/pyparsing/py3/pyparsing/tools/__init__.py b/contrib/python/pyparsing/py3/pyparsing/tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/pyparsing/py3/pyparsing/tools/__init__.py diff --git a/contrib/python/pyparsing/py3/pyparsing/tools/cvt_pyparsing_pep8_names.py b/contrib/python/pyparsing/py3/pyparsing/tools/cvt_pyparsing_pep8_names.py new file mode 100644 index 0000000000..f4a8bd9f51 --- /dev/null +++ b/contrib/python/pyparsing/py3/pyparsing/tools/cvt_pyparsing_pep8_names.py @@ -0,0 +1,116 @@ +from functools import lru_cache +import pyparsing as pp + + +@lru_cache(maxsize=None) +def camel_to_snake(s: str) -> str: + """ + Convert CamelCase to snake_case. + """ + return "".join("_" + c.lower() if c.isupper() else c for c in s).lstrip("_") + + +pre_pep8_method_names = """ +addCondition addParseAction anyCloseTag anyOpenTag asDict asList cStyleComment canParseNext conditionAsParseAction +convertToDate convertToDatetime convertToFloat convertToInteger countedArray cppStyleComment dblQuotedString +dblSlashComment defaultName dictOf disableMemoization downcaseTokens enableLeftRecursion enablePackrat getName +htmlComment ignoreWhitespace indentedBlock infixNotation inlineLiteralsUsing javaStyleComment leaveWhitespace +lineEnd lineStart locatedExpr matchOnlyAtCol matchPreviousExpr matchPreviousLiteral nestedExpr nullDebugAction oneOf +originalTextFor parseFile parseString parseWithTabs pythonStyleComment quotedString removeQuotes replaceWith +resetCache restOfLine runTests scanString searchString setBreak setDebug setDebugActions setDefaultWhitespaceChars +setFailAction setName setParseAction setResultsName setWhitespaceChars sglQuotedString stringEnd stringStart tokenMap +traceParseAction transformString tryParse unicodeString upcaseTokens withAttribute withClass +""".split() + +special_changes = { + "opAssoc": "OpAssoc", + "delimitedList": "DelimitedList", + "delimited_list": "DelimitedList", + "replaceHTMLEntity": "replace_html_entity", + "makeHTMLTags": "make_html_tags", + "makeXMLTags": "make_xml_tags", + "commonHTMLEntity": "common_html_entity", + "stripHTMLTags": "strip_html_tags", +} + +pre_pep8_arg_names = """parseAll maxMatches listAllMatches callDuringTry includeSeparators fullDump printResults +failureTests postParse matchString identChars maxMismatches initChars bodyChars asKeyword excludeChars asGroupList +asMatch quoteChar escChar escQuote unquoteResults endQuoteChar convertWhitespaceEscapes notChars wordChars stopOn +failOn joinString markerString intExpr useRegex asString ignoreExpr""".split() + +pre_pep8_method_name = pp.one_of(pre_pep8_method_names, as_keyword=True) +pre_pep8_method_name.set_parse_action(lambda t: camel_to_snake(t[0])) +special_pre_pep8_name = pp.one_of(special_changes, as_keyword=True) +special_pre_pep8_name.set_parse_action(lambda t: special_changes[t[0]]) +# only replace arg names if part of an arg list +pre_pep8_arg_name = pp.Regex( + rf"{pp.util.make_compressed_re(pre_pep8_arg_names)}\s*=" +) +pre_pep8_arg_name.set_parse_action(lambda t: camel_to_snake(t[0])) + +pep8_converter = pre_pep8_method_name | special_pre_pep8_name | pre_pep8_arg_name + +if __name__ == "__main__": + import argparse + from pathlib import Path + import sys + + argparser = argparse.ArgumentParser( + description = ( + "Utility to convert Python pyparsing scripts using legacy" + " camelCase names to use PEP8 snake_case names." + "\nBy default, this script will only show whether this script would make any changes." + ) + ) + argparser.add_argument("--verbose", "-v", action="store_true", help="Show unified diff for each source file") + argparser.add_argument("-vv", action="store_true", dest="verbose2", help="Show unified diff for each source file, plus names of scanned files with no changes") + argparser.add_argument("--update", "-u", action="store_true", help="Update source files in-place") + argparser.add_argument("--encoding", type=str, default="utf-8", help="Encoding of source files (default: utf-8)") + argparser.add_argument("--exit-zero-even-if-changed", "-exit0", action="store_true", help="Exit with status code 0 even if changes were made") + argparser.add_argument("source_filename", nargs="+", help="Source filenames or filename patterns of Python files to be converted") + args = argparser.parse_args() + + + def show_diffs(original, modified): + import difflib + + diff = difflib.unified_diff( + original.splitlines(), modified.splitlines(), lineterm="" + ) + sys.stdout.writelines(f"{diff_line}\n" for diff_line in diff) + + exit_status = 0 + + for filename_pattern in args.source_filename: + + for filename in Path().glob(filename_pattern): + if not Path(filename).is_file(): + continue + + try: + original_contents = Path(filename).read_text(encoding=args.encoding) + modified_contents = pep8_converter.transform_string( + original_contents + ) + + if modified_contents != original_contents: + if args.update: + Path(filename).write_text(modified_contents, encoding=args.encoding) + print(f"Converted {filename}") + else: + print(f"Found required changes in {filename}") + + if args.verbose: + show_diffs(original_contents, modified_contents) + print() + + exit_status = 1 + + else: + if args.verbose2: + print(f"No required changes in {filename}") + + except Exception as e: + print(f"Failed to convert {filename}: {type(e).__name__}: {e}") + + sys.exit(exit_status if not args.exit_zero_even_if_changed else 0) diff --git a/contrib/python/pyparsing/py3/pyparsing/util.py b/contrib/python/pyparsing/py3/pyparsing/util.py index 03a60d4fdd..1cb16e2e62 100644 --- a/contrib/python/pyparsing/py3/pyparsing/util.py +++ b/contrib/python/pyparsing/py3/pyparsing/util.py @@ -1,5 +1,6 @@ # util.py import contextlib +import re from functools import lru_cache, wraps import inspect import itertools @@ -193,7 +194,7 @@ class _GroupConsecutive: (3, iter(['p', 'q', 'r', 's'])) """ - def __init__(self): + def __init__(self) -> None: self.prev = 0 self.counter = itertools.count() self.value = -1 @@ -303,7 +304,11 @@ def _flatten(ll: Iterable) -> list: def make_compressed_re( - word_list: Iterable[str], max_level: int = 2, _level: int = 1 + word_list: Iterable[str], + max_level: int = 2, + *, + non_capturing_groups: bool = True, + _level: int = 1, ) -> str: """ Create a regular expression string from a list of words, collapsing by common @@ -320,15 +325,38 @@ def make_compressed_re( else: yield namelist[0][0], [namelist[0][1:]] + if _level == 1: + if not word_list: + raise ValueError("no words given to make_compressed_re()") + + if "" in word_list: + raise ValueError("word list cannot contain empty string") + else: + # internal recursive call, just return empty string if no words + if not word_list: + return "" + + # dedupe the word list + word_list = list({}.fromkeys(word_list)) + if max_level == 0: - return "|".join(sorted(word_list, key=len, reverse=True)) + if any(len(wd) > 1 for wd in word_list): + return "|".join( + sorted([re.escape(wd) for wd in word_list], key=len, reverse=True) + ) + else: + return f"[{''.join(_escape_regex_range_chars(wd) for wd in word_list)}]" ret = [] sep = "" + ncgroup = "?:" if non_capturing_groups else "" + for initial, suffixes in get_suffixes_from_common_prefixes(sorted(word_list)): ret.append(sep) sep = "|" + initial = re.escape(initial) + trailing = "" if "" in suffixes: trailing = "?" @@ -336,21 +364,33 @@ def make_compressed_re( if len(suffixes) > 1: if all(len(s) == 1 for s in suffixes): - ret.append(f"{initial}[{''.join(suffixes)}]{trailing}") + ret.append( + f"{initial}[{''.join(_escape_regex_range_chars(s) for s in suffixes)}]{trailing}" + ) else: if _level < max_level: suffix_re = make_compressed_re( - sorted(suffixes), max_level, _level + 1 + sorted(suffixes), + max_level, + non_capturing_groups=non_capturing_groups, + _level=_level + 1, ) - ret.append(f"{initial}({suffix_re}){trailing}") + ret.append(f"{initial}({ncgroup}{suffix_re}){trailing}") else: - suffixes.sort(key=len, reverse=True) - ret.append(f"{initial}({'|'.join(suffixes)}){trailing}") + if all(len(s) == 1 for s in suffixes): + ret.append( + f"{initial}[{''.join(_escape_regex_range_chars(s) for s in suffixes)}]{trailing}" + ) + else: + suffixes.sort(key=len, reverse=True) + ret.append( + f"{initial}({ncgroup}{'|'.join(re.escape(s) for s in suffixes)}){trailing}" + ) else: if suffixes: - suffix = suffixes[0] + suffix = re.escape(suffixes[0]) if len(suffix) > 1 and trailing: - ret.append(f"{initial}({suffix}){trailing}") + ret.append(f"{initial}({ncgroup}{suffix}){trailing}") else: ret.append(f"{initial}{suffix}{trailing}") else: diff --git a/contrib/python/pyparsing/py3/ya.make b/contrib/python/pyparsing/py3/ya.make index e229986ca6..a53ebf37ec 100644 --- a/contrib/python/pyparsing/py3/ya.make +++ b/contrib/python/pyparsing/py3/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() PROVIDES(pyparsing) -VERSION(3.2.1) +VERSION(3.2.2) LICENSE(MIT) @@ -25,6 +25,8 @@ PY_SRCS( pyparsing/helpers.py pyparsing/results.py pyparsing/testing.py + pyparsing/tools/__init__.py + pyparsing/tools/cvt_pyparsing_pep8_names.py pyparsing/unicode.py pyparsing/util.py ) diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index b6911ce75e..904414722e 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.19.3 +Version: 3.20.1 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index 71cfb8fa72..fbc5d148f8 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.19.3) +VERSION(3.20.1) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/ydb/_apis.py b/contrib/python/ydb/py3/ydb/_apis.py index fc28d0ceb2..fc6f16e287 100644 --- a/contrib/python/ydb/py3/ydb/_apis.py +++ b/contrib/python/ydb/py3/ydb/_apis.py @@ -115,6 +115,7 @@ class TopicService(object): DropTopic = "DropTopic" StreamRead = "StreamRead" StreamWrite = "StreamWrite" + UpdateOffsetsInTransaction = "UpdateOffsetsInTransaction" class QueryService(object): diff --git a/contrib/python/ydb/py3/ydb/_errors.py b/contrib/python/ydb/py3/ydb/_errors.py index 17002d2574..1e2308ef39 100644 --- a/contrib/python/ydb/py3/ydb/_errors.py +++ b/contrib/python/ydb/py3/ydb/_errors.py @@ -5,6 +5,7 @@ from . import issues _errors_retriable_fast_backoff_types = [ issues.Unavailable, + issues.ClientInternalError, ] _errors_retriable_slow_backoff_types = [ issues.Aborted, diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py index 5b22c7cf86..0f8a0f03a7 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py @@ -141,6 +141,18 @@ class UpdateTokenResponse(IFromProto): ######################################################################################################################## +@dataclass +class TransactionIdentity(IToProto): + tx_id: str + session_id: str + + def to_proto(self) -> ydb_topic_pb2.TransactionIdentity: + return ydb_topic_pb2.TransactionIdentity( + id=self.tx_id, + session=self.session_id, + ) + + class StreamWriteMessage: @dataclass() class InitRequest(IToProto): @@ -199,6 +211,7 @@ class StreamWriteMessage: class WriteRequest(IToProto): messages: typing.List["StreamWriteMessage.WriteRequest.MessageData"] codec: int + tx_identity: Optional[TransactionIdentity] @dataclass class MessageData(IToProto): @@ -237,6 +250,9 @@ class StreamWriteMessage: proto = ydb_topic_pb2.StreamWriteMessage.WriteRequest() proto.codec = self.codec + if self.tx_identity is not None: + proto.tx.CopyFrom(self.tx_identity.to_proto()) + for message in self.messages: proto_mess = proto.messages.add() proto_mess.CopyFrom(message.to_proto()) @@ -297,6 +313,8 @@ class StreamWriteMessage: ) except ValueError: message_write_status = reason + elif proto_ack.HasField("written_in_tx"): + message_write_status = StreamWriteMessage.WriteResponse.WriteAck.StatusWrittenInTx() else: raise NotImplementedError("unexpected ack status") @@ -309,6 +327,9 @@ class StreamWriteMessage: class StatusWritten: offset: int + class StatusWrittenInTx: + pass + @dataclass class StatusSkipped: reason: "StreamWriteMessage.WriteResponse.WriteAck.StatusSkipped.Reason" @@ -1197,6 +1218,52 @@ class MeteringMode(int, IFromProto, IFromPublic, IToPublic): @dataclass +class UpdateOffsetsInTransactionRequest(IToProto): + tx: TransactionIdentity + topics: List[UpdateOffsetsInTransactionRequest.TopicOffsets] + consumer: str + + def to_proto(self): + return ydb_topic_pb2.UpdateOffsetsInTransactionRequest( + tx=self.tx.to_proto(), + consumer=self.consumer, + topics=list( + map( + UpdateOffsetsInTransactionRequest.TopicOffsets.to_proto, + self.topics, + ) + ), + ) + + @dataclass + class TopicOffsets(IToProto): + path: str + partitions: List[UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets] + + def to_proto(self): + return ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets( + path=self.path, + partitions=list( + map( + UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets.to_proto, + self.partitions, + ) + ), + ) + + @dataclass + class PartitionOffsets(IToProto): + partition_id: int + partition_offsets: List[OffsetsRange] + + def to_proto(self) -> ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets: + return ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets( + partition_id=self.partition_id, + partition_offsets=list(map(OffsetsRange.to_proto, self.partition_offsets)), + ) + + +@dataclass class CreateTopicRequest(IToProto, IFromPublic): path: str partitioning_settings: "PartitioningSettings" diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py b/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py index b48501aff2..74f06a086f 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py @@ -108,6 +108,9 @@ class PartitionSession: waiter = self._ack_waiters.popleft() waiter._finish_ok() + def _update_last_commited_offset_if_needed(self, offset: int): + self.committed_offset = max(self.committed_offset, offset) + def close(self): if self.closed: return @@ -211,3 +214,9 @@ class PublicBatch(ICommittable, ISessionAlive): self._bytes_size = self._bytes_size - new_batch._bytes_size return new_batch + + def _update_partition_offsets(self, tx, exc=None): + if exc is not None: + return + offsets = self._commit_get_offsets_range() + self._partition_session._update_last_commited_offset_if_needed(offsets.end) diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py index 7061b4e449..87012554ef 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py @@ -5,7 +5,7 @@ import concurrent.futures import gzip import typing from asyncio import Task -from collections import OrderedDict +from collections import defaultdict, OrderedDict from typing import Optional, Set, Dict, Union, Callable import ydb @@ -19,17 +19,24 @@ from . import topic_reader from .._grpc.grpcwrapper.common_utils import ( IGrpcWrapperAsyncIO, SupportedDriverType, + to_thread, GrpcWrapperAsyncIO, ) from .._grpc.grpcwrapper.ydb_topic import ( StreamReadMessage, UpdateTokenRequest, UpdateTokenResponse, + UpdateOffsetsInTransactionRequest, Codec, ) from .._errors import check_retriable_error import logging +from ..query.base import TxEvent + +if typing.TYPE_CHECKING: + from ..query.transaction import BaseQueryTxContext + logger = logging.getLogger(__name__) @@ -77,7 +84,7 @@ class PublicAsyncIOReader: ): self._loop = asyncio.get_running_loop() self._closed = False - self._reconnector = ReaderReconnector(driver, settings) + self._reconnector = ReaderReconnector(driver, settings, self._loop) self._parent = _parent async def __aenter__(self): @@ -88,8 +95,12 @@ class PublicAsyncIOReader: def __del__(self): if not self._closed: - task = self._loop.create_task(self.close(flush=False)) - topic_common.wrap_set_name_for_asyncio_task(task, task_name="close reader") + try: + logger.warning("Topic reader was not closed properly. Consider using method close().") + task = self._loop.create_task(self.close(flush=False)) + topic_common.wrap_set_name_for_asyncio_task(task, task_name="close reader") + except BaseException: + logger.warning("Something went wrong during reader close in __del__") async def wait_message(self): """ @@ -112,6 +123,23 @@ class PublicAsyncIOReader: max_messages=max_messages, ) + async def receive_batch_with_tx( + self, + tx: "BaseQueryTxContext", + max_messages: typing.Union[int, None] = None, + ) -> typing.Union[datatypes.PublicBatch, None]: + """ + Get one messages batch with tx from reader. + All messages in a batch from same partition. + + use asyncio.wait_for for wait with timeout. + """ + await self._reconnector.wait_message() + return self._reconnector.receive_batch_with_tx_nowait( + tx=tx, + max_messages=max_messages, + ) + async def receive_message(self) -> typing.Optional[datatypes.PublicMessage]: """ Block until receive new message @@ -165,11 +193,18 @@ class ReaderReconnector: _state_changed: asyncio.Event _stream_reader: Optional["ReaderStream"] _first_error: asyncio.Future[YdbError] + _tx_to_batches_map: Dict[str, typing.List[datatypes.PublicBatch]] - def __init__(self, driver: Driver, settings: topic_reader.PublicReaderSettings): + def __init__( + self, + driver: Driver, + settings: topic_reader.PublicReaderSettings, + loop: Optional[asyncio.AbstractEventLoop] = None, + ): self._id = self._static_reader_reconnector_counter.inc_and_get() self._settings = settings self._driver = driver + self._loop = loop if loop is not None else asyncio.get_running_loop() self._background_tasks = set() self._state_changed = asyncio.Event() @@ -177,6 +212,8 @@ class ReaderReconnector: self._background_tasks.add(asyncio.create_task(self._connection_loop())) self._first_error = asyncio.get_running_loop().create_future() + self._tx_to_batches_map = dict() + async def _connection_loop(self): attempt = 0 while True: @@ -190,6 +227,7 @@ class ReaderReconnector: if not retry_info.is_retriable: self._set_first_error(err) return + await asyncio.sleep(retry_info.sleep_timeout_seconds) attempt += 1 @@ -222,9 +260,87 @@ class ReaderReconnector: max_messages=max_messages, ) + def receive_batch_with_tx_nowait(self, tx: "BaseQueryTxContext", max_messages: Optional[int] = None): + batch = self._stream_reader.receive_batch_nowait( + max_messages=max_messages, + ) + + self._init_tx(tx) + + self._tx_to_batches_map[tx.tx_id].append(batch) + + tx._add_callback(TxEvent.AFTER_COMMIT, batch._update_partition_offsets, self._loop) + + return batch + def receive_message_nowait(self): return self._stream_reader.receive_message_nowait() + def _init_tx(self, tx: "BaseQueryTxContext"): + if tx.tx_id not in self._tx_to_batches_map: # Init tx callbacks + self._tx_to_batches_map[tx.tx_id] = [] + tx._add_callback(TxEvent.BEFORE_COMMIT, self._commit_batches_with_tx, self._loop) + tx._add_callback(TxEvent.AFTER_COMMIT, self._handle_after_tx_commit, self._loop) + tx._add_callback(TxEvent.AFTER_ROLLBACK, self._handle_after_tx_rollback, self._loop) + + async def _commit_batches_with_tx(self, tx: "BaseQueryTxContext"): + grouped_batches = defaultdict(lambda: defaultdict(list)) + for batch in self._tx_to_batches_map[tx.tx_id]: + grouped_batches[batch._partition_session.topic_path][batch._partition_session.partition_id].append(batch) + + request = UpdateOffsetsInTransactionRequest(tx=tx._tx_identity(), consumer=self._settings.consumer, topics=[]) + + for topic_path in grouped_batches: + topic_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets(path=topic_path, partitions=[]) + for partition_id in grouped_batches[topic_path]: + partition_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets( + partition_id=partition_id, + partition_offsets=[ + batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id] + ], + ) + topic_offsets.partitions.append(partition_offsets) + request.topics.append(topic_offsets) + + try: + return await self._do_commit_batches_with_tx_call(request) + except BaseException: + exc = issues.ClientInternalError("Failed to update offsets in tx.") + tx._set_external_error(exc) + self._stream_reader._set_first_error(exc) + finally: + del self._tx_to_batches_map[tx.tx_id] + + async def _do_commit_batches_with_tx_call(self, request: UpdateOffsetsInTransactionRequest): + args = [ + request.to_proto(), + _apis.TopicService.Stub, + _apis.TopicService.UpdateOffsetsInTransaction, + topic_common.wrap_operation, + ] + + if asyncio.iscoroutinefunction(self._driver.__call__): + res = await self._driver(*args) + else: + res = await to_thread(self._driver, *args, executor=None) + + return res + + async def _handle_after_tx_rollback(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None: + if tx.tx_id in self._tx_to_batches_map: + del self._tx_to_batches_map[tx.tx_id] + exc = issues.ClientInternalError("Reconnect due to transaction rollback") + self._stream_reader._set_first_error(exc) + + async def _handle_after_tx_commit(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None: + if tx.tx_id in self._tx_to_batches_map: + del self._tx_to_batches_map[tx.tx_id] + + if exc is not None: + self._stream_reader._set_first_error( + issues.ClientInternalError("Reconnect due to transaction commit failed") + ) + def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.CommitAckWaiter: return self._stream_reader.commit(batch) diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py index eda1d374fc..31f2889927 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py @@ -1,5 +1,6 @@ import asyncio import concurrent.futures +import logging import typing from typing import List, Union, Optional @@ -20,6 +21,11 @@ from ydb._topic_reader.topic_reader_asyncio import ( TopicReaderClosedError, ) +if typing.TYPE_CHECKING: + from ..query.transaction import BaseQueryTxContext + +logger = logging.getLogger(__name__) + class TopicReaderSync: _caller: CallFromSyncToAsync @@ -52,7 +58,12 @@ class TopicReaderSync: self._parent = _parent def __del__(self): - self.close(flush=False) + if not self._closed: + try: + logger.warning("Topic reader was not closed properly. Consider using method close().") + self.close(flush=False) + except BaseException: + logger.warning("Something went wrong during reader close in __del__") def __enter__(self): return self @@ -109,6 +120,31 @@ class TopicReaderSync: timeout, ) + def receive_batch_with_tx( + self, + tx: "BaseQueryTxContext", + *, + max_messages: typing.Union[int, None] = None, + max_bytes: typing.Union[int, None] = None, + timeout: Union[float, None] = None, + ) -> Union[PublicBatch, None]: + """ + Get one messages batch with tx from reader + It has no async_ version for prevent lost messages, use async_wait_message as signal for new batches available. + + if no new message in timeout seconds (default - infinite): raise TimeoutError() + if timeout <= 0 - it will fast wait only one event loop cycle - without wait any i/o operations or pauses, get messages from internal buffer only. + """ + self._check_closed() + + return self._caller.safe_call_with_result( + self._async_reader.receive_batch_with_tx( + tx=tx, + max_messages=max_messages, + ), + timeout, + ) + def commit(self, mess: typing.Union[datatypes.PublicMessage, datatypes.PublicBatch]): """ Put commit message to internal buffer. diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py index aa5fe9749a..a3e407ed86 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py @@ -11,6 +11,7 @@ import typing import ydb.aio from .._grpc.grpcwrapper.ydb_topic import StreamWriteMessage +from .._grpc.grpcwrapper.ydb_topic import TransactionIdentity from .._grpc.grpcwrapper.common_utils import IToProto from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .. import connection @@ -53,8 +54,12 @@ class PublicWriteResult: class Skipped: pass + @dataclass(eq=True) + class WrittenInTx: + pass + -PublicWriteResultTypes = Union[PublicWriteResult.Written, PublicWriteResult.Skipped] +PublicWriteResultTypes = Union[PublicWriteResult.Written, PublicWriteResult.Skipped, PublicWriteResult.WrittenInTx] class WriterSettings(PublicWriterSettings): @@ -205,6 +210,7 @@ def default_serializer_message_content(data: Any) -> bytes: def messages_to_proto_requests( messages: List[InternalMessage], + tx_identity: Optional[TransactionIdentity], ) -> List[StreamWriteMessage.FromClient]: gropus = _slit_messages_for_send(messages) @@ -215,6 +221,7 @@ def messages_to_proto_requests( StreamWriteMessage.WriteRequest( messages=list(map(InternalMessage.to_message_data, group)), codec=group[0].codec, + tx_identity=tx_identity, ) ) res.append(req) @@ -239,6 +246,7 @@ _message_data_overhead = ( ), ], codec=20000, + tx_identity=None, ) ) .to_proto() diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py index 32d8fefe51..ec5b21661d 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py @@ -1,7 +1,6 @@ import asyncio import concurrent.futures import datetime -import functools import gzip import typing from collections import deque @@ -35,6 +34,7 @@ from .._grpc.grpcwrapper.ydb_topic import ( UpdateTokenRequest, UpdateTokenResponse, StreamWriteMessage, + TransactionIdentity, WriterMessagesFromServerToClient, ) from .._grpc.grpcwrapper.common_utils import ( @@ -43,6 +43,11 @@ from .._grpc.grpcwrapper.common_utils import ( GrpcWrapperAsyncIO, ) +from ..query.base import TxEvent + +if typing.TYPE_CHECKING: + from ..query.transaction import BaseQueryTxContext + logger = logging.getLogger(__name__) @@ -76,8 +81,12 @@ class WriterAsyncIO: def __del__(self): if self._closed or self._loop.is_closed(): return - - self._loop.call_soon(functools.partial(self.close, flush=False)) + try: + logger.warning("Topic writer was not closed properly. Consider using method close().") + task = self._loop.create_task(self.close(flush=False)) + topic_common.wrap_set_name_for_asyncio_task(task, task_name="close writer") + except BaseException: + logger.warning("Something went wrong during writer close in __del__") async def close(self, *, flush: bool = True): if self._closed: @@ -164,6 +173,57 @@ class WriterAsyncIO: return await self._reconnector.wait_init() +class TxWriterAsyncIO(WriterAsyncIO): + _tx: "BaseQueryTxContext" + + def __init__( + self, + tx: "BaseQueryTxContext", + driver: SupportedDriverType, + settings: PublicWriterSettings, + _client=None, + _is_implicit=False, + ): + self._tx = tx + self._loop = asyncio.get_running_loop() + self._closed = False + self._reconnector = WriterAsyncIOReconnector(driver=driver, settings=WriterSettings(settings), tx=self._tx) + self._parent = _client + self._is_implicit = _is_implicit + + # For some reason, creating partition could conflict with other session operations. + # Could be removed later. + self._first_write = True + + tx._add_callback(TxEvent.BEFORE_COMMIT, self._on_before_commit, self._loop) + tx._add_callback(TxEvent.BEFORE_ROLLBACK, self._on_before_rollback, self._loop) + + async def write( + self, + messages: Union[Message, List[Message]], + ): + """ + send one or number of messages to server. + it put message to internal buffer + + For wait with timeout use asyncio.wait_for. + """ + if self._first_write: + self._first_write = False + return await super().write_with_ack(messages) + return await super().write(messages) + + async def _on_before_commit(self, tx: "BaseQueryTxContext"): + if self._is_implicit: + return + await self.close() + + async def _on_before_rollback(self, tx: "BaseQueryTxContext"): + if self._is_implicit: + return + await self.close(flush=False) + + class WriterAsyncIOReconnector: _closed: bool _loop: asyncio.AbstractEventLoop @@ -178,6 +238,7 @@ class WriterAsyncIOReconnector: _codec_selector_batch_num: int _codec_selector_last_codec: Optional[PublicCodec] _codec_selector_check_batches_interval: int + _tx: Optional["BaseQueryTxContext"] if typing.TYPE_CHECKING: _messages_for_encode: asyncio.Queue[List[InternalMessage]] @@ -195,7 +256,9 @@ class WriterAsyncIOReconnector: _stop_reason: asyncio.Future _init_info: Optional[PublicWriterInitInfo] - def __init__(self, driver: SupportedDriverType, settings: WriterSettings): + def __init__( + self, driver: SupportedDriverType, settings: WriterSettings, tx: Optional["BaseQueryTxContext"] = None + ): self._closed = False self._loop = asyncio.get_running_loop() self._driver = driver @@ -205,6 +268,7 @@ class WriterAsyncIOReconnector: self._init_info = None self._stream_connected = asyncio.Event() self._settings = settings + self._tx = tx self._codec_functions = { PublicCodec.RAW: lambda data: data, @@ -354,10 +418,12 @@ class WriterAsyncIOReconnector: # noinspection PyBroadException stream_writer = None try: + tx_identity = None if self._tx is None else self._tx._tx_identity() stream_writer = await WriterAsyncIOStream.create( self._driver, self._init_message, self._settings.update_token_interval, + tx_identity=tx_identity, ) try: if self._init_info is None: @@ -387,7 +453,7 @@ class WriterAsyncIOReconnector: done.pop().result() # need for raise exception - reason of stop task except issues.Error as err: err_info = check_retriable_error(err, retry_settings, attempt) - if not err_info.is_retriable: + if not err_info.is_retriable or self._tx is not None: # no retries in tx writer self._stop(err) return @@ -533,6 +599,8 @@ class WriterAsyncIOReconnector: result = PublicWriteResult.Skipped() elif isinstance(status, write_ack_msg.StatusWritten): result = PublicWriteResult.Written(offset=status.offset) + elif isinstance(status, write_ack_msg.StatusWrittenInTx): + result = PublicWriteResult.WrittenInTx() else: raise TopicWriterError("internal error - receive unexpected ack message.") message_future.set_result(result) @@ -597,10 +665,13 @@ class WriterAsyncIOStream: _update_token_event: asyncio.Event _get_token_function: Optional[Callable[[], str]] + _tx_identity: Optional[TransactionIdentity] + def __init__( self, update_token_interval: Optional[Union[int, float]] = None, get_token_function: Optional[Callable[[], str]] = None, + tx_identity: Optional[TransactionIdentity] = None, ): self._closed = False @@ -609,6 +680,8 @@ class WriterAsyncIOStream: self._update_token_event = asyncio.Event() self._update_token_task = None + self._tx_identity = tx_identity + async def close(self): if self._closed: return @@ -625,6 +698,7 @@ class WriterAsyncIOStream: driver: SupportedDriverType, init_request: StreamWriteMessage.InitRequest, update_token_interval: Optional[Union[int, float]] = None, + tx_identity: Optional[TransactionIdentity] = None, ) -> "WriterAsyncIOStream": stream = GrpcWrapperAsyncIO(StreamWriteMessage.FromServer.from_proto) @@ -634,6 +708,7 @@ class WriterAsyncIOStream: writer = WriterAsyncIOStream( update_token_interval=update_token_interval, get_token_function=creds.get_auth_token if creds else lambda: "", + tx_identity=tx_identity, ) await writer._start(stream, init_request) return writer @@ -680,7 +755,7 @@ class WriterAsyncIOStream: if self._closed: raise RuntimeError("Can not write on closed stream.") - for request in messages_to_proto_requests(messages): + for request in messages_to_proto_requests(messages, self._tx_identity): self._stream.write(request) async def _update_token_loop(self): diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py index a5193caf7c..954864c968 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging import typing from concurrent.futures import Future from typing import Union, List, Optional @@ -14,13 +15,23 @@ from .topic_writer import ( TopicWriterClosedError, ) -from .topic_writer_asyncio import WriterAsyncIO +from ..query.base import TxEvent + +from .topic_writer_asyncio import ( + TxWriterAsyncIO, + WriterAsyncIO, +) from .._topic_common.common import ( _get_shared_event_loop, TimeoutType, CallFromSyncToAsync, ) +if typing.TYPE_CHECKING: + from ..query.transaction import BaseQueryTxContext + +logger = logging.getLogger(__name__) + class WriterSync: _caller: CallFromSyncToAsync @@ -63,7 +74,12 @@ class WriterSync: raise def __del__(self): - self.close(flush=False) + if not self._closed: + try: + logger.warning("Topic writer was not closed properly. Consider using method close().") + self.close(flush=False) + except BaseException: + logger.warning("Something went wrong during writer close in __del__") def close(self, *, flush: bool = True, timeout: TimeoutType = None): if self._closed: @@ -122,3 +138,39 @@ class WriterSync: self._check_closed() return self._caller.unsafe_call_with_result(self._async_writer.write_with_ack(messages), timeout=timeout) + + +class TxWriterSync(WriterSync): + def __init__( + self, + tx: "BaseQueryTxContext", + driver: SupportedDriverType, + settings: PublicWriterSettings, + *, + eventloop: Optional[asyncio.AbstractEventLoop] = None, + _parent=None, + ): + + self._closed = False + + if eventloop: + loop = eventloop + else: + loop = _get_shared_event_loop() + + self._caller = CallFromSyncToAsync(loop) + + async def create_async_writer(): + return TxWriterAsyncIO(tx, driver, settings, _is_implicit=True) + + self._async_writer = self._caller.safe_call_with_result(create_async_writer(), None) + self._parent = _parent + + tx._add_callback(TxEvent.BEFORE_COMMIT, self._on_before_commit, None) + tx._add_callback(TxEvent.BEFORE_ROLLBACK, self._on_before_rollback, None) + + def _on_before_commit(self, tx: "BaseQueryTxContext"): + self.close() + + def _on_before_rollback(self, tx: "BaseQueryTxContext"): + self.close(flush=False) diff --git a/contrib/python/ydb/py3/ydb/aio/driver.py b/contrib/python/ydb/py3/ydb/aio/driver.py index 9cd6fd2b74..267997fbcc 100644 --- a/contrib/python/ydb/py3/ydb/aio/driver.py +++ b/contrib/python/ydb/py3/ydb/aio/driver.py @@ -62,4 +62,5 @@ class Driver(pool.ConnectionPool): async def stop(self, timeout=10): await self.table_client._stop_pool_if_needed(timeout=timeout) + self.topic_client.close() await super().stop(timeout=timeout) diff --git a/contrib/python/ydb/py3/ydb/aio/query/pool.py b/contrib/python/ydb/py3/ydb/aio/query/pool.py index 947db65872..f1ca68d1cf 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/pool.py +++ b/contrib/python/ydb/py3/ydb/aio/query/pool.py @@ -158,6 +158,8 @@ class QuerySessionPool: async def wrapped_callee(): async with self.checkout() as session: async with session.transaction(tx_mode=tx_mode) as tx: + if tx_mode.name in ["serializable_read_write", "snapshot_read_only"]: + await tx.begin() result = await callee(tx, *args, **kwargs) await tx.commit() return result @@ -213,12 +215,6 @@ class QuerySessionPool: async def __aexit__(self, exc_type, exc_val, exc_tb): await self.stop() - def __del__(self): - if self._should_stop.is_set() or self._loop.is_closed(): - return - - self._loop.call_soon(self.stop) - class SimpleQuerySessionCheckoutAsync: def __init__(self, pool: QuerySessionPool): diff --git a/contrib/python/ydb/py3/ydb/aio/query/transaction.py b/contrib/python/ydb/py3/ydb/aio/query/transaction.py index 5b63a32b48..f0547e5f01 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/aio/query/transaction.py @@ -16,6 +16,28 @@ logger = logging.getLogger(__name__) class QueryTxContext(BaseQueryTxContext): + def __init__(self, driver, session_state, session, tx_mode): + """ + An object that provides a simple transaction context manager that allows statements execution + in a transaction. You don't have to open transaction explicitly, because context manager encapsulates + transaction control logic, and opens new transaction if: + + 1) By explicit .begin() method; + 2) On execution of a first statement, which is strictly recommended method, because that avoids useless round trip + + This context manager is not thread-safe, so you should not manipulate on it concurrently. + + :param driver: A driver instance + :param session_state: A state of session + :param tx_mode: Transaction mode, which is a one from the following choises: + 1) QuerySerializableReadWrite() which is default mode; + 2) QueryOnlineReadOnly(allow_inconsistent_reads=False); + 3) QuerySnapshotReadOnly(); + 4) QueryStaleReadOnly(). + """ + super().__init__(driver, session_state, session, tx_mode) + self._init_callback_handler(base.CallbackHandlerMode.ASYNC) + async def __aenter__(self) -> "QueryTxContext": """ Enters a context manager and returns a transaction @@ -30,7 +52,7 @@ class QueryTxContext(BaseQueryTxContext): it is not finished explicitly """ await self._ensure_prev_stream_finished() - if self._tx_state._state == QueryTxStateEnum.BEGINED: + if self._tx_state._state == QueryTxStateEnum.BEGINED and self._external_error is None: # It's strictly recommended to close transactions directly # by using commit_tx=True flag while executing statement or by # .commit() or .rollback() methods, but here we trying to do best @@ -65,7 +87,9 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): + self._check_external_error_set() + + if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -74,7 +98,13 @@ class QueryTxContext(BaseQueryTxContext): await self._ensure_prev_stream_finished() - await self._commit_call(settings) + try: + await self._execute_callbacks_async(base.TxEvent.BEFORE_COMMIT) + await self._commit_call(settings) + await self._execute_callbacks_async(base.TxEvent.AFTER_COMMIT, exc=None) + except BaseException as e: + await self._execute_callbacks_async(base.TxEvent.AFTER_COMMIT, exc=e) + raise e async def rollback(self, settings: Optional[BaseRequestSettings] = None) -> None: """Calls rollback on a transaction if it is open otherwise is no-op. If transaction execution @@ -84,7 +114,9 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - if self._tx_state._already_in(QueryTxStateEnum.ROLLBACKED): + self._check_external_error_set() + + if self._tx_state._should_skip(QueryTxStateEnum.ROLLBACKED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -93,7 +125,13 @@ class QueryTxContext(BaseQueryTxContext): await self._ensure_prev_stream_finished() - await self._rollback_call(settings) + try: + await self._execute_callbacks_async(base.TxEvent.BEFORE_ROLLBACK) + await self._rollback_call(settings) + await self._execute_callbacks_async(base.TxEvent.AFTER_ROLLBACK, exc=None) + except BaseException as e: + await self._execute_callbacks_async(base.TxEvent.AFTER_ROLLBACK, exc=e) + raise e async def execute( self, diff --git a/contrib/python/ydb/py3/ydb/driver.py b/contrib/python/ydb/py3/ydb/driver.py index 49bd223c90..3998aeeef5 100644 --- a/contrib/python/ydb/py3/ydb/driver.py +++ b/contrib/python/ydb/py3/ydb/driver.py @@ -288,4 +288,5 @@ class Driver(pool.ConnectionPool): def stop(self, timeout=10): self.table_client._stop_pool_if_needed(timeout=timeout) + self.topic_client.close() super().stop(timeout=timeout) diff --git a/contrib/python/ydb/py3/ydb/issues.py b/contrib/python/ydb/py3/ydb/issues.py index f38f99f925..4e76f5ed2b 100644 --- a/contrib/python/ydb/py3/ydb/issues.py +++ b/contrib/python/ydb/py3/ydb/issues.py @@ -178,6 +178,10 @@ class SessionPoolEmpty(Error, queue.Empty): status = StatusCode.SESSION_POOL_EMPTY +class ClientInternalError(Error): + status = StatusCode.CLIENT_INTERNAL_ERROR + + class UnexpectedGrpcMessage(Error): def __init__(self, message: str): super().__init__(message) diff --git a/contrib/python/ydb/py3/ydb/query/base.py b/contrib/python/ydb/py3/ydb/query/base.py index 57a769bb1a..a5ebedd95b 100644 --- a/contrib/python/ydb/py3/ydb/query/base.py +++ b/contrib/python/ydb/py3/ydb/query/base.py @@ -1,6 +1,8 @@ import abc +import asyncio import enum import functools +from collections import defaultdict import typing from typing import ( @@ -17,6 +19,10 @@ from .. import issues from .. import _utilities from .. import _apis +from ydb._topic_common.common import CallFromSyncToAsync, _get_shared_event_loop +from ydb._grpc.grpcwrapper.common_utils import to_thread + + if typing.TYPE_CHECKING: from .transaction import BaseQueryTxContext @@ -196,3 +202,64 @@ def wrap_execute_query_response( return convert.ResultSet.from_message(response_pb.result_set, settings) return None + + +class TxEvent(enum.Enum): + BEFORE_COMMIT = "BEFORE_COMMIT" + AFTER_COMMIT = "AFTER_COMMIT" + BEFORE_ROLLBACK = "BEFORE_ROLLBACK" + AFTER_ROLLBACK = "AFTER_ROLLBACK" + + +class CallbackHandlerMode(enum.Enum): + SYNC = "SYNC" + ASYNC = "ASYNC" + + +def _get_sync_callback(method: typing.Callable, loop: Optional[asyncio.AbstractEventLoop]): + if asyncio.iscoroutinefunction(method): + if loop is None: + loop = _get_shared_event_loop() + + def async_to_sync_callback(*args, **kwargs): + caller = CallFromSyncToAsync(loop) + return caller.safe_call_with_result(method(*args, **kwargs), 10) + + return async_to_sync_callback + return method + + +def _get_async_callback(method: typing.Callable): + if asyncio.iscoroutinefunction(method): + return method + + async def sync_to_async_callback(*args, **kwargs): + return await to_thread(method, *args, **kwargs, executor=None) + + return sync_to_async_callback + + +class CallbackHandler: + def _init_callback_handler(self, mode: CallbackHandlerMode) -> None: + self._callbacks = defaultdict(list) + self._callback_mode = mode + + def _execute_callbacks_sync(self, event_name: str, *args, **kwargs) -> None: + for callback in self._callbacks[event_name]: + callback(self, *args, **kwargs) + + async def _execute_callbacks_async(self, event_name: str, *args, **kwargs) -> None: + tasks = [asyncio.create_task(callback(self, *args, **kwargs)) for callback in self._callbacks[event_name]] + if not tasks: + return + await asyncio.gather(*tasks) + + def _prepare_callback( + self, callback: typing.Callable, loop: Optional[asyncio.AbstractEventLoop] + ) -> typing.Callable: + if self._callback_mode == CallbackHandlerMode.SYNC: + return _get_sync_callback(callback, loop) + return _get_async_callback(callback) + + def _add_callback(self, event_name: str, callback: typing.Callable, loop: Optional[asyncio.AbstractEventLoop]): + self._callbacks[event_name].append(self._prepare_callback(callback, loop)) diff --git a/contrib/python/ydb/py3/ydb/query/pool.py b/contrib/python/ydb/py3/ydb/query/pool.py index e3775c4dd1..b25f7db855 100644 --- a/contrib/python/ydb/py3/ydb/query/pool.py +++ b/contrib/python/ydb/py3/ydb/query/pool.py @@ -167,6 +167,8 @@ class QuerySessionPool: def wrapped_callee(): with self.checkout(timeout=retry_settings.max_session_acquire_timeout) as session: with session.transaction(tx_mode=tx_mode) as tx: + if tx_mode.name in ["serializable_read_write", "snapshot_read_only"]: + tx.begin() result = callee(tx, *args, **kwargs) tx.commit() return result @@ -224,9 +226,6 @@ class QuerySessionPool: def __exit__(self, exc_type, exc_val, exc_tb): self.stop() - def __del__(self): - self.stop() - class SimpleQuerySessionCheckout: def __init__(self, pool: QuerySessionPool, timeout: Optional[float]): diff --git a/contrib/python/ydb/py3/ydb/query/transaction.py b/contrib/python/ydb/py3/ydb/query/transaction.py index 414401da4d..ae7642dbe2 100644 --- a/contrib/python/ydb/py3/ydb/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/query/transaction.py @@ -11,6 +11,7 @@ from .. import ( _apis, issues, ) +from .._grpc.grpcwrapper import ydb_topic as _ydb_topic from .._grpc.grpcwrapper import ydb_query as _ydb_query from ..connection import _RpcState as RpcState @@ -42,11 +43,23 @@ class QueryTxStateHelper(abc.ABC): QueryTxStateEnum.DEAD: [], } + _SKIP_TRANSITIONS = { + QueryTxStateEnum.NOT_INITIALIZED: [], + QueryTxStateEnum.BEGINED: [], + QueryTxStateEnum.COMMITTED: [QueryTxStateEnum.COMMITTED, QueryTxStateEnum.ROLLBACKED], + QueryTxStateEnum.ROLLBACKED: [QueryTxStateEnum.COMMITTED, QueryTxStateEnum.ROLLBACKED], + QueryTxStateEnum.DEAD: [], + } + @classmethod def valid_transition(cls, before: QueryTxStateEnum, after: QueryTxStateEnum) -> bool: return after in cls._VALID_TRANSITIONS[before] @classmethod + def should_skip(cls, before: QueryTxStateEnum, after: QueryTxStateEnum) -> bool: + return after in cls._SKIP_TRANSITIONS[before] + + @classmethod def terminal(cls, state: QueryTxStateEnum) -> bool: return len(cls._VALID_TRANSITIONS[state]) == 0 @@ -88,8 +101,8 @@ class QueryTxState: if QueryTxStateHelper.terminal(self._state): raise RuntimeError(f"Transaction is in terminal state: {self._state.value}") - def _already_in(self, target: QueryTxStateEnum) -> bool: - return self._state == target + def _should_skip(self, target: QueryTxStateEnum) -> bool: + return QueryTxStateHelper.should_skip(self._state, target) def _construct_tx_settings(tx_state: QueryTxState) -> _ydb_query.TransactionSettings: @@ -170,7 +183,7 @@ def wrap_tx_rollback_response( return tx -class BaseQueryTxContext: +class BaseQueryTxContext(base.CallbackHandler): def __init__(self, driver, session_state, session, tx_mode): """ An object that provides a simple transaction context manager that allows statements execution @@ -196,6 +209,7 @@ class BaseQueryTxContext: self._session_state = session_state self.session = session self._prev_stream = None + self._external_error = None @property def session_id(self) -> str: @@ -215,6 +229,19 @@ class BaseQueryTxContext: """ return self._tx_state.tx_id + def _tx_identity(self) -> _ydb_topic.TransactionIdentity: + if not self.tx_id: + raise RuntimeError("Unable to get tx identity without started tx.") + return _ydb_topic.TransactionIdentity(self.tx_id, self.session_id) + + def _set_external_error(self, exc: BaseException) -> None: + self._external_error = exc + + def _check_external_error_set(self): + if self._external_error is None: + return + raise issues.ClientInternalError("Transaction was failed by external error.") from self._external_error + def _begin_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": self._tx_state._check_invalid_transition(QueryTxStateEnum.BEGINED) @@ -228,6 +255,7 @@ class BaseQueryTxContext: ) def _commit_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": + self._check_external_error_set() self._tx_state._check_invalid_transition(QueryTxStateEnum.COMMITTED) return self._driver( @@ -240,6 +268,7 @@ class BaseQueryTxContext: ) def _rollback_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": + self._check_external_error_set() self._tx_state._check_invalid_transition(QueryTxStateEnum.ROLLBACKED) return self._driver( @@ -262,6 +291,7 @@ class BaseQueryTxContext: settings: Optional[BaseRequestSettings], ) -> Iterable[_apis.ydb_query.ExecuteQueryResponsePart]: self._tx_state._check_tx_ready_to_use() + self._check_external_error_set() request = base.create_execute_query_request( query=query, @@ -283,18 +313,41 @@ class BaseQueryTxContext: ) def _move_to_beginned(self, tx_id: str) -> None: - if self._tx_state._already_in(QueryTxStateEnum.BEGINED) or not tx_id: + if self._tx_state._should_skip(QueryTxStateEnum.BEGINED) or not tx_id: return self._tx_state._change_state(QueryTxStateEnum.BEGINED) self._tx_state.tx_id = tx_id def _move_to_commited(self) -> None: - if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): + if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): return self._tx_state._change_state(QueryTxStateEnum.COMMITTED) class QueryTxContext(BaseQueryTxContext): + def __init__(self, driver, session_state, session, tx_mode): + """ + An object that provides a simple transaction context manager that allows statements execution + in a transaction. You don't have to open transaction explicitly, because context manager encapsulates + transaction control logic, and opens new transaction if: + + 1) By explicit .begin() method; + 2) On execution of a first statement, which is strictly recommended method, because that avoids useless round trip + + This context manager is not thread-safe, so you should not manipulate on it concurrently. + + :param driver: A driver instance + :param session_state: A state of session + :param tx_mode: Transaction mode, which is a one from the following choises: + 1) QuerySerializableReadWrite() which is default mode; + 2) QueryOnlineReadOnly(allow_inconsistent_reads=False); + 3) QuerySnapshotReadOnly(); + 4) QueryStaleReadOnly(). + """ + + super().__init__(driver, session_state, session, tx_mode) + self._init_callback_handler(base.CallbackHandlerMode.SYNC) + def __enter__(self) -> "BaseQueryTxContext": """ Enters a context manager and returns a transaction @@ -309,7 +362,7 @@ class QueryTxContext(BaseQueryTxContext): it is not finished explicitly """ self._ensure_prev_stream_finished() - if self._tx_state._state == QueryTxStateEnum.BEGINED: + if self._tx_state._state == QueryTxStateEnum.BEGINED and self._external_error is None: # It's strictly recommended to close transactions directly # by using commit_tx=True flag while executing statement or by # .commit() or .rollback() methods, but here we trying to do best @@ -345,7 +398,8 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): + self._check_external_error_set() + if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -354,7 +408,13 @@ class QueryTxContext(BaseQueryTxContext): self._ensure_prev_stream_finished() - self._commit_call(settings) + try: + self._execute_callbacks_sync(base.TxEvent.BEFORE_COMMIT) + self._commit_call(settings) + self._execute_callbacks_sync(base.TxEvent.AFTER_COMMIT, exc=None) + except BaseException as e: # TODO: probably should be less wide + self._execute_callbacks_sync(base.TxEvent.AFTER_COMMIT, exc=e) + raise e def rollback(self, settings: Optional[BaseRequestSettings] = None) -> None: """Calls rollback on a transaction if it is open otherwise is no-op. If transaction execution @@ -364,7 +424,8 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - if self._tx_state._already_in(QueryTxStateEnum.ROLLBACKED): + self._check_external_error_set() + if self._tx_state._should_skip(QueryTxStateEnum.ROLLBACKED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -373,7 +434,13 @@ class QueryTxContext(BaseQueryTxContext): self._ensure_prev_stream_finished() - self._rollback_call(settings) + try: + self._execute_callbacks_sync(base.TxEvent.BEFORE_ROLLBACK) + self._rollback_call(settings) + self._execute_callbacks_sync(base.TxEvent.AFTER_ROLLBACK, exc=None) + except BaseException as e: # TODO: probably should be less wide + self._execute_callbacks_sync(base.TxEvent.AFTER_ROLLBACK, exc=e) + raise e def execute( self, diff --git a/contrib/python/ydb/py3/ydb/table.py b/contrib/python/ydb/py3/ydb/table.py index 945e918767..ac73902f3c 100644 --- a/contrib/python/ydb/py3/ydb/table.py +++ b/contrib/python/ydb/py3/ydb/table.py @@ -545,6 +545,9 @@ class TableStats(object): def __init__(self): self.partitions = None self.store_size = 0 + self.rows_estimate = 0 + self.creation_time = None + self.modification_time = None def with_store_size(self, store_size): self.store_size = store_size @@ -554,6 +557,18 @@ class TableStats(object): self.partitions = partitions return self + def with_rows_estimate(self, rows_estimate): + self.rows_estimate = rows_estimate + return self + + def with_creation_time(self, creation_time): + self.creation_time = creation_time + return self + + def with_modification_time(self, modification_time): + self.modification_time = modification_time + return self + class ReadReplicasSettings(object): def __init__(self): @@ -1577,7 +1592,22 @@ class TableSchemeEntry(scheme.SchemeEntry): self.table_stats = None if table_stats is not None: + from ._grpc.grpcwrapper.common_utils import datetime_from_proto_timestamp + self.table_stats = TableStats() + if table_stats.creation_time: + self.table_stats = self.table_stats.with_creation_time( + datetime_from_proto_timestamp(table_stats.creation_time) + ) + + if table_stats.modification_time: + self.table_stats = self.table_stats.with_modification_time( + datetime_from_proto_timestamp(table_stats.modification_time) + ) + + if table_stats.rows_estimate != 0: + self.table_stats = self.table_stats.with_rows_estimate(table_stats.rows_estimate) + if table_stats.partitions != 0: self.table_stats = self.table_stats.with_partitions(table_stats.partitions) diff --git a/contrib/python/ydb/py3/ydb/topic.py b/contrib/python/ydb/py3/ydb/topic.py index 55f4ea04c5..a501f9d275 100644 --- a/contrib/python/ydb/py3/ydb/topic.py +++ b/contrib/python/ydb/py3/ydb/topic.py @@ -25,6 +25,8 @@ __all__ = [ "TopicWriteResult", "TopicWriter", "TopicWriterAsyncIO", + "TopicTxWriter", + "TopicTxWriterAsyncIO", "TopicWriterInitInfo", "TopicWriterMessage", "TopicWriterSettings", @@ -33,6 +35,7 @@ __all__ = [ import concurrent.futures import datetime from dataclasses import dataclass +import logging from typing import List, Union, Mapping, Optional, Dict, Callable from . import aio, Credentials, _apis, issues @@ -65,8 +68,10 @@ from ._topic_writer.topic_writer import ( # noqa: F401 PublicWriteResult as TopicWriteResult, ) +from ydb._topic_writer.topic_writer_asyncio import TxWriterAsyncIO as TopicTxWriterAsyncIO from ydb._topic_writer.topic_writer_asyncio import WriterAsyncIO as TopicWriterAsyncIO from ._topic_writer.topic_writer_sync import WriterSync as TopicWriter +from ._topic_writer.topic_writer_sync import TxWriterSync as TopicTxWriter from ._topic_common.common import ( wrap_operation as _wrap_operation, @@ -88,6 +93,8 @@ from ._grpc.grpcwrapper.ydb_topic_public_types import ( # noqa: F401 PublicAlterAutoPartitioningSettings as TopicAlterAutoPartitioningSettings, ) +logger = logging.getLogger(__name__) + class TopicClientAsyncIO: _closed: bool @@ -108,7 +115,12 @@ class TopicClientAsyncIO: ) def __del__(self): - self.close() + if not self._closed: + try: + logger.warning("Topic client was not closed properly. Consider using method close().") + self.close() + except BaseException: + logger.warning("Something went wrong during topic client close in __del__") async def create_topic( self, @@ -276,6 +288,35 @@ class TopicClientAsyncIO: return TopicWriterAsyncIO(self._driver, settings, _client=self) + def tx_writer( + self, + tx, + topic, + *, + producer_id: Optional[str] = None, # default - random + session_metadata: Mapping[str, str] = None, + partition_id: Union[int, None] = None, + auto_seqno: bool = True, + auto_created_at: bool = True, + codec: Optional[TopicCodec] = None, # default mean auto-select + # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes + # the func will be called from multiply threads in parallel. + encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, + # custom encoder executor for call builtin and custom decoders. If None - use shared executor pool. + # If max_worker in the executor is 1 - then encoders will be called from the thread without parallel. + encoder_executor: Optional[concurrent.futures.Executor] = None, + ) -> TopicTxWriterAsyncIO: + args = locals().copy() + del args["self"] + del args["tx"] + + settings = TopicWriterSettings(**args) + + if not settings.encoder_executor: + settings.encoder_executor = self._executor + + return TopicTxWriterAsyncIO(tx=tx, driver=self._driver, settings=settings, _client=self) + def close(self): if self._closed: return @@ -287,7 +328,7 @@ class TopicClientAsyncIO: if not self._closed: return - raise RuntimeError("Topic client closed") + raise issues.Error("Topic client closed") class TopicClient: @@ -310,7 +351,12 @@ class TopicClient: ) def __del__(self): - self.close() + if not self._closed: + try: + logger.warning("Topic client was not closed properly. Consider using method close().") + self.close() + except BaseException: + logger.warning("Something went wrong during topic client close in __del__") def create_topic( self, @@ -487,6 +533,36 @@ class TopicClient: return TopicWriter(self._driver, settings, _parent=self) + def tx_writer( + self, + tx, + topic, + *, + producer_id: Optional[str] = None, # default - random + session_metadata: Mapping[str, str] = None, + partition_id: Union[int, None] = None, + auto_seqno: bool = True, + auto_created_at: bool = True, + codec: Optional[TopicCodec] = None, # default mean auto-select + # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes + # the func will be called from multiply threads in parallel. + encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, + # custom encoder executor for call builtin and custom decoders. If None - use shared executor pool. + # If max_worker in the executor is 1 - then encoders will be called from the thread without parallel. + encoder_executor: Optional[concurrent.futures.Executor] = None, # default shared client executor pool + ) -> TopicWriter: + args = locals().copy() + del args["self"] + del args["tx"] + self._check_closed() + + settings = TopicWriterSettings(**args) + + if not settings.encoder_executor: + settings.encoder_executor = self._executor + + return TopicTxWriter(tx, self._driver, settings, _parent=self) + def close(self): if self._closed: return @@ -498,7 +574,7 @@ class TopicClient: if not self._closed: return - raise RuntimeError("Topic client closed") + raise issues.Error("Topic client closed") @dataclass diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index 8bd658d49e..4a5c580f99 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.19.3" +VERSION = "3.20.1" |