diff options
Diffstat (limited to 'contrib/python')
47 files changed, 2408 insertions, 1544 deletions
diff --git a/contrib/python/multidict/.dist-info/METADATA b/contrib/python/multidict/.dist-info/METADATA index 93f85177b97..b5c6dad90ba 100644 --- a/contrib/python/multidict/.dist-info/METADATA +++ b/contrib/python/multidict/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: multidict -Version: 6.1.0 +Version: 6.2.0 Summary: multidict implementation Home-page: https://github.com/aio-libs/multidict Author: Andrew Svetlov @@ -20,16 +20,15 @@ Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE -Requires-Dist: typing-extensions >=4.1.0 ; python_version < "3.11" +Requires-Dist: typing-extensions>=4.1.0; python_version < "3.11" ========= multidict diff --git a/contrib/python/multidict/multidict/__init__.py b/contrib/python/multidict/multidict/__init__.py index 25ddca41e9b..b6b532a1f25 100644 --- a/contrib/python/multidict/multidict/__init__.py +++ b/contrib/python/multidict/multidict/__init__.py @@ -5,6 +5,8 @@ multidict. It behaves mostly like a dict but it can have several values for the same key. """ +from typing import TYPE_CHECKING + from ._abc import MultiMapping, MutableMultiMapping from ._compat import USE_EXTENSIONS @@ -20,13 +22,11 @@ __all__ = ( "getversion", ) -__version__ = "6.1.0" +__version__ = "6.2.0" -try: - if not USE_EXTENSIONS: - raise ImportError - from ._multidict import ( +if TYPE_CHECKING or not USE_EXTENSIONS: + from ._multidict_py import ( CIMultiDict, CIMultiDictProxy, MultiDict, @@ -34,8 +34,8 @@ try: getversion, istr, ) -except ImportError: # pragma: no cover - from ._multidict_py import ( +else: + from ._multidict import ( CIMultiDict, CIMultiDictProxy, MultiDict, diff --git a/contrib/python/multidict/multidict/__init__.pyi b/contrib/python/multidict/multidict/__init__.pyi deleted file mode 100644 index 0940340f81e..00000000000 --- a/contrib/python/multidict/multidict/__init__.pyi +++ /dev/null @@ -1,152 +0,0 @@ -import abc -from typing import ( - Generic, - Iterable, - Iterator, - Mapping, - MutableMapping, - TypeVar, - overload, -) - -class istr(str): ... - -upstr = istr - -_S = str | istr - -_T = TypeVar("_T") - -_T_co = TypeVar("_T_co", covariant=True) - -_D = TypeVar("_D") - -class MultiMapping(Mapping[_S, _T_co]): - @overload - @abc.abstractmethod - def getall(self, key: _S) -> list[_T_co]: ... - @overload - @abc.abstractmethod - def getall(self, key: _S, default: _D) -> list[_T_co] | _D: ... - @overload - @abc.abstractmethod - def getone(self, key: _S) -> _T_co: ... - @overload - @abc.abstractmethod - def getone(self, key: _S, default: _D) -> _T_co | _D: ... - -_Arg = ( - Mapping[str, _T] - | Mapping[istr, _T] - | dict[str, _T] - | dict[istr, _T] - | MultiMapping[_T] - | Iterable[tuple[str, _T]] - | Iterable[tuple[istr, _T]] -) - -class MutableMultiMapping(MultiMapping[_T], MutableMapping[_S, _T], Generic[_T]): - @abc.abstractmethod - def add(self, key: _S, value: _T) -> None: ... - @abc.abstractmethod - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - @abc.abstractmethod - def popone(self, key: _S) -> _T: ... - @overload - @abc.abstractmethod - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - @abc.abstractmethod - def popall(self, key: _S) -> list[_T]: ... - @overload - @abc.abstractmethod - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class MultiDict(MutableMultiMapping[_T], Generic[_T]): - def __init__(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - def copy(self) -> MultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __setitem__(self, k: _S, v: _T) -> None: ... - def __delitem__(self, v: _S) -> None: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def add(self, key: _S, value: _T) -> None: ... - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - def popone(self, key: _S) -> _T: ... - @overload - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - def popall(self, key: _S) -> list[_T]: ... - @overload - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class CIMultiDict(MutableMultiMapping[_T], Generic[_T]): - def __init__(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - def copy(self) -> CIMultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __setitem__(self, k: _S, v: _T) -> None: ... - def __delitem__(self, v: _S) -> None: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def add(self, key: _S, value: _T) -> None: ... - def extend(self, arg: _Arg[_T] = ..., **kwargs: _T) -> None: ... - @overload - def popone(self, key: _S) -> _T: ... - @overload - def popone(self, key: _S, default: _D) -> _T | _D: ... - @overload - def popall(self, key: _S) -> list[_T]: ... - @overload - def popall(self, key: _S, default: _D) -> list[_T] | _D: ... - -class MultiDictProxy(MultiMapping[_T], Generic[_T]): - def __init__(self, arg: MultiMapping[_T] | MutableMultiMapping[_T]) -> None: ... - def copy(self) -> MultiDict[_T]: ... - def __getitem__(self, k: _S) -> _T: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - -class CIMultiDictProxy(MultiMapping[_T], Generic[_T]): - def __init__(self, arg: MultiMapping[_T] | MutableMultiMapping[_T]) -> None: ... - def __getitem__(self, k: _S) -> _T: ... - def __iter__(self) -> Iterator[_S]: ... - def __len__(self) -> int: ... - @overload - def getall(self, key: _S) -> list[_T]: ... - @overload - def getall(self, key: _S, default: _D) -> list[_T] | _D: ... - @overload - def getone(self, key: _S) -> _T: ... - @overload - def getone(self, key: _S, default: _D) -> _T | _D: ... - def copy(self) -> CIMultiDict[_T]: ... - -def getversion( - md: MultiDict[_T] | CIMultiDict[_T] | MultiDictProxy[_T] | CIMultiDictProxy[_T], -) -> int: ... diff --git a/contrib/python/multidict/multidict/_abc.py b/contrib/python/multidict/multidict/_abc.py index 0603cdd2447..ff0e2a6976c 100644 --- a/contrib/python/multidict/multidict/_abc.py +++ b/contrib/python/multidict/multidict/_abc.py @@ -1,48 +1,69 @@ import abc -import sys -import types -from collections.abc import Mapping, MutableMapping +from collections.abc import Iterable, Mapping, MutableMapping +from typing import TYPE_CHECKING, Protocol, TypeVar, Union, overload +if TYPE_CHECKING: + from ._multidict_py import istr +else: + istr = str -class _TypingMeta(abc.ABCMeta): - # A fake metaclass to satisfy typing deps in runtime - # basically MultiMapping[str] and other generic-like type instantiations - # are emulated. - # Note: real type hints are provided by __init__.pyi stub file - if sys.version_info >= (3, 9): +_V = TypeVar("_V") +_V_co = TypeVar("_V_co", covariant=True) +_T = TypeVar("_T") - def __getitem__(self, key): - return types.GenericAlias(self, key) - else: +class SupportsKeys(Protocol[_V_co]): + def keys(self) -> Iterable[str]: ... + def __getitem__(self, key: str, /) -> _V_co: ... - def __getitem__(self, key): - return self +class SupportsIKeys(Protocol[_V_co]): + def keys(self) -> Iterable[istr]: ... + def __getitem__(self, key: istr, /) -> _V_co: ... -class MultiMapping(Mapping, metaclass=_TypingMeta): + +MDArg = Union[SupportsKeys[_V], SupportsIKeys[_V], Iterable[tuple[str, _V]], None] + + +class MultiMapping(Mapping[str, _V_co]): + @overload + def getall(self, key: str) -> list[_V_co]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V_co], _T]: ... @abc.abstractmethod - def getall(self, key, default=None): - raise KeyError + def getall(self, key: str, default: _T = ...) -> Union[list[_V_co], _T]: + """Return all values for key.""" + @overload + def getone(self, key: str) -> _V_co: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V_co, _T]: ... @abc.abstractmethod - def getone(self, key, default=None): - raise KeyError + def getone(self, key: str, default: _T = ...) -> Union[_V_co, _T]: + """Return first value for key.""" -class MutableMultiMapping(MultiMapping, MutableMapping): +class MutableMultiMapping(MultiMapping[_V], MutableMapping[str, _V]): @abc.abstractmethod - def add(self, key, value): - raise NotImplementedError + def add(self, key: str, value: _V) -> None: + """Add value to list.""" @abc.abstractmethod - def extend(self, *args, **kwargs): - raise NotImplementedError + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Add everything from arg and kwargs to the mapping.""" + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... @abc.abstractmethod - def popone(self, key, default=None): - raise KeyError + def popone(self, key: str, default: _T = ...) -> Union[_V, _T]: + """Remove specified key and return the corresponding value.""" + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... @abc.abstractmethod - def popall(self, key, default=None): - raise KeyError + def popall(self, key: str, default: _T = ...) -> Union[list[_V], _T]: + """Remove all occurrences of key and return the list of corresponding values.""" diff --git a/contrib/python/multidict/multidict/_multidict.c b/contrib/python/multidict/multidict/_multidict.c index 60864953b16..ebb1949f0a7 100644 --- a/contrib/python/multidict/multidict/_multidict.c +++ b/contrib/python/multidict/multidict/_multidict.c @@ -1,6 +1,8 @@ #include "Python.h" #include "structmember.h" +#include "_multilib/pythoncapi_compat.h" + // Include order important #include "_multilib/defs.h" #include "_multilib/istr.h" @@ -9,7 +11,7 @@ #include "_multilib/iter.h" #include "_multilib/views.h" -#if PY_MAJOR_VERSION < 3 || PY_MINOR_VERSION < 12 +#if PY_MINOR_VERSION < 12 #ifndef _PyArg_UnpackKeywords #define FASTCALL_OLD #endif @@ -19,14 +21,13 @@ static PyObject *collections_abc_mapping; static PyObject *collections_abc_mut_mapping; static PyObject *collections_abc_mut_multi_mapping; +static PyObject *repr_func; static PyTypeObject multidict_type; static PyTypeObject cimultidict_type; static PyTypeObject multidict_proxy_type; static PyTypeObject cimultidict_proxy_type; -static PyObject *repr_func; - #define MultiDict_CheckExact(o) (Py_TYPE(o) == &multidict_type) #define CIMultiDict_CheckExact(o) (Py_TYPE(o) == &cimultidict_type) #define MultiDictProxy_CheckExact(o) (Py_TYPE(o) == &multidict_proxy_type) @@ -155,13 +156,17 @@ _multidict_append_items_seq(MultiDictObject *self, PyObject *arg, Py_INCREF(value); } else if (PyList_CheckExact(item)) { - if (PyList_GET_SIZE(item) != 2) { + if (PyList_Size(item) != 2) { + goto invalid_type; + } + key = PyList_GetItemRef(item, 0); + if (key == NULL) { + goto invalid_type; + } + value = PyList_GetItemRef(item, 1); + if (value == NULL) { goto invalid_type; } - key = PyList_GET_ITEM(item, 0); - Py_INCREF(key); - value = PyList_GET_ITEM(item, 1); - Py_INCREF(value); } else if (PySequence_Check(item)) { if (PySequence_Size(item) != 2) { @@ -339,8 +344,8 @@ _multidict_extend(MultiDictObject *self, PyObject *args, PyObject *kwds, if (args && PyObject_Length(args) > 1) { PyErr_Format( PyExc_TypeError, - "%s takes at most 1 positional argument (%zd given)", - name, PyObject_Length(args), NULL + "%s takes from 1 to 2 positional arguments but %zd were given", + name, PyObject_Length(args) + 1, NULL ); return -1; } @@ -769,21 +774,13 @@ static inline void multidict_tp_dealloc(MultiDictObject *self) { PyObject_GC_UnTrack(self); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_TRASHCAN_BEGIN(self, multidict_tp_dealloc) -#else - Py_TRASHCAN_SAFE_BEGIN(self); -#endif if (self->weaklist != NULL) { PyObject_ClearWeakRefs((PyObject *)self); }; pair_list_dealloc(&self->pairs); Py_TYPE(self)->tp_free((PyObject *)self); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_TRASHCAN_END // there should be no code after this -#else - Py_TRASHCAN_SAFE_END(self); -#endif } static inline int @@ -1230,16 +1227,7 @@ PyDoc_STRVAR(multidict_update_doc, "Update the dictionary from *other*, overwriting existing keys."); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 #define multidict_class_getitem Py_GenericAlias -#else -static inline PyObject * -multidict_class_getitem(PyObject *self, PyObject *arg) -{ - Py_INCREF(self); - return self; -} -#endif PyDoc_STRVAR(sizeof__doc__, @@ -1941,9 +1929,7 @@ getversion(PyObject *self, PyObject *md) static inline void module_free(void *m) { -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_CLEAR(multidict_str_lower); -#endif Py_CLEAR(collections_abc_mapping); Py_CLEAR(collections_abc_mut_mapping); Py_CLEAR(collections_abc_mut_multi_mapping); @@ -1972,29 +1958,14 @@ static PyModuleDef multidict_module = { PyMODINIT_FUNC PyInit__multidict(void) { -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 multidict_str_lower = PyUnicode_InternFromString("lower"); if (multidict_str_lower == NULL) { goto fail; } -#endif PyObject *module = NULL, *reg_func_call_result = NULL; -#define WITH_MOD(NAME) \ - Py_CLEAR(module); \ - module = PyImport_ImportModule(NAME); \ - if (module == NULL) { \ - goto fail; \ - } - -#define GET_MOD_ATTR(VAR, NAME) \ - VAR = PyObject_GetAttrString(module, NAME); \ - if (VAR == NULL) { \ - goto fail; \ - } - if (multidict_views_init() < 0) { goto fail; } @@ -2015,18 +1986,31 @@ PyInit__multidict(void) goto fail; } +#define WITH_MOD(NAME) \ + Py_CLEAR(module); \ + module = PyImport_ImportModule(NAME); \ + if (module == NULL) { \ + goto fail; \ + } + +#define GET_MOD_ATTR(VAR, NAME) \ + VAR = PyObject_GetAttrString(module, NAME); \ + if (VAR == NULL) { \ + goto fail; \ + } + WITH_MOD("collections.abc"); GET_MOD_ATTR(collections_abc_mapping, "Mapping"); WITH_MOD("multidict._abc"); GET_MOD_ATTR(collections_abc_mut_mapping, "MultiMapping"); - - WITH_MOD("multidict._abc"); GET_MOD_ATTR(collections_abc_mut_multi_mapping, "MutableMultiMapping"); WITH_MOD("multidict._multidict_base"); GET_MOD_ATTR(repr_func, "_mdrepr"); + Py_CLEAR(module); \ + /* Register in _abc mappings (CI)MultiDict and (CI)MultiDictProxy */ reg_func_call_result = PyObject_CallMethod( collections_abc_mut_mapping, @@ -2070,6 +2054,13 @@ PyInit__multidict(void) /* Instantiate this module */ module = PyModule_Create(&multidict_module); + if (module == NULL) { + goto fail; + } + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif Py_INCREF(&istr_type); if (PyModule_AddObject( @@ -2109,9 +2100,7 @@ PyInit__multidict(void) return module; fail: -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 Py_XDECREF(multidict_str_lower); -#endif Py_XDECREF(collections_abc_mapping); Py_XDECREF(collections_abc_mut_mapping); Py_XDECREF(collections_abc_mut_multi_mapping); diff --git a/contrib/python/multidict/multidict/_multidict_base.py b/contrib/python/multidict/multidict/_multidict_base.py index de2f762a5c0..df0d70097a1 100644 --- a/contrib/python/multidict/multidict/_multidict_base.py +++ b/contrib/python/multidict/multidict/_multidict_base.py @@ -1,5 +1,19 @@ import sys -from collections.abc import ItemsView, Iterable, KeysView, Set, ValuesView +from collections.abc import ( + Container, + ItemsView, + Iterable, + KeysView, + Mapping, + Set, + ValuesView, +) +from typing import Literal, Union + +if sys.version_info >= (3, 10): + from types import NotImplementedType +else: + from typing import Any as NotImplementedType if sys.version_info >= (3, 11): from typing import assert_never @@ -7,26 +21,28 @@ else: from typing_extensions import assert_never -def _abc_itemsview_register(view_cls): +def _abc_itemsview_register(view_cls: type[object]) -> None: ItemsView.register(view_cls) -def _abc_keysview_register(view_cls): +def _abc_keysview_register(view_cls: type[object]) -> None: KeysView.register(view_cls) -def _abc_valuesview_register(view_cls): +def _abc_valuesview_register(view_cls: type[object]) -> None: ValuesView.register(view_cls) -def _viewbaseset_richcmp(view, other, op): +def _viewbaseset_richcmp( + view: set[object], other: object, op: Literal[0, 1, 2, 3, 4, 5] +) -> Union[bool, NotImplementedType]: if op == 0: # < if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) < len(other) and view <= other elif op == 1: # <= if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if len(view) > len(other): return False for elem in view: @@ -35,17 +51,17 @@ def _viewbaseset_richcmp(view, other, op): return True elif op == 2: # == if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) == len(other) and view <= other elif op == 3: # != return not view == other elif op == 4: # > if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] return len(view) > len(other) and view >= other elif op == 5: # >= if not isinstance(other, Set): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if len(view) < len(other): return False for elem in other: @@ -56,9 +72,11 @@ def _viewbaseset_richcmp(view, other, op): assert_never(op) -def _viewbaseset_and(view, other): +def _viewbaseset_and( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -68,9 +86,11 @@ def _viewbaseset_and(view, other): return view & other -def _viewbaseset_or(view, other): +def _viewbaseset_or( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -80,9 +100,11 @@ def _viewbaseset_or(view, other): return view | other -def _viewbaseset_sub(view, other): +def _viewbaseset_sub( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -92,9 +114,11 @@ def _viewbaseset_sub(view, other): return view - other -def _viewbaseset_xor(view, other): +def _viewbaseset_xor( + view: set[object], other: object +) -> Union[set[object], NotImplementedType]: if not isinstance(other, Iterable): - return NotImplemented + return NotImplemented # type: ignore[no-any-return] if isinstance(view, Set): view = set(iter(view)) if isinstance(other, Set): @@ -104,7 +128,7 @@ def _viewbaseset_xor(view, other): return view ^ other -def _itemsview_isdisjoint(view, other): +def _itemsview_isdisjoint(view: Container[object], other: Iterable[object]) -> bool: "Return True if two sets have a null intersection." for v in other: if v in view: @@ -112,7 +136,7 @@ def _itemsview_isdisjoint(view, other): return True -def _itemsview_repr(view): +def _itemsview_repr(view: Iterable[tuple[object, object]]) -> str: lst = [] for k, v in view: lst.append("{!r}: {!r}".format(k, v)) @@ -120,7 +144,7 @@ def _itemsview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _keysview_isdisjoint(view, other): +def _keysview_isdisjoint(view: Container[object], other: Iterable[object]) -> bool: "Return True if two sets have a null intersection." for k in other: if k in view: @@ -128,7 +152,7 @@ def _keysview_isdisjoint(view, other): return True -def _keysview_repr(view): +def _keysview_repr(view: Iterable[object]) -> str: lst = [] for k in view: lst.append("{!r}".format(k)) @@ -136,7 +160,7 @@ def _keysview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _valuesview_repr(view): +def _valuesview_repr(view: Iterable[object]) -> str: lst = [] for v in view: lst.append("{!r}".format(v)) @@ -144,7 +168,7 @@ def _valuesview_repr(view): return "{}({})".format(view.__class__.__name__, body) -def _mdrepr(md): +def _mdrepr(md: Mapping[object, object]) -> str: lst = [] for k, v in md.items(): lst.append("'{}': {!r}".format(k, v)) diff --git a/contrib/python/multidict/multidict/_multidict_py.py b/contrib/python/multidict/multidict/_multidict_py.py index 79c45aa19c5..b8ecb8b9623 100644 --- a/contrib/python/multidict/multidict/_multidict_py.py +++ b/contrib/python/multidict/multidict/_multidict_py.py @@ -1,47 +1,56 @@ +import enum import sys -import types from array import array -from collections import abc +from collections.abc import ( + Callable, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + ValuesView, +) +from typing import ( + TYPE_CHECKING, + Generic, + NoReturn, + TypeVar, + Union, + cast, + overload, +) -from ._abc import MultiMapping, MutableMultiMapping +from ._abc import MDArg, MultiMapping, MutableMultiMapping, SupportsKeys -_marker = object() - -if sys.version_info >= (3, 9): - GenericAlias = types.GenericAlias +if sys.version_info >= (3, 11): + from typing import Self else: - - def GenericAlias(cls): - return cls + from typing_extensions import Self class istr(str): - """Case insensitive str.""" __is_istr__ = True -upstr = istr # for relaxing backward compatibility problems - - -def getversion(md): - if not isinstance(md, _Base): - raise TypeError("Parameter should be multidict or proxy") - return md._impl._version +_V = TypeVar("_V") +_T = TypeVar("_T") +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel _version = array("Q", [0]) -class _Impl: +class _Impl(Generic[_V]): __slots__ = ("_items", "_version") - def __init__(self): - self._items = [] + def __init__(self) -> None: + self._items: list[tuple[str, str, _V]] = [] self.incr_version() - def incr_version(self): + def incr_version(self) -> None: global _version v = _version v[0] += 1 @@ -49,25 +58,138 @@ class _Impl: if sys.implementation.name != "pypy": - def __sizeof__(self): + def __sizeof__(self) -> int: return object.__sizeof__(self) + sys.getsizeof(self._items) -class _Base: - def _title(self, key): +class _Iter(Generic[_T]): + __slots__ = ("_size", "_iter") + + def __init__(self, size: int, iterator: Iterator[_T]): + self._size = size + self._iter = iterator + + def __iter__(self) -> Self: + return self + + def __next__(self) -> _T: + return next(self._iter) + + def __length_hint__(self) -> int: + return self._size + + +class _ViewBase(Generic[_V]): + def __init__(self, impl: _Impl[_V]): + self._impl = impl + + def __len__(self) -> int: + return len(self._impl._items) + + +class _ItemsView(_ViewBase[_V], ItemsView[str, _V]): + def __contains__(self, item: object) -> bool: + if not isinstance(item, (tuple, list)) or len(item) != 2: + return False + for i, k, v in self._impl._items: + if item[0] == k and item[1] == v: + return True + return False + + def __iter__(self) -> _Iter[tuple[str, _V]]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[tuple[str, _V]]: + for i, k, v in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield k, v + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}: {!r}".format(item[1], item[2])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _ValuesView(_ViewBase[_V], ValuesView[_V]): + def __contains__(self, value: object) -> bool: + for item in self._impl._items: + if item[2] == value: + return True + return False + + def __iter__(self) -> _Iter[_V]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[_V]: + for item in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield item[2] + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}".format(item[2])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _KeysView(_ViewBase[_V], KeysView[str]): + def __contains__(self, key: object) -> bool: + for item in self._impl._items: + if item[1] == key: + return True + return False + + def __iter__(self) -> _Iter[str]: + return _Iter(len(self), self._iter(self._impl._version)) + + def _iter(self, version: int) -> Iterator[str]: + for item in self._impl._items: + if version != self._impl._version: + raise RuntimeError("Dictionary changed during iteration") + yield item[1] + + def __repr__(self) -> str: + lst = [] + for item in self._impl._items: + lst.append("{!r}".format(item[1])) + body = ", ".join(lst) + return "{}({})".format(self.__class__.__name__, body) + + +class _Base(MultiMapping[_V]): + _impl: _Impl[_V] + + def _title(self, key: str) -> str: return key - def getall(self, key, default=_marker): + @overload + def getall(self, key: str) -> list[_V]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def getall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: """Return a list of all values matching the key.""" identity = self._title(key) res = [v for i, k, v in self._impl._items if i == identity] if res: return res - if not res and default is not _marker: + if not res and default is not sentinel: return default raise KeyError("Key not found: %r" % key) - def getone(self, key, default=_marker): + @overload + def getone(self, key: str) -> _V: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V, _T]: ... + def getone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: """Get first value matching the key. Raises KeyError if the key is not found and no default is provided. @@ -76,42 +198,46 @@ class _Base: for i, k, v in self._impl._items: if i == identity: return v - if default is not _marker: + if default is not sentinel: return default raise KeyError("Key not found: %r" % key) # Mapping interface # - def __getitem__(self, key): + def __getitem__(self, key: str) -> _V: return self.getone(key) - def get(self, key, default=None): + @overload + def get(self, key: str, /) -> Union[_V, None]: ... + @overload + def get(self, key: str, /, default: _T) -> Union[_V, _T]: ... + def get(self, key: str, default: Union[_T, None] = None) -> Union[_V, _T, None]: """Get first value matching the key. If the key is not found, returns the default (or None if no default is provided) """ return self.getone(key, default) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.keys()) - def __len__(self): + def __len__(self) -> int: return len(self._impl._items) - def keys(self): + def keys(self) -> KeysView[str]: """Return a new view of the dictionary's keys.""" return _KeysView(self._impl) - def items(self): + def items(self) -> ItemsView[str, _V]: """Return a new view of the dictionary's items *(key, value) pairs).""" return _ItemsView(self._impl) - def values(self): + def values(self) -> _ValuesView[_V]: """Return a new view of the dictionary's values.""" return _ValuesView(self._impl) - def __eq__(self, other): - if not isinstance(other, abc.Mapping): + def __eq__(self, other: object) -> bool: + if not isinstance(other, Mapping): return NotImplemented if isinstance(other, _Base): lft = self._impl._items @@ -125,124 +251,83 @@ class _Base: if len(self._impl._items) != len(other): return False for k, v in self.items(): - nv = other.get(k, _marker) + nv = other.get(k, sentinel) if v != nv: return False return True - def __contains__(self, key): + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False identity = self._title(key) for i, k, v in self._impl._items: if i == identity: return True return False - def __repr__(self): + def __repr__(self) -> str: body = ", ".join("'{}': {!r}".format(k, v) for k, v in self.items()) return "<{}({})>".format(self.__class__.__name__, body) - __class_getitem__ = classmethod(GenericAlias) - -class MultiDictProxy(_Base, MultiMapping): - """Read-only proxy for MultiDict instance.""" - - def __init__(self, arg): - if not isinstance(arg, (MultiDict, MultiDictProxy)): - raise TypeError( - "ctor requires MultiDict or MultiDictProxy instance" - ", not {}".format(type(arg)) - ) - - self._impl = arg._impl - - def __reduce__(self): - raise TypeError("can't pickle {} objects".format(self.__class__.__name__)) - - def copy(self): - """Return a copy of itself.""" - return MultiDict(self.items()) - - -class CIMultiDictProxy(MultiDictProxy): - """Read-only proxy for CIMultiDict instance.""" - - def __init__(self, arg): - if not isinstance(arg, (CIMultiDict, CIMultiDictProxy)): - raise TypeError( - "ctor requires CIMultiDict or CIMultiDictProxy instance" - ", not {}".format(type(arg)) - ) - - self._impl = arg._impl - - def _title(self, key): - return key.title() - - def copy(self): - """Return a copy of itself.""" - return CIMultiDict(self.items()) - - -class MultiDict(_Base, MutableMultiMapping): +class MultiDict(_Base[_V], MutableMultiMapping[_V]): """Dictionary with the support for duplicate keys.""" - def __init__(self, *args, **kwargs): + def __init__(self, arg: MDArg[_V] = None, /, **kwargs: _V): self._impl = _Impl() - self._extend(args, kwargs, self.__class__.__name__, self._extend_items) + self._extend(arg, kwargs, self.__class__.__name__, self._extend_items) if sys.implementation.name != "pypy": - def __sizeof__(self): + def __sizeof__(self) -> int: return object.__sizeof__(self) + sys.getsizeof(self._impl) - def __reduce__(self): + def __reduce__(self) -> tuple[type[Self], tuple[list[tuple[str, _V]]]]: return (self.__class__, (list(self.items()),)) - def _title(self, key): + def _title(self, key: str) -> str: return key - def _key(self, key): + def _key(self, key: str) -> str: if isinstance(key, str): return key else: - raise TypeError( - "MultiDict keys should be either str " "or subclasses of str" - ) + raise TypeError("MultiDict keys should be either str or subclasses of str") - def add(self, key, value): + def add(self, key: str, value: _V) -> None: identity = self._title(key) self._impl._items.append((identity, self._key(key), value)) self._impl.incr_version() - def copy(self): + def copy(self) -> Self: """Return a copy of itself.""" cls = self.__class__ return cls(self.items()) __copy__ = copy - def extend(self, *args, **kwargs): + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: """Extend current MultiDict with more values. This method must be used instead of update. """ - self._extend(args, kwargs, "extend", self._extend_items) + self._extend(arg, kwargs, "extend", self._extend_items) - def _extend(self, args, kwargs, name, method): - if len(args) > 1: - raise TypeError( - "{} takes at most 1 positional argument" - " ({} given)".format(name, len(args)) - ) - if args: - arg = args[0] - if isinstance(args[0], (MultiDict, MultiDictProxy)) and not kwargs: + def _extend( + self, + arg: MDArg[_V], + kwargs: Mapping[str, _V], + name: str, + method: Callable[[list[tuple[str, str, _V]]], None], + ) -> None: + if arg: + if isinstance(arg, (MultiDict, MultiDictProxy)) and not kwargs: items = arg._impl._items else: - if hasattr(arg, "items"): - arg = arg.items() + if hasattr(arg, "keys"): + arg = cast(SupportsKeys[_V], arg) + arg = [(k, arg[k]) for k in arg.keys()] if kwargs: arg = list(arg) arg.extend(list(kwargs.items())) @@ -264,21 +349,21 @@ class MultiDict(_Base, MutableMultiMapping): ] ) - def _extend_items(self, items): + def _extend_items(self, items: Iterable[tuple[str, str, _V]]) -> None: for identity, key, value in items: self.add(key, value) - def clear(self): + def clear(self) -> None: """Remove all items from MultiDict.""" self._impl._items.clear() self._impl.incr_version() # Mapping interface # - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: _V) -> None: self._replace(key, value) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: identity = self._title(key) items = self._impl._items found = False @@ -291,16 +376,28 @@ class MultiDict(_Base, MutableMultiMapping): else: self._impl.incr_version() - def setdefault(self, key, default=None): + @overload + def setdefault( + self: "MultiDict[Union[_T, None]]", key: str, default: None = None + ) -> Union[_T, None]: ... + @overload + def setdefault(self, key: str, default: _V) -> _V: ... + def setdefault(self, key: str, default: Union[_V, None] = None) -> Union[_V, None]: # type: ignore[misc] """Return value for key, set value to default if key is not present.""" identity = self._title(key) for i, k, v in self._impl._items: if i == identity: return v - self.add(key, default) + self.add(key, default) # type: ignore[arg-type] return default - def popone(self, key, default=_marker): + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... + def popone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: """Remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise @@ -314,14 +411,22 @@ class MultiDict(_Base, MutableMultiMapping): del self._impl._items[i] self._impl.incr_version() return value - if default is _marker: + if default is sentinel: raise KeyError(key) else: return default - pop = popone # type: ignore + # Type checking will inherit signature for pop() if we don't confuse it here. + if not TYPE_CHECKING: + pop = popone - def popall(self, key, default=_marker): + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def popall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: """Remove all occurrences of key and return the list of corresponding values. @@ -340,7 +445,7 @@ class MultiDict(_Base, MutableMultiMapping): self._impl.incr_version() found = True if not found: - if default is _marker: + if default is sentinel: raise KeyError(key) else: return default @@ -348,7 +453,7 @@ class MultiDict(_Base, MutableMultiMapping): ret.reverse() return ret - def popitem(self): + def popitem(self) -> tuple[str, _V]: """Remove and return an arbitrary (key, value) pair.""" if self._impl._items: i = self._impl._items.pop(0) @@ -357,14 +462,14 @@ class MultiDict(_Base, MutableMultiMapping): else: raise KeyError("empty multidict") - def update(self, *args, **kwargs): + def update(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: """Update the dictionary from *other*, overwriting existing keys.""" - self._extend(args, kwargs, "update", self._update_items) + self._extend(arg, kwargs, "update", self._update_items) - def _update_items(self, items): + def _update_items(self, items: list[tuple[str, str, _V]]) -> None: if not items: return - used_keys = {} + used_keys: dict[str, int] = {} for identity, key, value in items: start = used_keys.get(identity, 0) for i in range(start, len(self._impl._items)): @@ -393,7 +498,7 @@ class MultiDict(_Base, MutableMultiMapping): self._impl.incr_version() - def _replace(self, key, value): + def _replace(self, key: str, value: _V) -> None: key = self._key(key) identity = self._title(key) items = self._impl._items @@ -412,7 +517,8 @@ class MultiDict(_Base, MutableMultiMapping): return # remove all tail items - i = rgt + 1 + # Mypy bug: https://github.com/python/mypy/issues/14209 + i = rgt + 1 # type: ignore[possibly-undefined] while i < len(items): item = items[i] if item[0] == identity: @@ -421,107 +527,54 @@ class MultiDict(_Base, MutableMultiMapping): i += 1 -class CIMultiDict(MultiDict): +class CIMultiDict(MultiDict[_V]): """Dictionary with the support for duplicate case-insensitive keys.""" - def _title(self, key): + def _title(self, key: str) -> str: return key.title() -class _Iter: - __slots__ = ("_size", "_iter") - - def __init__(self, size, iterator): - self._size = size - self._iter = iterator - - def __iter__(self): - return self - - def __next__(self): - return next(self._iter) - - def __length_hint__(self): - return self._size - - -class _ViewBase: - def __init__(self, impl): - self._impl = impl - - def __len__(self): - return len(self._impl._items) - - -class _ItemsView(_ViewBase, abc.ItemsView): - def __contains__(self, item): - assert isinstance(item, tuple) or isinstance(item, list) - assert len(item) == 2 - for i, k, v in self._impl._items: - if item[0] == k and item[1] == v: - return True - return False - - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) +class MultiDictProxy(_Base[_V]): + """Read-only proxy for MultiDict instance.""" - def _iter(self, version): - for i, k, v in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield k, v + def __init__(self, arg: Union[MultiDict[_V], "MultiDictProxy[_V]"]): + if not isinstance(arg, (MultiDict, MultiDictProxy)): + raise TypeError( + "ctor requires MultiDict or MultiDictProxy instance" + ", not {}".format(type(arg)) + ) - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}: {!r}".format(item[1], item[2])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) + self._impl = arg._impl + def __reduce__(self) -> NoReturn: + raise TypeError("can't pickle {} objects".format(self.__class__.__name__)) -class _ValuesView(_ViewBase, abc.ValuesView): - def __contains__(self, value): - for item in self._impl._items: - if item[2] == value: - return True - return False + def copy(self) -> MultiDict[_V]: + """Return a copy of itself.""" + return MultiDict(self.items()) - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) - def _iter(self, version): - for item in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield item[2] +class CIMultiDictProxy(MultiDictProxy[_V]): + """Read-only proxy for CIMultiDict instance.""" - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}".format(item[2])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) + def __init__(self, arg: Union[MultiDict[_V], MultiDictProxy[_V]]): + if not isinstance(arg, (CIMultiDict, CIMultiDictProxy)): + raise TypeError( + "ctor requires CIMultiDict or CIMultiDictProxy instance" + ", not {}".format(type(arg)) + ) + self._impl = arg._impl -class _KeysView(_ViewBase, abc.KeysView): - def __contains__(self, key): - for item in self._impl._items: - if item[1] == key: - return True - return False + def _title(self, key: str) -> str: + return key.title() - def __iter__(self): - return _Iter(len(self), self._iter(self._impl._version)) + def copy(self) -> CIMultiDict[_V]: + """Return a copy of itself.""" + return CIMultiDict(self.items()) - def _iter(self, version): - for item in self._impl._items: - if version != self._impl._version: - raise RuntimeError("Dictionary changed during iteration") - yield item[1] - def __repr__(self): - lst = [] - for item in self._impl._items: - lst.append("{!r}".format(item[1])) - body = ", ".join(lst) - return "{}({})".format(self.__class__.__name__, body) +def getversion(md: Union[MultiDict[object], MultiDictProxy[object]]) -> int: + if not isinstance(md, _Base): + raise TypeError("Parameter should be multidict or proxy") + return md._impl._version diff --git a/contrib/python/multidict/multidict/_multilib/defs.h b/contrib/python/multidict/multidict/_multilib/defs.h index 55c21074ddd..51a6639c428 100644 --- a/contrib/python/multidict/multidict/_multilib/defs.h +++ b/contrib/python/multidict/multidict/_multilib/defs.h @@ -5,11 +5,7 @@ extern "C" { #endif -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 static PyObject *multidict_str_lower = NULL; -#else -_Py_IDENTIFIER(lower); -#endif /* We link this module statically for convenience. If compiled as a shared library instead, some compilers don't allow addresses of Python objects diff --git a/contrib/python/multidict/multidict/_multilib/istr.h b/contrib/python/multidict/multidict/_multilib/istr.h index 61dc61aec6f..8454f78b88b 100644 --- a/contrib/python/multidict/multidict/_multilib/istr.h +++ b/contrib/python/multidict/multidict/_multilib/istr.h @@ -43,11 +43,7 @@ istr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!ret) { goto fail; } -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 9 s = PyObject_CallMethodNoArgs(ret, multidict_str_lower); -#else - s =_PyObject_CallMethodId(ret, &PyId_lower, NULL); -#endif if (!s) { goto fail; } diff --git a/contrib/python/multidict/multidict/_multilib/pair_list.h b/contrib/python/multidict/multidict/_multilib/pair_list.h index 15291d46a81..b23150dfad7 100644 --- a/contrib/python/multidict/multidict/_multilib/pair_list.h +++ b/contrib/python/multidict/multidict/_multilib/pair_list.h @@ -31,11 +31,7 @@ The embedded buffer intention is to fit the vast majority of possible HTTP headers into the buffer without allocating an extra memory block. */ -#if (PY_VERSION_HEX < 0x03080000) -#define EMBEDDED_CAPACITY 28 -#else #define EMBEDDED_CAPACITY 29 -#endif typedef struct pair_list { Py_ssize_t capacity; @@ -110,11 +106,7 @@ ci_key_to_str(PyObject *key) return ret; } if (PyUnicode_Check(key)) { -#if PY_VERSION_HEX < 0x03090000 - return _PyObject_CallMethodId(key, &PyId_lower, NULL); -#else return PyObject_CallMethodNoArgs(key, multidict_str_lower); -#endif } PyErr_SetString(PyExc_TypeError, "CIMultiDict keys should be either str " @@ -497,6 +489,10 @@ pair_list_contains(pair_list_t *list, PyObject *key) PyObject *identity = NULL; int tmp; + if (!PyUnicode_Check(key)) { + return 0; + } + ident = pair_list_calc_identity(list, key); if (ident == NULL) { goto fail; @@ -916,13 +912,18 @@ _pair_list_post_update(pair_list_t *list, PyObject* used_keys, Py_ssize_t pos) for (; pos < list->size; pos++) { pair = pair_list_get(list, pos); - tmp = PyDict_GetItem(used_keys, pair->identity); - if (tmp == NULL) { + int status = PyDict_GetItemRef(used_keys, pair->identity, &tmp); + if (status == -1) { + // exception set + return -1; + } + else if (status == 0) { // not found continue; } num = PyLong_AsSsize_t(tmp); + Py_DECREF(tmp); if (num == -1) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "invalid internal state"); @@ -955,12 +956,18 @@ _pair_list_update(pair_list_t *list, PyObject *key, int found; int ident_cmp_res; - item = PyDict_GetItem(used_keys, identity); - if (item == NULL) { + int status = PyDict_GetItemRef(used_keys, identity, &item); + if (status == -1) { + // exception set + return -1; + } + else if (status == 0) { + // not found pos = 0; } else { pos = PyLong_AsSsize_t(item); + Py_DECREF(item); if (pos == -1) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "invalid internal state"); @@ -1087,18 +1094,28 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) } // Convert item to sequence, and verify length 2. +#ifdef Py_GIL_DISABLED + if (!PySequence_Check(item)) { +#else fast = PySequence_Fast(item, ""); if (fast == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { +#endif PyErr_Format(PyExc_TypeError, "multidict cannot convert sequence element #%zd" " to a sequence", i); +#ifndef Py_GIL_DISABLED } +#endif goto fail_1; } +#ifdef Py_GIL_DISABLED + n = PySequence_Size(item); +#else n = PySequence_Fast_GET_SIZE(fast); +#endif if (n != 2) { PyErr_Format(PyExc_ValueError, "multidict update sequence element #%zd " @@ -1107,10 +1124,27 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) goto fail_1; } +#ifdef Py_GIL_DISABLED + key = PySequence_ITEM(item, 0); + if (key == NULL) { + PyErr_Format(PyExc_ValueError, + "multidict update sequence element #%zd's " + "key could not be fetched", i); + goto fail_1; + } + value = PySequence_ITEM(item, 1); + if (value == NULL) { + PyErr_Format(PyExc_ValueError, + "multidict update sequence element #%zd's " + "value could not be fetched", i); + goto fail_1; + } +#else key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); Py_INCREF(key); Py_INCREF(value); +#endif identity = pair_list_calc_identity(list, key); if (identity == NULL) { @@ -1128,7 +1162,9 @@ pair_list_update_from_seq(pair_list_t *list, PyObject *seq) Py_DECREF(key); Py_DECREF(value); +#ifndef Py_GIL_DISABLED Py_DECREF(fast); +#endif Py_DECREF(item); Py_DECREF(identity); } diff --git a/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h b/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h new file mode 100644 index 00000000000..971981993ba --- /dev/null +++ b/contrib/python/multidict/multidict/_multilib/pythoncapi_compat.h @@ -0,0 +1,1142 @@ +// Header file providing new C API functions to old Python versions. +// +// File distributed under the Zero Clause BSD (0BSD) license. +// Copyright Contributors to the pythoncapi_compat project. +// +// Homepage: +// https://github.com/python/pythoncapi_compat +// +// Latest version: +// https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h +// +// The vendored version comes from commit: +// https://raw.githubusercontent.com/python/pythoncapi-compat/2d18aecd7b2f549d38a13e27b682ea4966f37bd8/pythoncapi_compat.h +// +// SPDX-License-Identifier: 0BSD + +#ifndef PYTHONCAPI_COMPAT +#define PYTHONCAPI_COMPAT + +#ifdef __cplusplus +extern "C" { +#endif + +#include <Python.h> + +// Python 3.11.0b4 added PyFrame_Back() to Python.h +#if PY_VERSION_HEX < 0x030b00B4 && !defined(PYPY_VERSION) +# include "frameobject.h" // PyFrameObject, PyFrame_GetBack() +#endif + + +#ifndef _Py_CAST +# define _Py_CAST(type, expr) ((type)(expr)) +#endif + +// Static inline functions should use _Py_NULL rather than using directly NULL +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +#else +# define _Py_NULL NULL +#endif + +// Cast argument to PyObject* type. +#ifndef _PyObject_CAST +# define _PyObject_CAST(op) _Py_CAST(PyObject*, op) +#endif + + +// bpo-42262 added Py_NewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} +#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-42262 added Py_XNewRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} +#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#endif + + +// bpo-43753 added Py_Is(), Py_IsNone(), Py_IsTrue() and Py_IsFalse() +// to Python 3.10.0b1. +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_Is) +# define Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) +# define Py_IsNone(x) Py_Is(x, Py_None) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsTrue) +# define Py_IsTrue(x) Py_Is(x, Py_True) +#endif +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsFalse) +# define Py_IsFalse(x) Py_Is(x, Py_False) +#endif + + +#if defined(PYPY_VERSION) +static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) +{ + assert(frame != _Py_NULL); + assert(frame->f_code != _Py_NULL); + return _Py_CAST(PyCodeObject*, Py_NewRef(frame->f_code)); +} +#endif + +static inline PyCodeObject* _PyFrame_GetCodeBorrow(PyFrameObject *frame) +{ + PyCodeObject *code = PyFrame_GetCode(frame); + Py_DECREF(code); + return code; +} + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* _PyFrame_GetBackBorrow(PyFrameObject *frame) +{ + PyFrameObject *back = PyFrame_GetBack(frame); + Py_XDECREF(back); + return back; +} +#endif + + +// bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetLocals(PyFrameObject *frame) +{ + if (PyFrame_FastToLocalsWithError(frame) < 0) { + return NULL; + } + return Py_NewRef(frame->f_locals); +} +#endif + + +// bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetGlobals(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_globals); +} +#endif + + +// bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 +#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) +{ + return Py_NewRef(frame->f_builtins); +} +#endif + + +// bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline int PyFrame_GetLasti(PyFrameObject *frame) +{ +#if PY_VERSION_HEX >= 0x030A00A7 + // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, + // not a bytes offset anymore. Python uses 16-bit "wordcode" (2 bytes) + // instructions. + if (frame->f_lasti < 0) { + return -1; + } + return frame->f_lasti * 2; +#else + return frame->f_lasti; +#endif +} +#endif + + +// gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name) +{ + PyObject *locals, *value; + + locals = PyFrame_GetLocals(frame); + if (locals == NULL) { + return NULL; + } + value = PyDict_GetItemWithError(locals, name); + Py_DECREF(locals); + + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + PyErr_Format(PyExc_NameError, "variable %R does not exist", name); + return NULL; + } + return Py_NewRef(value); +} +#endif + + +// gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* +PyFrame_GetVarString(PyFrameObject *frame, const char *name) +{ + PyObject *name_obj, *value; + name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { + return NULL; + } + value = PyFrame_GetVar(frame, name_obj); + Py_DECREF(name_obj); + return value; +} +#endif + + +#if defined(PYPY_VERSION) +static inline PyInterpreterState * +PyThreadState_GetInterpreter(PyThreadState *tstate) +{ + assert(tstate != _Py_NULL); + return tstate->interp; +} +#endif + +#if !defined(PYPY_VERSION) +static inline PyFrameObject* +_PyThreadState_GetFrameBorrow(PyThreadState *tstate) +{ + PyFrameObject *frame = PyThreadState_GetFrame(tstate); + Py_XDECREF(frame); + return frame; +} +#endif + + +#if defined(PYPY_VERSION) +static inline PyInterpreterState* PyInterpreterState_Get(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_GET(); + if (tstate == _Py_NULL) { + Py_FatalError("GIL released (tstate is NULL)"); + } + interp = tstate->interp; + if (interp == _Py_NULL) { + Py_FatalError("no current interpreter"); + } + return interp; +} +#endif + +// bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_EnterTracing(PyThreadState *tstate) +{ + tstate->tracing++; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = 0; +#else + tstate->use_tracing = 0; +#endif +} +#endif + +// bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 +#if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) +static inline void PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + int use_tracing = (tstate->c_tracefunc != _Py_NULL + || tstate->c_profilefunc != _Py_NULL); + tstate->tracing--; +#if PY_VERSION_HEX >= 0x030A00A1 + tstate->cframe->use_tracing = use_tracing; +#else + tstate->use_tracing = use_tracing; +#endif +} +#endif + + +// bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 +#if PY_VERSION_HEX < 0x030A00A3 +static inline int +PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) +{ + int res; + + if (!value && !PyErr_Occurred()) { + // PyModule_AddObject() raises TypeError in this case + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + return -1; + } + + Py_XINCREF(value); + res = PyModule_AddObject(module, name, value); + if (res < 0) { + Py_XDECREF(value); + } + return res; +} +#endif + + +// bpo-46906 added PyFloat_Pack2() and PyFloat_Unpack2() to Python 3.11a7. +// Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal +// C API: Python 3.11a2-3.11a6 versions are not supported. +#if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack2(double x, char *p, int le) +{ return _PyFloat_Pack2(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack2(const char *p, int le) +{ return _PyFloat_Unpack2((const unsigned char *)p, le); } +#endif + + +// bpo-46906 added PyFloat_Pack4(), PyFloat_Pack8(), PyFloat_Unpack4() and +// PyFloat_Unpack8() to Python 3.11a7. +// Python 3.11a2 moved _PyFloat_Pack4(), _PyFloat_Pack8(), _PyFloat_Unpack4() +// and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions +// are not supported. +#if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) +static inline int PyFloat_Pack4(double x, char *p, int le) +{ return _PyFloat_Pack4(x, (unsigned char*)p, le); } + +static inline int PyFloat_Pack8(double x, char *p, int le) +{ return _PyFloat_Pack8(x, (unsigned char*)p, le); } + +static inline double PyFloat_Unpack4(const char *p, int le) +{ return _PyFloat_Unpack4((const unsigned char *)p, le); } + +static inline double PyFloat_Unpack8(const char *p, int le) +{ return _PyFloat_Unpack8((const unsigned char *)p, le); } +#endif + + +// gh-92154 added PyCode_GetCode() to Python 3.11.0b1 +#if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCode(PyCodeObject *code) +{ + return Py_NewRef(code->co_code); +} +#endif + + +// gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetVarnames(PyCodeObject *code) +{ + return Py_NewRef(code->co_varnames); +} +#endif + +// gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetFreevars(PyCodeObject *code) +{ + return Py_NewRef(code->co_freevars); +} +#endif + +// gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCellvars(PyCodeObject *code) +{ + return Py_NewRef(code->co_cellvars); +} +#endif + + +// gh-105922 added PyImport_AddModuleRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A0 +static inline PyObject* PyImport_AddModuleRef(const char *name) +{ + return Py_XNewRef(PyImport_AddModule(name)); +} +#endif + + +// gh-105927 added PyWeakref_GetRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D0000 +static inline int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj; + if (ref != NULL && !PyWeakref_Check(ref)) { + *pobj = NULL; + PyErr_SetString(PyExc_TypeError, "expected a weakref"); + return -1; + } + obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + *pobj = Py_NewRef(obj); + return (*pobj != NULL); +} +#endif + + +// gh-106521 added PyObject_GetOptionalAttr() and +// PyObject_GetOptionalAttrString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result) +{ + return _PyObject_LookupAttr(obj, attr_name, result); +} + +static inline int +PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result) +{ + PyObject *name_obj; + int rc; + name_obj = PyUnicode_FromString(attr_name); + if (name_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyObject_GetOptionalAttr(obj, name_obj, result); + Py_DECREF(name_obj); + return rc; +} +#endif + + +// gh-106307 added PyObject_GetOptionalAttr() and +// PyMapping_GetOptionalItemString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) +{ + *result = PyObject_GetItem(obj, key); + if (*result) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; +} + +static inline int +PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) +{ + PyObject *key_obj; + int rc; + key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyMapping_GetOptionalItem(obj, key_obj, result); + Py_DECREF(key_obj); + return rc; +} +#endif + +// gh-108511 added PyMapping_HasKeyWithError() and +// PyMapping_HasKeyStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItem(obj, key, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyMapping_HasKeyStringWithError(PyObject *obj, const char *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItemString(obj, key, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-108511 added PyObject_HasAttrWithError() and +// PyObject_HasAttrStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_HasAttrWithError(PyObject *obj, PyObject *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttr(obj, attr, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyObject_HasAttrStringWithError(PyObject *obj, const char *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, attr, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-106004 added PyDict_GetItemRef() and PyDict_GetItemStringRef() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ + PyObject *item = PyDict_GetItemWithError(mp, key); + if (item != NULL) { + *result = Py_NewRef(item); + return 1; // found + } + if (!PyErr_Occurred()) { + *result = NULL; + return 0; // not found + } + *result = NULL; + return -1; +} + +static inline int +PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result) +{ + int res; + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + *result = NULL; + return -1; + } + res = PyDict_GetItemRef(mp, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-106307 added PyModule_Add() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} +#endif + + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +// _Py_IsFinalizing() was added to PyPy 7.3.0. +#if (PY_VERSION_HEX < 0x030D00A1) \ + && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x7030000) +static inline int Py_IsFinalizing(void) +{ + return _Py_IsFinalizing(); +} +#endif + + +// gh-108323 added PyDict_ContainsString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-108445 added PyLong_AsInt() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyLong_AsInt(PyObject *obj) +{ +#ifdef PYPY_VERSION + long value = PyLong_AsLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + if (value < (long)INT_MIN || (long)INT_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)value; +#else + return _PyLong_AsInt(obj); +#endif +} +#endif + + +// gh-107073 added PyObject_VisitManagedDict() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return -1; + } + Py_VISIT(*dict); + return 0; +} + +static inline void +PyObject_ClearManagedDict(PyObject *obj) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return; + } + Py_CLEAR(*dict); +} +#endif + +// gh-108867 added PyThreadState_GetUnchecked() to Python 3.13.0a1. +#if PY_VERSION_HEX < 0x030D00A1 +static inline PyThreadState* +PyThreadState_GetUnchecked(void) +{ + return _PyThreadState_UncheckedGet(); +} +#endif + +// gh-110289 added PyUnicode_EqualToUTF8() and PyUnicode_EqualToUTF8AndSize() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *str, Py_ssize_t str_len) +{ + Py_ssize_t len; + const void *utf8; + PyObject *exc_type, *exc_value, *exc_tb; + int res; + + // API cannot report errors so save/restore the exception + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + if (PyUnicode_IS_ASCII(unicode)) { + utf8 = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + } + else { + utf8 = PyUnicode_AsUTF8AndSize(unicode, &len); + if (utf8 == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + } + + if (len != str_len) { + res = 0; + goto done; + } + res = (memcmp(utf8, str, (size_t)len) == 0); + +done: + PyErr_Restore(exc_type, exc_value, exc_tb); + return res; +} + +static inline int +PyUnicode_EqualToUTF8(PyObject *unicode, const char *str) +{ + return PyUnicode_EqualToUTF8AndSize(unicode, str, (Py_ssize_t)strlen(str)); +} +#endif + + +// gh-111138 added PyList_Extend() and PyList_Clear() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyList_Extend(PyObject *list, PyObject *iterable) +{ + return PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable); +} + +static inline int +PyList_Clear(PyObject *list) +{ + return PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL); +} +#endif + +// gh-111262 added PyDict_Pop() and PyDict_PopString() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result) +{ + PyObject *value; + + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + if (result) { + *result = NULL; + } + return -1; + } + + // Python 3.13.0a1 removed _PyDict_Pop(). +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x030D0000 + value = PyObject_CallMethod(dict, "pop", "O", key); +#else + value = _PyDict_Pop(dict, key, NULL); +#endif + if (value == NULL) { + if (result) { + *result = NULL; + } + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; +} + +static inline int +PyDict_PopString(PyObject *dict, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(dict, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-111545 added Py_HashPointer() to Python 3.13.0a3 +#if PY_VERSION_HEX < 0x030D00A3 +static inline Py_hash_t Py_HashPointer(const void *ptr) +{ +#if !defined(PYPY_VERSION) + return _Py_HashPointer(ptr); +#else + return _Py_HashPointer(_Py_CAST(void*, ptr)); +#endif +} +#endif + + +// Python 3.13a4 added a PyTime API. +#if PY_VERSION_HEX < 0x030D00A4 +typedef _PyTime_t PyTime_t; +#define PyTime_MIN _PyTime_MIN +#define PyTime_MAX _PyTime_MAX + +static inline double PyTime_AsSecondsDouble(PyTime_t t) +{ return _PyTime_AsSecondsDouble(t); } + +static inline int PyTime_Monotonic(PyTime_t *result) +{ return _PyTime_GetMonotonicClockWithInfo(result, NULL); } + +static inline int PyTime_Time(PyTime_t *result) +{ return _PyTime_GetSystemClockWithInfo(result, NULL); } + +static inline int PyTime_PerfCounter(PyTime_t *result) +{ +#if !defined(PYPY_VERSION) + return _PyTime_GetPerfCounterWithInfo(result, NULL); +#else + // Call time.perf_counter_ns() and convert Python int object to PyTime_t. + // Cache time.perf_counter_ns() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter_ns"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + long long value = PyLong_AsLongLong(res); + Py_DECREF(res); + + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(value) >= sizeof(PyTime_t)); + *result = (PyTime_t)value; + return 0; +#endif +} + +#endif + +// gh-111389 added hash constants to Python 3.13.0a5. These constants were +// added first as private macros to Python 3.4.0b1 and PyPy 7.3.9. +#if (!defined(PyHASH_BITS) \ + && (!defined(PYPY_VERSION) \ + || (defined(PYPY_VERSION) && PYPY_VERSION_NUM >= 0x07090000))) +# define PyHASH_BITS _PyHASH_BITS +# define PyHASH_MODULUS _PyHASH_MODULUS +# define PyHASH_INF _PyHASH_INF +# define PyHASH_IMAG _PyHASH_IMAG +#endif + + +// gh-111545 added Py_GetConstant() and Py_GetConstantBorrowed() +// to Python 3.13.0a6 +#if PY_VERSION_HEX < 0x030D00A6 && !defined(Py_CONSTANT_NONE) + +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +static inline PyObject* Py_GetConstant(unsigned int constant_id) +{ + static PyObject* constants[Py_CONSTANT_EMPTY_TUPLE + 1] = {NULL}; + + if (constants[Py_CONSTANT_NONE] == NULL) { + constants[Py_CONSTANT_NONE] = Py_None; + constants[Py_CONSTANT_FALSE] = Py_False; + constants[Py_CONSTANT_TRUE] = Py_True; + constants[Py_CONSTANT_ELLIPSIS] = Py_Ellipsis; + constants[Py_CONSTANT_NOT_IMPLEMENTED] = Py_NotImplemented; + + constants[Py_CONSTANT_ZERO] = PyLong_FromLong(0); + if (constants[Py_CONSTANT_ZERO] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_ONE] = PyLong_FromLong(1); + if (constants[Py_CONSTANT_ONE] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_STR] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_BYTES] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); + if (constants[Py_CONSTANT_EMPTY_TUPLE] == NULL) { + goto fatal_error; + } + // goto dance to avoid compiler warnings about Py_FatalError() + goto init_done; + +fatal_error: + // This case should never happen + Py_FatalError("Py_GetConstant() failed to get constants"); + } + +init_done: + if (constant_id <= Py_CONSTANT_EMPTY_TUPLE) { + return Py_NewRef(constants[constant_id]); + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + +static inline PyObject* Py_GetConstantBorrowed(unsigned int constant_id) +{ + PyObject *obj = Py_GetConstant(constant_id); + Py_XDECREF(obj); + return obj; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t index) +{ + PyObject *item = PyList_GetItem(op, index); + Py_XINCREF(item); + return item; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + PyObject *value; + if (PyDict_GetItemRef(d, key, &value) < 0) { + // get error + if (result) { + *result = NULL; + } + return -1; + } + if (value != NULL) { + // present + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; + } + + // missing: set the item + if (PyDict_SetItem(d, key, default_value) < 0) { + // set error + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = Py_NewRef(default_value); + } + return 0; +} +#endif + +#if PY_VERSION_HEX < 0x030D00B3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) { +# define Py_END_CRITICAL_SECTION2() } +#endif + +#if PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) +typedef struct PyUnicodeWriter PyUnicodeWriter; + +static inline void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + +static inline PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == _Py_NULL) { + PyErr_NoMemory(); + return _Py_NULL; + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + return pub_writer; +} + +static inline PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + +static inline int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > 0x10ffff) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + +static inline int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Repr(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + PyObject *str_obj = PyUnicode_FromStringAndSize(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, + const wchar_t *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)wcslen(str); + } + + PyObject *str_obj = PyUnicode_FromWideChar(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + +static inline int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (str == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030E0000 + +// gh-116560 added PyLong_GetSign() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyLong_GetSign(PyObject *obj, int *sign) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + + *sign = _PyLong_Sign(obj); + return 0; +} +#endif + + +#ifdef __cplusplus +} +#endif +#endif // PYTHONCAPI_COMPAT diff --git a/contrib/python/multidict/tests/conftest.py b/contrib/python/multidict/tests/conftest.py index 0d003950cd7..a37f58f2d1f 100644 --- a/contrib/python/multidict/tests/conftest.py +++ b/contrib/python/multidict/tests/conftest.py @@ -3,26 +3,22 @@ from __future__ import annotations import argparse import pickle from dataclasses import dataclass +from functools import cached_property from importlib import import_module -from sys import version_info as _version_info from types import ModuleType -from typing import Callable, Type - -try: - from functools import cached_property # Python 3.8+ -except ImportError: - from functools import lru_cache as _lru_cache - - def cached_property(func): - return property(_lru_cache()(func)) - +from typing import Callable, Type, Union import pytest -from multidict import MultiMapping, MutableMultiMapping +from multidict import ( + CIMultiDict, + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) C_EXT_MARK = pytest.mark.c_extension -PY_38_AND_BELOW = _version_info < (3, 9) @dataclass(frozen=True) @@ -51,7 +47,7 @@ class MultidictImplementation: importable_module = "_multidict_py" if self.is_pure_python else "_multidict" return import_module(f"multidict.{importable_module}") - def __str__(self): + def __str__(self) -> str: """Render the implementation facade instance as a string.""" return f"{self.tag}-module" @@ -69,7 +65,7 @@ class MultidictImplementation: ) def multidict_implementation(request: pytest.FixtureRequest) -> MultidictImplementation: """Return a multidict variant facade.""" - return request.param + return request.param # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -87,7 +83,7 @@ def multidict_module( ) def any_multidict_class_name(request: pytest.FixtureRequest) -> str: """Return a class name of a mutable multidict implementation.""" - return request.param + return request.param # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -96,29 +92,29 @@ def any_multidict_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a class object of a mutable multidict implementation.""" - return getattr(multidict_module, any_multidict_class_name) + return getattr(multidict_module, any_multidict_class_name) # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_sensitive_multidict_class( multidict_module: ModuleType, -) -> Type[MutableMultiMapping[str]]: +) -> Type[MultiDict[str]]: """Return a case-sensitive mutable multidict class.""" - return multidict_module.MultiDict + return multidict_module.MultiDict # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_insensitive_multidict_class( multidict_module: ModuleType, -) -> Type[MutableMultiMapping[str]]: +) -> Type[CIMultiDict[str]]: """Return a case-insensitive mutable multidict class.""" - return multidict_module.CIMultiDict + return multidict_module.CIMultiDict # type: ignore[no-any-return] @pytest.fixture(scope="session") def case_insensitive_str_class(multidict_module: ModuleType) -> Type[str]: """Return a case-insensitive string class.""" - return multidict_module.istr + return multidict_module.istr # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -133,7 +129,7 @@ def any_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MultiMapping[str]]: """Return an immutable multidict implementation class object.""" - return getattr(multidict_module, any_multidict_proxy_class_name) + return getattr(multidict_module, any_multidict_proxy_class_name) # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -141,7 +137,7 @@ def case_sensitive_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a case-sensitive immutable multidict class.""" - return multidict_module.MultiDictProxy + return multidict_module.MultiDictProxy # type: ignore[no-any-return] @pytest.fixture(scope="session") @@ -149,13 +145,15 @@ def case_insensitive_multidict_proxy_class( multidict_module: ModuleType, ) -> Type[MutableMultiMapping[str]]: """Return a case-insensitive immutable multidict class.""" - return multidict_module.CIMultiDictProxy + return multidict_module.CIMultiDictProxy # type: ignore[no-any-return] @pytest.fixture(scope="session") -def multidict_getversion_callable(multidict_module: ModuleType) -> Callable: +def multidict_getversion_callable( + multidict_module: ModuleType, +) -> Callable[[Union[MultiDict[object], MultiDictProxy[object]]], int]: """Return a ``getversion()`` function for current implementation.""" - return multidict_module.getversion + return multidict_module.getversion # type: ignore[no-any-return] def pytest_addoption( @@ -171,20 +169,12 @@ def pytest_addoption( parser.addoption( "--c-extensions", # disabled with `--no-c-extensions` - action="store_true" if PY_38_AND_BELOW else argparse.BooleanOptionalAction, + action=argparse.BooleanOptionalAction, default=True, dest="c_extensions", help="Test C-extensions (on by default)", ) - if PY_38_AND_BELOW: - parser.addoption( - "--no-c-extensions", - action="store_false", - dest="c_extensions", - help="Skip testing C-extensions (on by default)", - ) - def pytest_collection_modifyitems( session: pytest.Session, @@ -197,8 +187,8 @@ def pytest_collection_modifyitems( if test_c_extensions: return - selected_tests = [] - deselected_tests = [] + selected_tests: list[pytest.Item] = [] + deselected_tests: list[pytest.Item] = [] for item in items: c_ext = item.get_closest_marker(C_EXT_MARK.name) is not None @@ -218,7 +208,7 @@ def pytest_configure(config: pytest.Config) -> None: ) -def pytest_generate_tests(metafunc): +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "pickle_protocol" in metafunc.fixturenames: metafunc.parametrize( "pickle_protocol", list(range(pickle.HIGHEST_PROTOCOL + 1)), scope="session" diff --git a/contrib/python/multidict/tests/gen_pickles.py b/contrib/python/multidict/tests/gen_pickles.py index 4e0d268bedd..72f41b7565f 100644 --- a/contrib/python/multidict/tests/gen_pickles.py +++ b/contrib/python/multidict/tests/gen_pickles.py @@ -1,18 +1,22 @@ import pickle from importlib import import_module from pathlib import Path +from typing import Union + +from multidict import CIMultiDict, MultiDict TESTS_DIR = Path(__file__).parent.resolve() +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] -def write(tag, cls, proto): +def write(tag: str, cls: _MD_Classes, proto: int) -> None: d = cls([("a", 1), ("a", 2)]) file_basename = f"{cls.__name__.lower()}-{tag}" with (TESTS_DIR / f"{file_basename}.pickle.{proto}").open("wb") as f: pickle.dump(d, f, proto) -def generate(): +def generate() -> None: _impl_map = { "c-extension": "_multidict", "pure-python": "_multidict_py", diff --git a/contrib/python/multidict/tests/test_abc.py b/contrib/python/multidict/tests/test_abc.py index e18ad83f820..611d0fa8c35 100644 --- a/contrib/python/multidict/tests/test_abc.py +++ b/contrib/python/multidict/tests/test_abc.py @@ -1,94 +1,32 @@ from collections.abc import Mapping, MutableMapping -import pytest +from multidict import ( + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) -from multidict import MultiMapping, MutableMultiMapping - -def test_abc_inheritance(): +def test_abc_inheritance() -> None: assert issubclass(MultiMapping, Mapping) assert not issubclass(MultiMapping, MutableMapping) assert issubclass(MutableMultiMapping, Mapping) assert issubclass(MutableMultiMapping, MutableMapping) -class A(MultiMapping): - def __getitem__(self, key): - pass - - def __iter__(self): - pass - - def __len__(self): - pass - - def getall(self, key, default=None): - super().getall(key, default) - - def getone(self, key, default=None): - super().getone(key, default) - - -def test_abc_getall(): - with pytest.raises(KeyError): - A().getall("key") - - -def test_abc_getone(): - with pytest.raises(KeyError): - A().getone("key") - - -class B(A, MutableMultiMapping): - def __setitem__(self, key, value): - pass - - def __delitem__(self, key): - pass - - def add(self, key, value): - super().add(key, value) - - def extend(self, *args, **kwargs): - super().extend(*args, **kwargs) - - def popall(self, key, default=None): - super().popall(key, default) - - def popone(self, key, default=None): - super().popone(key, default) - - -def test_abc_add(): - with pytest.raises(NotImplementedError): - B().add("key", "val") - - -def test_abc_extend(): - with pytest.raises(NotImplementedError): - B().extend() - - -def test_abc_popone(): - with pytest.raises(KeyError): - B().popone("key") - - -def test_abc_popall(): - with pytest.raises(KeyError): - B().popall("key") - - -def test_multidict_inheritance(any_multidict_class): +def test_multidict_inheritance(any_multidict_class: type[MultiDict[str]]) -> None: assert issubclass(any_multidict_class, MultiMapping) assert issubclass(any_multidict_class, MutableMultiMapping) -def test_proxy_inheritance(any_multidict_proxy_class): +def test_proxy_inheritance( + any_multidict_proxy_class: type[MultiDictProxy[str]], +) -> None: assert issubclass(any_multidict_proxy_class, MultiMapping) assert not issubclass(any_multidict_proxy_class, MutableMultiMapping) -def test_generic_type_in_runtime(): +def test_generic_type_in_runtime() -> None: MultiMapping[str] MutableMultiMapping[str] diff --git a/contrib/python/multidict/tests/test_copy.py b/contrib/python/multidict/tests/test_copy.py index cd926cdc1d3..deff64db372 100644 --- a/contrib/python/multidict/tests/test_copy.py +++ b/contrib/python/multidict/tests/test_copy.py @@ -1,7 +1,13 @@ import copy +from typing import Union +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy -def test_copy(any_multidict_class): +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] +_MDP_Classes = Union[type[MultiDictProxy[int]], type[CIMultiDictProxy[int]]] + + +def test_copy(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class() d["foo"] = 6 d2 = d.copy() @@ -10,7 +16,9 @@ def test_copy(any_multidict_class): assert d2["foo"] == 7 -def test_copy_proxy(any_multidict_class, any_multidict_proxy_class): +def test_copy_proxy( + any_multidict_class: _MD_Classes, any_multidict_proxy_class: _MDP_Classes +) -> None: d = any_multidict_class() d["foo"] = 6 p = any_multidict_proxy_class(d) @@ -21,7 +29,7 @@ def test_copy_proxy(any_multidict_class, any_multidict_proxy_class): assert d2["foo"] == 7 -def test_copy_std_copy(any_multidict_class): +def test_copy_std_copy(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class() d["foo"] = 6 d2 = copy.copy(d) @@ -30,7 +38,7 @@ def test_copy_std_copy(any_multidict_class): assert d2["foo"] == 7 -def test_ci_multidict_clone(any_multidict_class): +def test_ci_multidict_clone(any_multidict_class: _MD_Classes) -> None: d = any_multidict_class(foo=6) d2 = any_multidict_class(d) d2["foo"] = 7 diff --git a/contrib/python/multidict/tests/test_guard.py b/contrib/python/multidict/tests/test_guard.py index 225da67c8d2..c877fbf803c 100644 --- a/contrib/python/multidict/tests/test_guard.py +++ b/contrib/python/multidict/tests/test_guard.py @@ -1,12 +1,10 @@ -from typing import Type - import pytest -from multidict import MultiMapping +from multidict import MultiDict def test_guard_items( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.items()) @@ -16,7 +14,7 @@ def test_guard_items( def test_guard_keys( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.keys()) @@ -26,7 +24,7 @@ def test_guard_keys( def test_guard_values( - case_sensitive_multidict_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[MultiDict[str]], ) -> None: md = case_sensitive_multidict_class({"a": "b"}) it = iter(md.values()) diff --git a/contrib/python/multidict/tests/test_istr.py b/contrib/python/multidict/tests/test_istr.py index 1918153532b..101f5fe8e5d 100644 --- a/contrib/python/multidict/tests/test_istr.py +++ b/contrib/python/multidict/tests/test_istr.py @@ -71,4 +71,4 @@ def test_leak(create_istrs: Callable[[], None]) -> None: gc.collect() cnt2 = len(gc.get_objects()) - assert abs(cnt - cnt2) < 10 # on PyPy these numbers are not equal + assert abs(cnt - cnt2) < 50 # on PyPy these numbers are not equal diff --git a/contrib/python/multidict/tests/test_multidict.py b/contrib/python/multidict/tests/test_multidict.py index bcfa699c15b..d144130a41f 100644 --- a/contrib/python/multidict/tests/test_multidict.py +++ b/contrib/python/multidict/tests/test_multidict.py @@ -5,27 +5,20 @@ import operator import sys import weakref from collections import deque -from collections.abc import Mapping +from collections.abc import Callable, Iterable, Iterator, KeysView, Mapping from types import ModuleType -from typing import ( - Callable, - Dict, - Iterable, - Iterator, - KeysView, - List, - Mapping, - Set, - Tuple, - Type, - Union, - cast, -) +from typing import Union, cast import pytest import multidict -from multidict import CIMultiDict, MultiDict, MultiMapping, MutableMultiMapping +from multidict import ( + CIMultiDict, + MultiDict, + MultiDictProxy, + MultiMapping, + MutableMultiMapping, +) def chained_callable( @@ -71,7 +64,7 @@ def cls( # type: ignore[misc] def test_exposed_names(any_multidict_class_name: str) -> None: - assert any_multidict_class_name in multidict.__all__ # type: ignore[attr-defined] + assert any_multidict_class_name in multidict.__all__ @pytest.mark.parametrize( @@ -86,8 +79,8 @@ def test_exposed_names(any_multidict_class_name: str) -> None: indirect=["cls"], ) def test__iter__types( - cls: Type[MultiDict[Union[str, int]]], - key_cls: Type[object], + cls: type[MultiDict[Union[str, int]]], + key_cls: type[str], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) for i in d: @@ -95,26 +88,26 @@ def test__iter__types( def test_proxy_copy( - any_multidict_class: Type[MutableMultiMapping[str]], - any_multidict_proxy_class: Type[MultiMapping[str]], + any_multidict_class: type[MultiDict[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: d1 = any_multidict_class(key="value", a="b") p1 = any_multidict_proxy_class(d1) - d2 = p1.copy() # type: ignore[attr-defined] + d2 = p1.copy() assert d1 == d2 assert d1 is not d2 def test_multidict_subclassing( - any_multidict_class: Type[MutableMultiMapping[str]], + any_multidict_class: type[MultiDict[str]], ) -> None: class DummyMultidict(any_multidict_class): # type: ignore[valid-type,misc] pass def test_multidict_proxy_subclassing( - any_multidict_proxy_class: Type[MultiMapping[str]], + any_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: class DummyMultidictProxy( any_multidict_proxy_class, # type: ignore[valid-type,misc] @@ -123,7 +116,7 @@ def test_multidict_proxy_subclassing( class BaseMultiDictTest: - def test_instantiate__empty(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_instantiate__empty(self, cls: type[MutableMultiMapping[str]]) -> None: d = cls() empty: Mapping[str, str] = {} assert d == empty @@ -133,14 +126,14 @@ class BaseMultiDictTest: assert list(d.items()) == [] assert cls() != list() # type: ignore[comparison-overlap] - with pytest.raises(TypeError, match=r"(2 given)"): + with pytest.raises(TypeError, match=r"3 were given"): cls(("key1", "value1"), ("key2", "value2")) # type: ignore[call-arg] # noqa: E501 @pytest.mark.parametrize("arg0", ([("key", "value1")], {"key": "value1"})) def test_instantiate__from_arg0( self, - cls: Type[MutableMultiMapping[str]], - arg0: Union[List[Tuple[str, str]], Dict[str, str]], + cls: type[MultiDict[str]], + arg0: Union[list[tuple[str, str]], dict[str, str]], ) -> None: d = cls(arg0) @@ -152,7 +145,7 @@ class BaseMultiDictTest: def test_instantiate__with_kwargs( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: d = cls([("key", "value1")], key2="value2") @@ -163,7 +156,7 @@ class BaseMultiDictTest: assert sorted(d.items()) == [("key", "value1"), ("key2", "value2")] def test_instantiate__from_generator( - self, cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]] + self, cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]] ) -> None: d = cls((str(i), i) for i in range(2)) @@ -175,7 +168,7 @@ class BaseMultiDictTest: def test_instantiate__from_list_of_lists( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MutableMultiMapping[str]], ) -> None: # Should work at runtime, but won't type check. d = cls([["key", "value1"]]) # type: ignore[call-arg] @@ -183,7 +176,7 @@ class BaseMultiDictTest: def test_instantiate__from_list_of_custom_pairs( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: class Pair: def __len__(self) -> int: @@ -193,10 +186,10 @@ class BaseMultiDictTest: return ("key", "value1")[pos] # Works at runtime, but won't type check. - d = cls([Pair()]) + d = cls([Pair()]) # type: ignore[list-item] assert d == {"key": "value1"} - def test_getone(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_getone(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert d.getone("key") == "value1" @@ -210,25 +203,42 @@ class BaseMultiDictTest: assert d.getone("key2", "default") == "default" - def test_call_with_kwargs(self, cls: Type[MultiDict[str]]) -> None: + def test_call_with_kwargs(self, cls: type[MultiDict[str]]) -> None: d = cls([("present", "value")]) assert d.getall(default="missing", key="notfound") == "missing" def test__iter__( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) assert list(d) == ["key", "key2", "key"] + def test__contains( + self, + cls: Union[ + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], + ], + ) -> None: + d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) + + assert list(d) == ["key", "key2", "key"] + + assert "key" in d + assert "key2" in d + + assert "foo" not in d + assert 42 not in d # type: ignore[comparison-overlap] + def test_keys__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key2", "two"), ("key", 3)]) @@ -239,12 +249,13 @@ class BaseMultiDictTest: assert "key2" in d.keys() assert "foo" not in d.keys() + assert 42 not in d.keys() # type: ignore[comparison-overlap] def test_values__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key", "two"), ("key", 3)]) @@ -260,8 +271,8 @@ class BaseMultiDictTest: def test_items__contains( self, cls: Union[ - Type[MultiDict[Union[str, int]]], - Type[CIMultiDict[Union[str, int]]], + type[MultiDict[Union[str, int]]], + type[CIMultiDict[Union[str, int]]], ], ) -> None: d = cls([("key", "one"), ("key", "two"), ("key", 3)]) @@ -273,15 +284,17 @@ class BaseMultiDictTest: assert ("key", 3) in d.items() assert ("foo", "bar") not in d.items() + assert (42, 3) not in d.items() # type: ignore[comparison-overlap] + assert 42 not in d.items() # type: ignore[comparison-overlap] def test_cannot_create_from_unaccepted( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MutableMultiMapping[str]], ) -> None: with pytest.raises(TypeError): cls([(1, 2, 3)]) # type: ignore[call-arg] - def test_keys_is_set_less(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_less(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() < {"key", "key2"} @@ -297,8 +310,8 @@ class BaseMultiDictTest: ) def test_keys_is_set_less_equal( self, - cls: Type[MutableMultiMapping[str]], - contents: List[Tuple[str, str]], + cls: type[MultiDict[str]], + contents: list[tuple[str, str]], expected: bool, ) -> None: d = cls(contents) @@ -306,12 +319,17 @@ class BaseMultiDictTest: result = d.keys() <= {"key", "key2"} assert result is expected - def test_keys_is_set_equal(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_equal(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() == {"key"} - def test_keys_is_set_greater(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_items_is_set_equal(self, cls: type[MultiDict[str]]) -> None: + d = cls([("key", "value1")]) + + assert d.items() == {("key", "value1")} + + def test_keys_is_set_greater(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert d.keys() > {"key"} @@ -326,16 +344,14 @@ class BaseMultiDictTest: ), ) def test_keys_is_set_greater_equal( - self, cls: Type[MutableMultiMapping[str]], set_: Set[str], expected: bool + self, cls: type[MultiDict[str]], set_: set[str], expected: bool ) -> None: d = cls([("key", "value1"), ("key2", "value2")]) result = d.keys() >= set_ assert result is expected - def test_keys_less_than_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_less_than_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -348,7 +364,7 @@ class BaseMultiDictTest: assert (d.keys() < RightOperand()) is sentinel_operation_result def test_keys_less_than_or_equal_not_implemented( - self, cls: Type[MutableMultiMapping[str]] + self, cls: type[MultiDict[str]] ) -> None: d = cls([("key", "value1")]) @@ -361,9 +377,7 @@ class BaseMultiDictTest: assert (d.keys() <= RightOperand()) is sentinel_operation_result - def test_keys_greater_than_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_greater_than_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -376,7 +390,7 @@ class BaseMultiDictTest: assert (d.keys() > RightOperand()) is sentinel_operation_result def test_keys_greater_than_or_equal_not_implemented( - self, cls: Type[MutableMultiMapping[str]] + self, cls: type[MultiDict[str]] ) -> None: d = cls([("key", "value1")]) @@ -389,30 +403,28 @@ class BaseMultiDictTest: assert (d.keys() >= RightOperand()) is sentinel_operation_result - def test_keys_is_set_not_equal(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_keys_is_set_not_equal(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d.keys() != {"key2"} - def test_keys_not_equal_unrelated_type( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_keys_not_equal_unrelated_type(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) - assert d.keys() != "other" + assert d.keys() != "other" # type: ignore[comparison-overlap] - def test_eq(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key": "value1"} == d - def test_eq2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq2(self, cls: type[MultiDict[str]]) -> None: d1 = cls([("key", "value1")]) d2 = cls([("key2", "value1")]) assert d1 != d2 - def test_eq3(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_eq3(self, cls: type[MultiDict[str]]) -> None: d1 = cls([("key", "value1")]) d2 = cls() @@ -420,7 +432,7 @@ class BaseMultiDictTest: def test_eq_other_mapping_contains_more_keys( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], ) -> None: d1 = cls(foo="bar") d2 = dict(foo="bar", bar="baz") @@ -428,7 +440,7 @@ class BaseMultiDictTest: assert d1 != d2 def test_eq_bad_mapping_len( - self, cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]] + self, cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]] ) -> None: class BadMapping(Mapping[str, int]): def __getitem__(self, key: str) -> int: @@ -437,8 +449,8 @@ class BaseMultiDictTest: def __iter__(self) -> Iterator[str]: yield "a" # pragma: no cover # `len()` fails earlier - def __len__(self) -> int: # type: ignore[return] - 1 / 0 + def __len__(self) -> int: + return 1 // 0 d1 = cls(a=1) d2 = BadMapping() @@ -447,11 +459,11 @@ class BaseMultiDictTest: def test_eq_bad_mapping_getitem( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: class BadMapping(Mapping[str, int]): - def __getitem__(self, key: str) -> int: # type: ignore[return] - 1 / 0 + def __getitem__(self, key: str) -> int: + return 1 // 0 def __iter__(self) -> Iterator[str]: yield "a" # pragma: no cover # foreign objects no iterated @@ -464,24 +476,22 @@ class BaseMultiDictTest: with pytest.raises(ZeroDivisionError): d1 == d2 - def test_ne(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_ne(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert d != {"key": "another_value"} - def test_and(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_and(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == d.keys() & {"key", "key2"} - def test_and2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_and2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == {"key", "key2"} & d.keys() - def test_bitwise_and_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_and_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -493,26 +503,22 @@ class BaseMultiDictTest: assert d.keys() & RightOperand() is sentinel_operation_result - def test_bitwise_and_iterable_not_set( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_and_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key"} == d.keys() & ["key", "key2"] - def test_or(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_or(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == d.keys() | {"key2"} - def test_or2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_or2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == {"key2"} | d.keys() - def test_bitwise_or_not_implemented( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_or_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) sentinel_operation_result = object() @@ -524,24 +530,22 @@ class BaseMultiDictTest: assert d.keys() | RightOperand() is sentinel_operation_result - def test_bitwise_or_iterable_not_set( - self, cls: Type[MutableMultiMapping[str]] - ) -> None: + def test_bitwise_or_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")]) assert {"key", "key2"} == d.keys() | ["key2"] - def test_sub(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key"} == d.keys() - {"key2"} - def test_sub2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key3"} == {"key", "key2", "key3"} - d.keys() - def test_sub_not_implemented(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) sentinel_operation_result = object() @@ -553,22 +557,22 @@ class BaseMultiDictTest: assert d.keys() - RightOperand() is sentinel_operation_result - def test_sub_iterable_not_set(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_sub_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key"} == d.keys() - ["key2"] - def test_xor(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == d.keys() ^ {"key2", "key3"} - def test_xor2(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor2(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == {"key2", "key3"} ^ d.keys() - def test_xor_not_implemented(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor_not_implemented(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) sentinel_operation_result = object() @@ -580,7 +584,7 @@ class BaseMultiDictTest: assert d.keys() ^ RightOperand() is sentinel_operation_result - def test_xor_iterable_not_set(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_xor_iterable_not_set(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1"), ("key2", "value2")]) assert {"key", "key3"} == d.keys() ^ ["key2", "key3"] @@ -590,13 +594,13 @@ class BaseMultiDictTest: (("key2", "v", True), ("key", "value1", False)), ) def test_isdisjoint( - self, cls: Type[MutableMultiMapping[str]], key: str, value: str, expected: bool + self, cls: type[MultiDict[str]], key: str, value: str, expected: bool ) -> None: d = cls([("key", "value1")]) assert d.items().isdisjoint({(key, value)}) is expected assert d.keys().isdisjoint({key}) is expected - def test_repr_aiohttp_issue_410(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_repr_aiohttp_issue_410(self, cls: type[MutableMultiMapping[str]]) -> None: d = cls() try: @@ -614,9 +618,9 @@ class BaseMultiDictTest: @pytest.mark.parametrize("other", ({"other"},)) def test_op_issue_aiohttp_issue_410( self, - cls: Type[MutableMultiMapping[str]], + cls: type[MultiDict[str]], op: Callable[[object, object], object], - other: Set[str], + other: set[str], ) -> None: d = cls([("key", "value")]) @@ -628,7 +632,7 @@ class BaseMultiDictTest: assert sys.exc_info()[1] == e # noqa: PT017 - def test_weakref(self, cls: Type[MutableMultiMapping[str]]) -> None: + def test_weakref(self, cls: type[MutableMultiMapping[str]]) -> None: called = False def cb(wr: object) -> None: @@ -644,7 +648,7 @@ class BaseMultiDictTest: def test_iter_length_hint_keys( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.keys()) @@ -652,7 +656,7 @@ class BaseMultiDictTest: def test_iter_length_hint_items( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.items()) @@ -660,15 +664,15 @@ class BaseMultiDictTest: def test_iter_length_hint_values( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: md = cls(a=1, b=2) it = iter(md.values()) - assert it.__length_hint__() == 2 # type: ignore[attr-defined] + assert it.__length_hint__() == 2 def test_ctor_list_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = [("a", 1)] obj = cls(arg, b=2) @@ -677,7 +681,7 @@ class BaseMultiDictTest: def test_ctor_tuple_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = (("a", 1),) obj = cls(arg, b=2) @@ -686,7 +690,7 @@ class BaseMultiDictTest: def test_ctor_deque_arg_and_kwds( self, - cls: Union[Type[MultiDict[int]], Type[CIMultiDict[int]]], + cls: Union[type[MultiDict[int]], type[CIMultiDict[int]]], ) -> None: arg = deque([("a", 1)]) obj = cls(arg, b=2) @@ -709,7 +713,7 @@ class TestMultiDict(BaseMultiDictTest): """Make a case-sensitive multidict class/proxy constructor.""" return chained_callable(multidict_module, request.param) - def test__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls() _cls = type(d) @@ -719,7 +723,7 @@ class TestMultiDict(BaseMultiDictTest): assert str(d) == "<%s('key': 'one', 'key': 'two')>" % _cls.__name__ - def test_getall(self, cls: Type[MultiDict[str]]) -> None: + def test_getall(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert d != {"key": "value1"} @@ -735,27 +739,27 @@ class TestMultiDict(BaseMultiDictTest): def test_preserve_stable_ordering( self, - cls: Type[MultiDict[Union[str, int]]], + cls: type[MultiDict[Union[str, int]]], ) -> None: d = cls([("a", 1), ("b", "2"), ("a", 3)]) s = "&".join("{}={}".format(k, v) for k, v in d.items()) assert s == "a=1&b=2&a=3" - def test_get(self, cls: Type[MultiDict[int]]) -> None: + def test_get(self, cls: type[MultiDict[int]]) -> None: d = cls([("a", 1), ("a", 2)]) assert d["a"] == 1 - def test_items__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_items__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") expected = "_ItemsView('key': 'value1', 'key': 'value2')" assert repr(d.items()) == expected - def test_keys__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_keys__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert repr(d.keys()) == "_KeysView('key', 'key')" - def test_values__repr__(self, cls: Type[MultiDict[str]]) -> None: + def test_values__repr__(self, cls: type[MultiDict[str]]) -> None: d = cls([("key", "value1")], key="value2") assert repr(d.values()) == "_ValuesView('value1', 'value2')" @@ -775,7 +779,7 @@ class TestCIMultiDict(BaseMultiDictTest): """Make a case-insensitive multidict class/proxy constructor.""" return chained_callable(multidict_module, request.param) - def test_basics(self, cls: Type[CIMultiDict[str]]) -> None: + def test_basics(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], KEY="value2") assert d.getone("key") == "value1" @@ -789,7 +793,7 @@ class TestCIMultiDict(BaseMultiDictTest): with pytest.raises(KeyError, match="key2"): d.getone("key2") - def test_getall(self, cls: Type[CIMultiDict[str]]) -> None: + def test_getall(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], KEY="value2") assert not d == {"KEY": "value1"} @@ -800,26 +804,26 @@ class TestCIMultiDict(BaseMultiDictTest): with pytest.raises(KeyError, match="some_key"): d.getall("some_key") - def test_get(self, cls: Type[CIMultiDict[int]]) -> None: + def test_get(self, cls: type[CIMultiDict[int]]) -> None: d = cls([("A", 1), ("a", 2)]) assert 1 == d["a"] - def test__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") _cls = type(d) expected = "<%s('KEY': 'value1', 'key': 'value2')>" % _cls.__name__ assert str(d) == expected - def test_items__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_items__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") expected = "_ItemsView('KEY': 'value1', 'key': 'value2')" assert repr(d.items()) == expected - def test_keys__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_keys__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") assert repr(d.keys()) == "_KeysView('KEY', 'key')" - def test_values__repr__(self, cls: Type[CIMultiDict[str]]) -> None: + def test_values__repr__(self, cls: type[CIMultiDict[str]]) -> None: d = cls([("KEY", "value1")], key="value2") assert repr(d.values()) == "_ValuesView('value1', 'value2')" diff --git a/contrib/python/multidict/tests/test_multidict_benchmarks.py b/contrib/python/multidict/tests/test_multidict_benchmarks.py new file mode 100644 index 00000000000..e6a538f3ccf --- /dev/null +++ b/contrib/python/multidict/tests/test_multidict_benchmarks.py @@ -0,0 +1,391 @@ +"""codspeed benchmarks for multidict.""" + +from typing import Dict, Union + +from pytest_codspeed import BenchmarkFixture + +from multidict import CIMultiDict, MultiDict, istr + +# Note that this benchmark should not be refactored to use pytest.mark.parametrize +# since each benchmark name should be unique. + +_SENTINEL = object() + + +def test_multidict_insert_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_cimultidict_insert_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_cimultidict_insert_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict() + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] = i + + +def test_multidict_add_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_cimultidict_add_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict() + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_cimultidict_add_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict() + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.add(i, i) + + +def test_multidict_pop_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_cimultidict_pop_str(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_cimultidict_pop_istr(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + md.pop(i) + + +def test_multidict_popitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md = md_base.copy() + for _ in range(100): + md.popitem() + + +def test_cimultidict_popitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md = md_base.copy() + for _ in range(100): + md.popitem() + + +def test_multidict_clear_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.clear() + + +def test_cimultidict_clear_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.clear() + + +def test_multidict_update_str(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_cimultidict_update_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_cimultidict_update_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items: Dict[Union[str, istr], istr] = {istr(i): istr(i) for i in range(100, 200)} + + @benchmark + def _run() -> None: + md.update(items) + + +def test_multidict_extend_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_cimultidict_extend_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = {str(i): str(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_cimultidict_extend_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = {istr(i): istr(i) for i in range(200)} + + @benchmark + def _run() -> None: + md.extend(items) + + +def test_multidict_delitem_str(benchmark: BenchmarkFixture) -> None: + md_base: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_cimultidict_delitem_str(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_cimultidict_delitem_istr(benchmark: BenchmarkFixture) -> None: + md_base: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + md = md_base.copy() + for i in items: + del md[i] + + +def test_multidict_getall_str_hit(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict(("all", str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall("all") + + +def test_cimultidict_getall_str_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict(("all", str(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall("all") + + +def test_cimultidict_getall_istr_hit(benchmark: BenchmarkFixture) -> None: + all_istr = istr("all") + md: CIMultiDict[istr] = CIMultiDict((all_istr, istr(i)) for i in range(100)) + + @benchmark + def _run() -> None: + md.getall(all_istr) + + +def test_multidict_fetch(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_cimultidict_fetch_str(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_cimultidict_fetch_istr(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md[i] + + +def test_multidict_get_hit(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_multidict_get_miss(benchmark: BenchmarkFixture) -> None: + md: MultiDict[str] = MultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_miss(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_istr_hit(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_istr_miss(benchmark: BenchmarkFixture) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i) + + +def test_cimultidict_get_hit_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_miss_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[str] = CIMultiDict((str(i), str(i)) for i in range(100)) + items = [str(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_istr_hit_with_default( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) + + +def test_cimultidict_get_istr_with_default_miss( + benchmark: BenchmarkFixture, +) -> None: + md: CIMultiDict[istr] = CIMultiDict((istr(i), istr(i)) for i in range(100)) + items = [istr(i) for i in range(100, 200)] + + @benchmark + def _run() -> None: + for i in items: + md.get(i, _SENTINEL) diff --git a/contrib/python/multidict/tests/test_mutable_multidict.py b/contrib/python/multidict/tests/test_mutable_multidict.py index 3cacec25af9..45f1cdf5f67 100644 --- a/contrib/python/multidict/tests/test_mutable_multidict.py +++ b/contrib/python/multidict/tests/test_mutable_multidict.py @@ -1,16 +1,16 @@ import string import sys -from typing import Type +from typing import Union import pytest -from multidict import MultiMapping, MutableMultiMapping +from multidict import CIMultiDict, CIMultiDictProxy, MultiDictProxy, istr class TestMutableMultiDict: def test_copy( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_sensitive_multidict_class(key="value", a="b") @@ -20,7 +20,7 @@ class TestMutableMultiDict: def test__repr__( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() assert str(d) == "<%s()>" % case_sensitive_multidict_class.__name__ @@ -35,7 +35,7 @@ class TestMutableMultiDict: def test_getall( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "value1")], key="value2") assert len(d) == 2 @@ -50,7 +50,7 @@ class TestMutableMultiDict: def test_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() @@ -73,7 +73,7 @@ class TestMutableMultiDict: def test_extend( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[Union[str, int]]], ) -> None: d = case_sensitive_multidict_class() assert d == {} @@ -101,12 +101,12 @@ class TestMutableMultiDict: assert 6 == len(d) with pytest.raises(TypeError): - d.extend("foo", "bar") + d.extend("foo", "bar") # type: ignore[arg-type, call-arg] def test_extend_from_proxy( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_sensitive_multidict_proxy_class: Type[MultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_sensitive_multidict_proxy_class: type[MultiDictProxy[str]], ) -> None: d = case_sensitive_multidict_class([("a", "a"), ("b", "b")]) proxy = case_sensitive_multidict_proxy_class(d) @@ -118,7 +118,7 @@ class TestMutableMultiDict: def test_clear( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one")], key="two", foo="bar") @@ -128,7 +128,7 @@ class TestMutableMultiDict: def test_del( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one"), ("key", "two")], foo="bar") assert list(d.keys()) == ["key", "key", "foo"] @@ -142,7 +142,7 @@ class TestMutableMultiDict: def test_set_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class([("key", "one"), ("key", "two")], foo="bar") assert "one" == d.setdefault("key", "three") @@ -152,7 +152,7 @@ class TestMutableMultiDict: def test_popitem( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -163,7 +163,7 @@ class TestMutableMultiDict: def test_popitem_empty_multidict( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() @@ -172,7 +172,7 @@ class TestMutableMultiDict: def test_pop( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -183,7 +183,7 @@ class TestMutableMultiDict: def test_pop2( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") @@ -195,7 +195,7 @@ class TestMutableMultiDict: def test_pop_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class(other="val") @@ -204,7 +204,7 @@ class TestMutableMultiDict: def test_pop_raises( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class(other="val") @@ -215,7 +215,7 @@ class TestMutableMultiDict: def test_replacement_order( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key1", "val1") @@ -231,16 +231,16 @@ class TestMutableMultiDict: def test_nonstr_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() with pytest.raises(TypeError): - d[1] = "val" + d[1] = "val" # type: ignore[index] def test_istr_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_sensitive_multidict_class() d[case_insensitive_str_class("1")] = "val" @@ -248,7 +248,7 @@ class TestMutableMultiDict: def test_str_derived_key( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: class A(str): pass @@ -259,8 +259,8 @@ class TestMutableMultiDict: def test_istr_key_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_sensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_sensitive_multidict_class() d.add(case_insensitive_str_class("1"), "val") @@ -268,7 +268,7 @@ class TestMutableMultiDict: def test_str_derived_key_add( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: class A(str): pass @@ -279,7 +279,7 @@ class TestMutableMultiDict: def test_popall( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() d.add("key1", "val1") @@ -291,14 +291,14 @@ class TestMutableMultiDict: def test_popall_default( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() assert "val" == d.popall("key", "val") def test_popall_key_error( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_sensitive_multidict_class() with pytest.raises(KeyError, match="key"): @@ -306,7 +306,7 @@ class TestMutableMultiDict: def test_large_multidict_resizing( self, - case_sensitive_multidict_class: Type[MutableMultiMapping[str]], + case_sensitive_multidict_class: type[CIMultiDict[int]], ) -> None: SIZE = 1024 d = case_sensitive_multidict_class() @@ -322,7 +322,7 @@ class TestMutableMultiDict: class TestCIMutableMultiDict: def test_getall( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class([("KEY", "value1")], KEY="value2") @@ -336,7 +336,7 @@ class TestCIMutableMultiDict: def test_ctor( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(k1="v1") assert "v1" == d["K1"] @@ -344,7 +344,7 @@ class TestCIMutableMultiDict: def test_setitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d["k1"] = "v1" @@ -353,7 +353,7 @@ class TestCIMutableMultiDict: def test_delitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d["k1"] = "v1" @@ -363,7 +363,7 @@ class TestCIMutableMultiDict: def test_copy( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_insensitive_multidict_class(key="KEY", a="b") @@ -374,7 +374,7 @@ class TestCIMutableMultiDict: def test__repr__( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() assert str(d) == "<%s()>" % case_insensitive_multidict_class.__name__ @@ -389,7 +389,7 @@ class TestCIMutableMultiDict: def test_add( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() @@ -421,7 +421,7 @@ class TestCIMutableMultiDict: def test_extend( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[Union[str, int]]], ) -> None: d = case_insensitive_multidict_class() assert d == {} @@ -450,12 +450,12 @@ class TestCIMutableMultiDict: assert 6 == len(d) with pytest.raises(TypeError): - d.extend("foo", "bar") + d.extend("foo", "bar") # type: ignore[arg-type, call-arg] def test_extend_from_proxy( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_multidict_proxy_class: Type[MultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_multidict_proxy_class: type[CIMultiDictProxy[str]], ) -> None: d = case_insensitive_multidict_class([("a", "a"), ("b", "b")]) proxy = case_insensitive_multidict_proxy_class(d) @@ -467,7 +467,7 @@ class TestCIMutableMultiDict: def test_clear( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class([("KEY", "one")], key="two", foo="bar") @@ -477,7 +477,7 @@ class TestCIMutableMultiDict: def test_del( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class( [("KEY", "one"), ("key", "two")], @@ -493,7 +493,7 @@ class TestCIMutableMultiDict: def test_set_default( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class( [("KEY", "one"), ("key", "two")], @@ -507,7 +507,7 @@ class TestCIMutableMultiDict: def test_popitem( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -520,7 +520,7 @@ class TestCIMutableMultiDict: def test_popitem_empty_multidict( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() @@ -529,7 +529,7 @@ class TestCIMutableMultiDict: def test_pop( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -540,7 +540,7 @@ class TestCIMutableMultiDict: def test_pop_lowercase( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") @@ -551,7 +551,7 @@ class TestCIMutableMultiDict: def test_pop_default( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(OTHER="val") @@ -560,7 +560,7 @@ class TestCIMutableMultiDict: def test_pop_raises( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d = case_insensitive_multidict_class(OTHER="val") @@ -571,8 +571,8 @@ class TestCIMutableMultiDict: def test_extend_with_istr( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[istr], ) -> None: us = case_insensitive_str_class("aBc") d = case_insensitive_multidict_class() @@ -582,8 +582,8 @@ class TestCIMutableMultiDict: def test_copy_istr( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], - case_insensitive_str_class: Type[str], + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[istr], ) -> None: d = case_insensitive_multidict_class({case_insensitive_str_class("Foo"): "bar"}) d2 = d.copy() @@ -591,7 +591,7 @@ class TestCIMutableMultiDict: def test_eq( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: d1 = case_insensitive_multidict_class(Key="val") d2 = case_insensitive_multidict_class(KEY="val") @@ -604,7 +604,7 @@ class TestCIMutableMultiDict: ) def test_sizeof( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: md = case_insensitive_multidict_class() s1 = sys.getsizeof(md) @@ -621,14 +621,14 @@ class TestCIMutableMultiDict: ) def test_min_sizeof( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: md = case_insensitive_multidict_class() assert sys.getsizeof(md) < 1024 def test_issue_620_items( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) @@ -639,7 +639,7 @@ class TestCIMutableMultiDict: def test_issue_620_keys( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) @@ -650,7 +650,7 @@ class TestCIMutableMultiDict: def test_issue_620_values( self, - case_insensitive_multidict_class: Type[MutableMultiMapping[str]], + case_insensitive_multidict_class: type[CIMultiDict[str]], ) -> None: # https://github.com/aio-libs/multidict/issues/620 d = case_insensitive_multidict_class({"a": "123, 456", "b": "789"}) diff --git a/contrib/python/multidict/tests/test_pickle.py b/contrib/python/multidict/tests/test_pickle.py index 48adea13f0f..3159ea45c64 100644 --- a/contrib/python/multidict/tests/test_pickle.py +++ b/contrib/python/multidict/tests/test_pickle.py @@ -1,13 +1,21 @@ import pickle from pathlib import Path +from typing import TYPE_CHECKING import pytest +from multidict import MultiDict, MultiDictProxy + +if TYPE_CHECKING: + from conftest import MultidictImplementation + import yatest.common as yc here = Path(yc.source_path(__file__)).resolve().parent -def test_pickle(any_multidict_class, pickle_protocol): +def test_pickle( + any_multidict_class: type[MultiDict[int]], pickle_protocol: int +) -> None: d = any_multidict_class([("a", 1), ("a", 2)]) pbytes = pickle.dumps(d, pickle_protocol) obj = pickle.loads(pbytes) @@ -15,14 +23,21 @@ def test_pickle(any_multidict_class, pickle_protocol): assert isinstance(obj, any_multidict_class) -def test_pickle_proxy(any_multidict_class, any_multidict_proxy_class): +def test_pickle_proxy( + any_multidict_class: type[MultiDict[int]], + any_multidict_proxy_class: type[MultiDictProxy[int]], +) -> None: d = any_multidict_class([("a", 1), ("a", 2)]) proxy = any_multidict_proxy_class(d) with pytest.raises(TypeError): pickle.dumps(proxy) -def test_load_from_file(any_multidict_class, multidict_implementation, pickle_protocol): +def test_load_from_file( + any_multidict_class: type[MultiDict[int]], + multidict_implementation: "MultidictImplementation", + pickle_protocol: int, +) -> None: multidict_class_name = any_multidict_class.__name__ pickle_file_basename = "-".join( ( diff --git a/contrib/python/multidict/tests/test_types.py b/contrib/python/multidict/tests/test_types.py index ceaa391e379..6339006b68f 100644 --- a/contrib/python/multidict/tests/test_types.py +++ b/contrib/python/multidict/tests/test_types.py @@ -1,52 +1,57 @@ -import sys import types import pytest -def test_proxies(multidict_module): +def test_proxies(multidict_module: types.ModuleType) -> None: assert issubclass( multidict_module.CIMultiDictProxy, multidict_module.MultiDictProxy, ) -def test_dicts(multidict_module): +def test_dicts(multidict_module: types.ModuleType) -> None: assert issubclass(multidict_module.CIMultiDict, multidict_module.MultiDict) -def test_proxy_not_inherited_from_dict(multidict_module): +def test_proxy_not_inherited_from_dict(multidict_module: types.ModuleType) -> None: assert not issubclass(multidict_module.MultiDictProxy, multidict_module.MultiDict) -def test_dict_not_inherited_from_proxy(multidict_module): +def test_dict_not_inherited_from_proxy(multidict_module: types.ModuleType) -> None: assert not issubclass(multidict_module.MultiDict, multidict_module.MultiDictProxy) -def test_multidict_proxy_copy_type(multidict_module): +def test_multidict_proxy_copy_type(multidict_module: types.ModuleType) -> None: d = multidict_module.MultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert isinstance(p.copy(), multidict_module.MultiDict) -def test_cimultidict_proxy_copy_type(multidict_module): +def test_cimultidict_proxy_copy_type(multidict_module: types.ModuleType) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.CIMultiDictProxy(d) assert isinstance(p.copy(), multidict_module.CIMultiDict) -def test_create_multidict_proxy_from_nonmultidict(multidict_module): +def test_create_multidict_proxy_from_nonmultidict( + multidict_module: types.ModuleType, +) -> None: with pytest.raises(TypeError): multidict_module.MultiDictProxy({}) -def test_create_multidict_proxy_from_cimultidict(multidict_module): +def test_create_multidict_proxy_from_cimultidict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert p == d -def test_create_multidict_proxy_from_multidict_proxy_from_mdict(multidict_module): +def test_create_multidict_proxy_from_multidict_proxy_from_mdict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.MultiDict(key="val") p = multidict_module.MultiDictProxy(d) assert p == d @@ -54,7 +59,9 @@ def test_create_multidict_proxy_from_multidict_proxy_from_mdict(multidict_module assert p2 == p -def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci(multidict_module): +def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.CIMultiDict(key="val") p = multidict_module.CIMultiDictProxy(d) assert p == d @@ -62,7 +69,9 @@ def test_create_cimultidict_proxy_from_cimultidict_proxy_from_ci(multidict_modul assert p2 == p -def test_create_cimultidict_proxy_from_nonmultidict(multidict_module): +def test_create_cimultidict_proxy_from_nonmultidict( + multidict_module: types.ModuleType, +) -> None: with pytest.raises( TypeError, match=( @@ -73,7 +82,9 @@ def test_create_cimultidict_proxy_from_nonmultidict(multidict_module): multidict_module.CIMultiDictProxy({}) -def test_create_ci_multidict_proxy_from_multidict(multidict_module): +def test_create_ci_multidict_proxy_from_multidict( + multidict_module: types.ModuleType, +) -> None: d = multidict_module.MultiDict(key="val") with pytest.raises( TypeError, @@ -85,20 +96,7 @@ def test_create_ci_multidict_proxy_from_multidict(multidict_module): multidict_module.CIMultiDictProxy(d) - sys.version_info >= (3, 9), reason="Python 3.9 uses GenericAlias which is different" -) -def test_generic_exists(multidict_module) -> None: - assert multidict_module.MultiDict[int] is multidict_module.MultiDict - assert multidict_module.MultiDictProxy[int] is multidict_module.MultiDictProxy - assert multidict_module.CIMultiDict[int] is multidict_module.CIMultiDict - assert multidict_module.CIMultiDictProxy[int] is multidict_module.CIMultiDictProxy - - - sys.version_info < (3, 9), reason="Python 3.9 is required for GenericAlias" -) -def test_generic_alias(multidict_module) -> None: +def test_generic_alias(multidict_module: types.ModuleType) -> None: assert multidict_module.MultiDict[int] == types.GenericAlias( multidict_module.MultiDict, (int,) ) diff --git a/contrib/python/multidict/tests/test_update.py b/contrib/python/multidict/tests/test_update.py index f4553278571..46ab30a08bd 100644 --- a/contrib/python/multidict/tests/test_update.py +++ b/contrib/python/multidict/tests/test_update.py @@ -1,10 +1,12 @@ from collections import deque -from typing import Type +from typing import Union -from multidict import MultiMapping +from multidict import CIMultiDict, MultiDict +_MD_Classes = Union[type[MultiDict[int]], type[CIMultiDict[int]]] -def test_update_replace(any_multidict_class: Type[MultiMapping[str]]) -> None: + +def test_update_replace(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4), ("b", 5), ("a", 6)]) obj1.update(obj2) @@ -12,7 +14,7 @@ def test_update_replace(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_append(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_append(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4), ("a", 5), ("a", 6)]) obj1.update(obj2) @@ -20,7 +22,7 @@ def test_update_append(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_remove(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_remove(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = any_multidict_class([("a", 4)]) obj1.update(obj2) @@ -28,7 +30,7 @@ def test_update_remove(any_multidict_class: Type[MultiMapping[str]]) -> None: assert list(obj1.items()) == expected -def test_update_replace_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_replace_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4), ("b", 5), ("a", 6)] obj1.update(obj2) @@ -36,14 +38,14 @@ def test_update_replace_seq(any_multidict_class: Type[MultiMapping[str]]) -> Non assert list(obj1.items()) == expected -def test_update_replace_seq2(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_replace_seq2(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj1.update([("a", 4)], b=5, a=6) expected = [("a", 4), ("b", 5), ("a", 6), ("c", 10)] assert list(obj1.items()) == expected -def test_update_append_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_append_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4), ("a", 5), ("a", 6)] obj1.update(obj2) @@ -51,7 +53,7 @@ def test_update_append_seq(any_multidict_class: Type[MultiMapping[str]]) -> None assert list(obj1.items()) == expected -def test_update_remove_seq(any_multidict_class: Type[MultiMapping[str]]) -> None: +def test_update_remove_seq(any_multidict_class: _MD_Classes) -> None: obj1 = any_multidict_class([("a", 1), ("b", 2), ("a", 3), ("c", 10)]) obj2 = [("a", 4)] obj1.update(obj2) @@ -59,9 +61,7 @@ def test_update_remove_seq(any_multidict_class: Type[MultiMapping[str]]) -> None assert list(obj1.items()) == expected -def test_update_md( - case_sensitive_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_md(case_sensitive_multidict_class: type[CIMultiDict[str]]) -> None: d = case_sensitive_multidict_class() d.add("key", "val1") d.add("key", "val2") @@ -73,8 +73,8 @@ def test_update_md( def test_update_istr_ci_md( - case_insensitive_multidict_class: Type[MultiMapping[str]], - case_insensitive_str_class: str, + case_insensitive_multidict_class: type[CIMultiDict[str]], + case_insensitive_str_class: type[str], ) -> None: d = case_insensitive_multidict_class() d.add(case_insensitive_str_class("KEY"), "val1") @@ -86,9 +86,7 @@ def test_update_istr_ci_md( assert [("key", "val"), ("key2", "val3")] == list(d.items()) -def test_update_ci_md( - case_insensitive_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_ci_md(case_insensitive_multidict_class: type[CIMultiDict[str]]) -> None: d = case_insensitive_multidict_class() d.add("KEY", "val1") d.add("key", "val2") @@ -99,9 +97,7 @@ def test_update_ci_md( assert [("Key", "val"), ("key2", "val3")] == list(d.items()) -def test_update_list_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_list_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = [("a", 1)] obj.update(arg, b=2) @@ -109,9 +105,7 @@ def test_update_list_arg_and_kwds( assert arg == [("a", 1)] -def test_update_tuple_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_tuple_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = (("a", 1),) obj.update(arg, b=2) @@ -119,9 +113,7 @@ def test_update_tuple_arg_and_kwds( assert arg == (("a", 1),) -def test_update_deque_arg_and_kwds( - any_multidict_class: Type[MultiMapping[str]], -) -> None: +def test_update_deque_arg_and_kwds(any_multidict_class: _MD_Classes) -> None: obj = any_multidict_class() arg = deque([("a", 1)]) obj.update(arg, b=2) diff --git a/contrib/python/multidict/tests/test_version.py b/contrib/python/multidict/tests/test_version.py index e004afa1124..4fe209c6786 100644 --- a/contrib/python/multidict/tests/test_version.py +++ b/contrib/python/multidict/tests/test_version.py @@ -1,18 +1,25 @@ -from typing import Callable, Type +from collections.abc import Callable +from typing import TypeVar, Union import pytest -from multidict import MultiMapping +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +_T = TypeVar("_T") +_MD_Types = Union[ + MultiDict[_T], CIMultiDict[_T], MultiDictProxy[_T], CIMultiDictProxy[_T] +] +GetVersion = Callable[[_MD_Types[_T]], int] -def test_getversion_bad_param(multidict_getversion_callable): + +def test_getversion_bad_param(multidict_getversion_callable: GetVersion[str]) -> None: with pytest.raises(TypeError): - multidict_getversion_callable(1) + multidict_getversion_callable(1) # type: ignore[arg-type] def test_ctor( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m1 = any_multidict_class() v1 = multidict_getversion_callable(m1) @@ -22,8 +29,8 @@ def test_ctor( def test_add( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() v = multidict_getversion_callable(m) @@ -32,8 +39,8 @@ def test_add( def test_delitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -43,8 +50,8 @@ def test_delitem( def test_delitem_not_found( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -55,8 +62,8 @@ def test_delitem_not_found( def test_setitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -66,8 +73,8 @@ def test_setitem( def test_setitem_not_found( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -77,8 +84,8 @@ def test_setitem_not_found( def test_clear( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -88,8 +95,8 @@ def test_clear( def test_setdefault( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -99,8 +106,8 @@ def test_setdefault( def test_popone( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -110,8 +117,8 @@ def test_popone( def test_popone_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -121,8 +128,8 @@ def test_popone_default( def test_popone_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -133,8 +140,8 @@ def test_popone_key_error( def test_pop( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -144,8 +151,8 @@ def test_pop( def test_pop_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -155,8 +162,8 @@ def test_pop_default( def test_pop_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -167,8 +174,8 @@ def test_pop_key_error( def test_popall( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -178,8 +185,8 @@ def test_popall( def test_popall_default( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -189,8 +196,8 @@ def test_popall_default( def test_popall_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -201,8 +208,8 @@ def test_popall_key_error( def test_popitem( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() m.add("key", "val") @@ -212,8 +219,8 @@ def test_popitem( def test_popitem_key_error( - any_multidict_class: Type[MultiMapping[str]], - multidict_getversion_callable: Callable, + any_multidict_class: type[MultiDict[str]], + multidict_getversion_callable: GetVersion[str], ) -> None: m = any_multidict_class() v = multidict_getversion_callable(m) diff --git a/contrib/python/multidict/ya.make b/contrib/python/multidict/ya.make index 8a2950eae96..626036249b8 100644 --- a/contrib/python/multidict/ya.make +++ b/contrib/python/multidict/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.1.0) +VERSION(6.2.0) LICENSE(Apache-2.0) @@ -25,7 +25,6 @@ PY_REGISTER( PY_SRCS( TOP_LEVEL multidict/__init__.py - multidict/__init__.pyi multidict/_abc.py multidict/_compat.py multidict/_multidict_base.py diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index 1f52419b882..b6911ce75e8 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.20.0 +Version: 3.19.3 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index 847d0def900..71cfb8fa720 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.20.0) +VERSION(3.19.3) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/ydb/_apis.py b/contrib/python/ydb/py3/ydb/_apis.py index fc6f16e287c..fc28d0ceb29 100644 --- a/contrib/python/ydb/py3/ydb/_apis.py +++ b/contrib/python/ydb/py3/ydb/_apis.py @@ -115,7 +115,6 @@ class TopicService(object): DropTopic = "DropTopic" StreamRead = "StreamRead" StreamWrite = "StreamWrite" - UpdateOffsetsInTransaction = "UpdateOffsetsInTransaction" class QueryService(object): diff --git a/contrib/python/ydb/py3/ydb/_errors.py b/contrib/python/ydb/py3/ydb/_errors.py index 1e2308ef394..17002d25749 100644 --- a/contrib/python/ydb/py3/ydb/_errors.py +++ b/contrib/python/ydb/py3/ydb/_errors.py @@ -5,7 +5,6 @@ from . import issues _errors_retriable_fast_backoff_types = [ issues.Unavailable, - issues.ClientInternalError, ] _errors_retriable_slow_backoff_types = [ issues.Aborted, diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/common_utils.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/common_utils.py index 10d98918c30..95a5744313e 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/common_utils.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/common_utils.py @@ -160,6 +160,9 @@ class GrpcWrapperAsyncIO(IGrpcWrapperAsyncIO): self._stream_call = None self._wait_executor = None + def __del__(self): + self._clean_executor(wait=False) + async def start(self, driver: SupportedDriverType, stub, method): if asyncio.iscoroutinefunction(driver.__call__): await self._start_asyncio_driver(driver, stub, method) diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py index 0f8a0f03a7a..5b22c7cf862 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py @@ -141,18 +141,6 @@ class UpdateTokenResponse(IFromProto): ######################################################################################################################## -@dataclass -class TransactionIdentity(IToProto): - tx_id: str - session_id: str - - def to_proto(self) -> ydb_topic_pb2.TransactionIdentity: - return ydb_topic_pb2.TransactionIdentity( - id=self.tx_id, - session=self.session_id, - ) - - class StreamWriteMessage: @dataclass() class InitRequest(IToProto): @@ -211,7 +199,6 @@ class StreamWriteMessage: class WriteRequest(IToProto): messages: typing.List["StreamWriteMessage.WriteRequest.MessageData"] codec: int - tx_identity: Optional[TransactionIdentity] @dataclass class MessageData(IToProto): @@ -250,9 +237,6 @@ class StreamWriteMessage: proto = ydb_topic_pb2.StreamWriteMessage.WriteRequest() proto.codec = self.codec - if self.tx_identity is not None: - proto.tx.CopyFrom(self.tx_identity.to_proto()) - for message in self.messages: proto_mess = proto.messages.add() proto_mess.CopyFrom(message.to_proto()) @@ -313,8 +297,6 @@ class StreamWriteMessage: ) except ValueError: message_write_status = reason - elif proto_ack.HasField("written_in_tx"): - message_write_status = StreamWriteMessage.WriteResponse.WriteAck.StatusWrittenInTx() else: raise NotImplementedError("unexpected ack status") @@ -327,9 +309,6 @@ class StreamWriteMessage: class StatusWritten: offset: int - class StatusWrittenInTx: - pass - @dataclass class StatusSkipped: reason: "StreamWriteMessage.WriteResponse.WriteAck.StatusSkipped.Reason" @@ -1218,52 +1197,6 @@ class MeteringMode(int, IFromProto, IFromPublic, IToPublic): @dataclass -class UpdateOffsetsInTransactionRequest(IToProto): - tx: TransactionIdentity - topics: List[UpdateOffsetsInTransactionRequest.TopicOffsets] - consumer: str - - def to_proto(self): - return ydb_topic_pb2.UpdateOffsetsInTransactionRequest( - tx=self.tx.to_proto(), - consumer=self.consumer, - topics=list( - map( - UpdateOffsetsInTransactionRequest.TopicOffsets.to_proto, - self.topics, - ) - ), - ) - - @dataclass - class TopicOffsets(IToProto): - path: str - partitions: List[UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets] - - def to_proto(self): - return ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets( - path=self.path, - partitions=list( - map( - UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets.to_proto, - self.partitions, - ) - ), - ) - - @dataclass - class PartitionOffsets(IToProto): - partition_id: int - partition_offsets: List[OffsetsRange] - - def to_proto(self) -> ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets: - return ydb_topic_pb2.UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets( - partition_id=self.partition_id, - partition_offsets=list(map(OffsetsRange.to_proto, self.partition_offsets)), - ) - - -@dataclass class CreateTopicRequest(IToProto, IFromPublic): path: str partitioning_settings: "PartitioningSettings" diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py b/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py index 74f06a086fc..b48501aff2f 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/datatypes.py @@ -108,9 +108,6 @@ class PartitionSession: waiter = self._ack_waiters.popleft() waiter._finish_ok() - def _update_last_commited_offset_if_needed(self, offset: int): - self.committed_offset = max(self.committed_offset, offset) - def close(self): if self.closed: return @@ -214,9 +211,3 @@ class PublicBatch(ICommittable, ISessionAlive): self._bytes_size = self._bytes_size - new_batch._bytes_size return new_batch - - def _update_partition_offsets(self, tx, exc=None): - if exc is not None: - return - offsets = self._commit_get_offsets_range() - self._partition_session._update_last_commited_offset_if_needed(offsets.end) diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py index c9704d5542a..7061b4e449c 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_asyncio.py @@ -5,7 +5,7 @@ import concurrent.futures import gzip import typing from asyncio import Task -from collections import defaultdict, OrderedDict +from collections import OrderedDict from typing import Optional, Set, Dict, Union, Callable import ydb @@ -19,24 +19,17 @@ from . import topic_reader from .._grpc.grpcwrapper.common_utils import ( IGrpcWrapperAsyncIO, SupportedDriverType, - to_thread, GrpcWrapperAsyncIO, ) from .._grpc.grpcwrapper.ydb_topic import ( StreamReadMessage, UpdateTokenRequest, UpdateTokenResponse, - UpdateOffsetsInTransactionRequest, Codec, ) from .._errors import check_retriable_error import logging -from ..query.base import TxEvent - -if typing.TYPE_CHECKING: - from ..query.transaction import BaseQueryTxContext - logger = logging.getLogger(__name__) @@ -84,7 +77,7 @@ class PublicAsyncIOReader: ): self._loop = asyncio.get_running_loop() self._closed = False - self._reconnector = ReaderReconnector(driver, settings, self._loop) + self._reconnector = ReaderReconnector(driver, settings) self._parent = _parent async def __aenter__(self): @@ -95,7 +88,8 @@ class PublicAsyncIOReader: def __del__(self): if not self._closed: - logger.warning("Topic reader was not closed properly. Consider using method close().") + task = self._loop.create_task(self.close(flush=False)) + topic_common.wrap_set_name_for_asyncio_task(task, task_name="close reader") async def wait_message(self): """ @@ -118,23 +112,6 @@ class PublicAsyncIOReader: max_messages=max_messages, ) - async def receive_batch_with_tx( - self, - tx: "BaseQueryTxContext", - max_messages: typing.Union[int, None] = None, - ) -> typing.Union[datatypes.PublicBatch, None]: - """ - Get one messages batch with tx from reader. - All messages in a batch from same partition. - - use asyncio.wait_for for wait with timeout. - """ - await self._reconnector.wait_message() - return self._reconnector.receive_batch_with_tx_nowait( - tx=tx, - max_messages=max_messages, - ) - async def receive_message(self) -> typing.Optional[datatypes.PublicMessage]: """ Block until receive new message @@ -188,18 +165,11 @@ class ReaderReconnector: _state_changed: asyncio.Event _stream_reader: Optional["ReaderStream"] _first_error: asyncio.Future[YdbError] - _tx_to_batches_map: Dict[str, typing.List[datatypes.PublicBatch]] - def __init__( - self, - driver: Driver, - settings: topic_reader.PublicReaderSettings, - loop: Optional[asyncio.AbstractEventLoop] = None, - ): + def __init__(self, driver: Driver, settings: topic_reader.PublicReaderSettings): self._id = self._static_reader_reconnector_counter.inc_and_get() self._settings = settings self._driver = driver - self._loop = loop if loop is not None else asyncio.get_running_loop() self._background_tasks = set() self._state_changed = asyncio.Event() @@ -207,8 +177,6 @@ class ReaderReconnector: self._background_tasks.add(asyncio.create_task(self._connection_loop())) self._first_error = asyncio.get_running_loop().create_future() - self._tx_to_batches_map = dict() - async def _connection_loop(self): attempt = 0 while True: @@ -222,7 +190,6 @@ class ReaderReconnector: if not retry_info.is_retriable: self._set_first_error(err) return - await asyncio.sleep(retry_info.sleep_timeout_seconds) attempt += 1 @@ -255,87 +222,9 @@ class ReaderReconnector: max_messages=max_messages, ) - def receive_batch_with_tx_nowait(self, tx: "BaseQueryTxContext", max_messages: Optional[int] = None): - batch = self._stream_reader.receive_batch_nowait( - max_messages=max_messages, - ) - - self._init_tx(tx) - - self._tx_to_batches_map[tx.tx_id].append(batch) - - tx._add_callback(TxEvent.AFTER_COMMIT, batch._update_partition_offsets, self._loop) - - return batch - def receive_message_nowait(self): return self._stream_reader.receive_message_nowait() - def _init_tx(self, tx: "BaseQueryTxContext"): - if tx.tx_id not in self._tx_to_batches_map: # Init tx callbacks - self._tx_to_batches_map[tx.tx_id] = [] - tx._add_callback(TxEvent.BEFORE_COMMIT, self._commit_batches_with_tx, self._loop) - tx._add_callback(TxEvent.AFTER_COMMIT, self._handle_after_tx_commit, self._loop) - tx._add_callback(TxEvent.AFTER_ROLLBACK, self._handle_after_tx_rollback, self._loop) - - async def _commit_batches_with_tx(self, tx: "BaseQueryTxContext"): - grouped_batches = defaultdict(lambda: defaultdict(list)) - for batch in self._tx_to_batches_map[tx.tx_id]: - grouped_batches[batch._partition_session.topic_path][batch._partition_session.partition_id].append(batch) - - request = UpdateOffsetsInTransactionRequest(tx=tx._tx_identity(), consumer=self._settings.consumer, topics=[]) - - for topic_path in grouped_batches: - topic_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets(path=topic_path, partitions=[]) - for partition_id in grouped_batches[topic_path]: - partition_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets( - partition_id=partition_id, - partition_offsets=[ - batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id] - ], - ) - topic_offsets.partitions.append(partition_offsets) - request.topics.append(topic_offsets) - - try: - return await self._do_commit_batches_with_tx_call(request) - except BaseException: - exc = issues.ClientInternalError("Failed to update offsets in tx.") - tx._set_external_error(exc) - self._stream_reader._set_first_error(exc) - finally: - del self._tx_to_batches_map[tx.tx_id] - - async def _do_commit_batches_with_tx_call(self, request: UpdateOffsetsInTransactionRequest): - args = [ - request.to_proto(), - _apis.TopicService.Stub, - _apis.TopicService.UpdateOffsetsInTransaction, - topic_common.wrap_operation, - ] - - if asyncio.iscoroutinefunction(self._driver.__call__): - res = await self._driver(*args) - else: - res = await to_thread(self._driver, *args, executor=None) - - return res - - async def _handle_after_tx_rollback(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None: - if tx.tx_id in self._tx_to_batches_map: - del self._tx_to_batches_map[tx.tx_id] - exc = issues.ClientInternalError("Reconnect due to transaction rollback") - self._stream_reader._set_first_error(exc) - - async def _handle_after_tx_commit(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None: - if tx.tx_id in self._tx_to_batches_map: - del self._tx_to_batches_map[tx.tx_id] - - if exc is not None: - self._stream_reader._set_first_error( - issues.ClientInternalError("Reconnect due to transaction commit failed") - ) - def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.CommitAckWaiter: return self._stream_reader.commit(batch) diff --git a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py index 3e6806d06fe..eda1d374fc3 100644 --- a/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py +++ b/contrib/python/ydb/py3/ydb/_topic_reader/topic_reader_sync.py @@ -1,6 +1,5 @@ import asyncio import concurrent.futures -import logging import typing from typing import List, Union, Optional @@ -21,11 +20,6 @@ from ydb._topic_reader.topic_reader_asyncio import ( TopicReaderClosedError, ) -if typing.TYPE_CHECKING: - from ..query.transaction import BaseQueryTxContext - -logger = logging.getLogger(__name__) - class TopicReaderSync: _caller: CallFromSyncToAsync @@ -58,8 +52,7 @@ class TopicReaderSync: self._parent = _parent def __del__(self): - if not self._closed: - logger.warning("Topic reader was not closed properly. Consider using method close().") + self.close(flush=False) def __enter__(self): return self @@ -116,31 +109,6 @@ class TopicReaderSync: timeout, ) - def receive_batch_with_tx( - self, - tx: "BaseQueryTxContext", - *, - max_messages: typing.Union[int, None] = None, - max_bytes: typing.Union[int, None] = None, - timeout: Union[float, None] = None, - ) -> Union[PublicBatch, None]: - """ - Get one messages batch with tx from reader - It has no async_ version for prevent lost messages, use async_wait_message as signal for new batches available. - - if no new message in timeout seconds (default - infinite): raise TimeoutError() - if timeout <= 0 - it will fast wait only one event loop cycle - without wait any i/o operations or pauses, get messages from internal buffer only. - """ - self._check_closed() - - return self._caller.safe_call_with_result( - self._async_reader.receive_batch_with_tx( - tx=tx, - max_messages=max_messages, - ), - timeout, - ) - def commit(self, mess: typing.Union[datatypes.PublicMessage, datatypes.PublicBatch]): """ Put commit message to internal buffer. diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py index a3e407ed86d..aa5fe9749a7 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer.py @@ -11,7 +11,6 @@ import typing import ydb.aio from .._grpc.grpcwrapper.ydb_topic import StreamWriteMessage -from .._grpc.grpcwrapper.ydb_topic import TransactionIdentity from .._grpc.grpcwrapper.common_utils import IToProto from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .. import connection @@ -54,12 +53,8 @@ class PublicWriteResult: class Skipped: pass - @dataclass(eq=True) - class WrittenInTx: - pass - -PublicWriteResultTypes = Union[PublicWriteResult.Written, PublicWriteResult.Skipped, PublicWriteResult.WrittenInTx] +PublicWriteResultTypes = Union[PublicWriteResult.Written, PublicWriteResult.Skipped] class WriterSettings(PublicWriterSettings): @@ -210,7 +205,6 @@ def default_serializer_message_content(data: Any) -> bytes: def messages_to_proto_requests( messages: List[InternalMessage], - tx_identity: Optional[TransactionIdentity], ) -> List[StreamWriteMessage.FromClient]: gropus = _slit_messages_for_send(messages) @@ -221,7 +215,6 @@ def messages_to_proto_requests( StreamWriteMessage.WriteRequest( messages=list(map(InternalMessage.to_message_data, group)), codec=group[0].codec, - tx_identity=tx_identity, ) ) res.append(req) @@ -246,7 +239,6 @@ _message_data_overhead = ( ), ], codec=20000, - tx_identity=None, ) ) .to_proto() diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py index 1ea6c25028b..32d8fefe51c 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_asyncio.py @@ -1,6 +1,7 @@ import asyncio import concurrent.futures import datetime +import functools import gzip import typing from collections import deque @@ -34,7 +35,6 @@ from .._grpc.grpcwrapper.ydb_topic import ( UpdateTokenRequest, UpdateTokenResponse, StreamWriteMessage, - TransactionIdentity, WriterMessagesFromServerToClient, ) from .._grpc.grpcwrapper.common_utils import ( @@ -43,11 +43,6 @@ from .._grpc.grpcwrapper.common_utils import ( GrpcWrapperAsyncIO, ) -from ..query.base import TxEvent - -if typing.TYPE_CHECKING: - from ..query.transaction import BaseQueryTxContext - logger = logging.getLogger(__name__) @@ -79,8 +74,10 @@ class WriterAsyncIO: raise def __del__(self): - if not self._closed: - logger.warning("Topic writer was not closed properly. Consider using method close().") + if self._closed or self._loop.is_closed(): + return + + self._loop.call_soon(functools.partial(self.close, flush=False)) async def close(self, *, flush: bool = True): if self._closed: @@ -167,57 +164,6 @@ class WriterAsyncIO: return await self._reconnector.wait_init() -class TxWriterAsyncIO(WriterAsyncIO): - _tx: "BaseQueryTxContext" - - def __init__( - self, - tx: "BaseQueryTxContext", - driver: SupportedDriverType, - settings: PublicWriterSettings, - _client=None, - _is_implicit=False, - ): - self._tx = tx - self._loop = asyncio.get_running_loop() - self._closed = False - self._reconnector = WriterAsyncIOReconnector(driver=driver, settings=WriterSettings(settings), tx=self._tx) - self._parent = _client - self._is_implicit = _is_implicit - - # For some reason, creating partition could conflict with other session operations. - # Could be removed later. - self._first_write = True - - tx._add_callback(TxEvent.BEFORE_COMMIT, self._on_before_commit, self._loop) - tx._add_callback(TxEvent.BEFORE_ROLLBACK, self._on_before_rollback, self._loop) - - async def write( - self, - messages: Union[Message, List[Message]], - ): - """ - send one or number of messages to server. - it put message to internal buffer - - For wait with timeout use asyncio.wait_for. - """ - if self._first_write: - self._first_write = False - return await super().write_with_ack(messages) - return await super().write(messages) - - async def _on_before_commit(self, tx: "BaseQueryTxContext"): - if self._is_implicit: - return - await self.close() - - async def _on_before_rollback(self, tx: "BaseQueryTxContext"): - if self._is_implicit: - return - await self.close(flush=False) - - class WriterAsyncIOReconnector: _closed: bool _loop: asyncio.AbstractEventLoop @@ -232,7 +178,6 @@ class WriterAsyncIOReconnector: _codec_selector_batch_num: int _codec_selector_last_codec: Optional[PublicCodec] _codec_selector_check_batches_interval: int - _tx: Optional["BaseQueryTxContext"] if typing.TYPE_CHECKING: _messages_for_encode: asyncio.Queue[List[InternalMessage]] @@ -250,9 +195,7 @@ class WriterAsyncIOReconnector: _stop_reason: asyncio.Future _init_info: Optional[PublicWriterInitInfo] - def __init__( - self, driver: SupportedDriverType, settings: WriterSettings, tx: Optional["BaseQueryTxContext"] = None - ): + def __init__(self, driver: SupportedDriverType, settings: WriterSettings): self._closed = False self._loop = asyncio.get_running_loop() self._driver = driver @@ -262,7 +205,6 @@ class WriterAsyncIOReconnector: self._init_info = None self._stream_connected = asyncio.Event() self._settings = settings - self._tx = tx self._codec_functions = { PublicCodec.RAW: lambda data: data, @@ -412,12 +354,10 @@ class WriterAsyncIOReconnector: # noinspection PyBroadException stream_writer = None try: - tx_identity = None if self._tx is None else self._tx._tx_identity() stream_writer = await WriterAsyncIOStream.create( self._driver, self._init_message, self._settings.update_token_interval, - tx_identity=tx_identity, ) try: if self._init_info is None: @@ -447,7 +387,7 @@ class WriterAsyncIOReconnector: done.pop().result() # need for raise exception - reason of stop task except issues.Error as err: err_info = check_retriable_error(err, retry_settings, attempt) - if not err_info.is_retriable or self._tx is not None: # no retries in tx writer + if not err_info.is_retriable: self._stop(err) return @@ -593,8 +533,6 @@ class WriterAsyncIOReconnector: result = PublicWriteResult.Skipped() elif isinstance(status, write_ack_msg.StatusWritten): result = PublicWriteResult.Written(offset=status.offset) - elif isinstance(status, write_ack_msg.StatusWrittenInTx): - result = PublicWriteResult.WrittenInTx() else: raise TopicWriterError("internal error - receive unexpected ack message.") message_future.set_result(result) @@ -659,13 +597,10 @@ class WriterAsyncIOStream: _update_token_event: asyncio.Event _get_token_function: Optional[Callable[[], str]] - _tx_identity: Optional[TransactionIdentity] - def __init__( self, update_token_interval: Optional[Union[int, float]] = None, get_token_function: Optional[Callable[[], str]] = None, - tx_identity: Optional[TransactionIdentity] = None, ): self._closed = False @@ -674,8 +609,6 @@ class WriterAsyncIOStream: self._update_token_event = asyncio.Event() self._update_token_task = None - self._tx_identity = tx_identity - async def close(self): if self._closed: return @@ -692,7 +625,6 @@ class WriterAsyncIOStream: driver: SupportedDriverType, init_request: StreamWriteMessage.InitRequest, update_token_interval: Optional[Union[int, float]] = None, - tx_identity: Optional[TransactionIdentity] = None, ) -> "WriterAsyncIOStream": stream = GrpcWrapperAsyncIO(StreamWriteMessage.FromServer.from_proto) @@ -702,7 +634,6 @@ class WriterAsyncIOStream: writer = WriterAsyncIOStream( update_token_interval=update_token_interval, get_token_function=creds.get_auth_token if creds else lambda: "", - tx_identity=tx_identity, ) await writer._start(stream, init_request) return writer @@ -749,7 +680,7 @@ class WriterAsyncIOStream: if self._closed: raise RuntimeError("Can not write on closed stream.") - for request in messages_to_proto_requests(messages, self._tx_identity): + for request in messages_to_proto_requests(messages): self._stream.write(request) async def _update_token_loop(self): diff --git a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py index 4796d7ac2d6..a5193caf7c5 100644 --- a/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py +++ b/contrib/python/ydb/py3/ydb/_topic_writer/topic_writer_sync.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import logging import typing from concurrent.futures import Future from typing import Union, List, Optional @@ -15,23 +14,13 @@ from .topic_writer import ( TopicWriterClosedError, ) -from ..query.base import TxEvent - -from .topic_writer_asyncio import ( - TxWriterAsyncIO, - WriterAsyncIO, -) +from .topic_writer_asyncio import WriterAsyncIO from .._topic_common.common import ( _get_shared_event_loop, TimeoutType, CallFromSyncToAsync, ) -if typing.TYPE_CHECKING: - from ..query.transaction import BaseQueryTxContext - -logger = logging.getLogger(__name__) - class WriterSync: _caller: CallFromSyncToAsync @@ -74,8 +63,7 @@ class WriterSync: raise def __del__(self): - if not self._closed: - logger.warning("Topic writer was not closed properly. Consider using method close().") + self.close(flush=False) def close(self, *, flush: bool = True, timeout: TimeoutType = None): if self._closed: @@ -134,39 +122,3 @@ class WriterSync: self._check_closed() return self._caller.unsafe_call_with_result(self._async_writer.write_with_ack(messages), timeout=timeout) - - -class TxWriterSync(WriterSync): - def __init__( - self, - tx: "BaseQueryTxContext", - driver: SupportedDriverType, - settings: PublicWriterSettings, - *, - eventloop: Optional[asyncio.AbstractEventLoop] = None, - _parent=None, - ): - - self._closed = False - - if eventloop: - loop = eventloop - else: - loop = _get_shared_event_loop() - - self._caller = CallFromSyncToAsync(loop) - - async def create_async_writer(): - return TxWriterAsyncIO(tx, driver, settings, _is_implicit=True) - - self._async_writer = self._caller.safe_call_with_result(create_async_writer(), None) - self._parent = _parent - - tx._add_callback(TxEvent.BEFORE_COMMIT, self._on_before_commit, None) - tx._add_callback(TxEvent.BEFORE_ROLLBACK, self._on_before_rollback, None) - - def _on_before_commit(self, tx: "BaseQueryTxContext"): - self.close() - - def _on_before_rollback(self, tx: "BaseQueryTxContext"): - self.close(flush=False) diff --git a/contrib/python/ydb/py3/ydb/aio/driver.py b/contrib/python/ydb/py3/ydb/aio/driver.py index 267997fbcc3..9cd6fd2b74d 100644 --- a/contrib/python/ydb/py3/ydb/aio/driver.py +++ b/contrib/python/ydb/py3/ydb/aio/driver.py @@ -62,5 +62,4 @@ class Driver(pool.ConnectionPool): async def stop(self, timeout=10): await self.table_client._stop_pool_if_needed(timeout=timeout) - self.topic_client.close() await super().stop(timeout=timeout) diff --git a/contrib/python/ydb/py3/ydb/aio/query/pool.py b/contrib/python/ydb/py3/ydb/aio/query/pool.py index f1ca68d1cf0..947db658726 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/pool.py +++ b/contrib/python/ydb/py3/ydb/aio/query/pool.py @@ -158,8 +158,6 @@ class QuerySessionPool: async def wrapped_callee(): async with self.checkout() as session: async with session.transaction(tx_mode=tx_mode) as tx: - if tx_mode.name in ["serializable_read_write", "snapshot_read_only"]: - await tx.begin() result = await callee(tx, *args, **kwargs) await tx.commit() return result @@ -215,6 +213,12 @@ class QuerySessionPool: async def __aexit__(self, exc_type, exc_val, exc_tb): await self.stop() + def __del__(self): + if self._should_stop.is_set() or self._loop.is_closed(): + return + + self._loop.call_soon(self.stop) + class SimpleQuerySessionCheckoutAsync: def __init__(self, pool: QuerySessionPool): diff --git a/contrib/python/ydb/py3/ydb/aio/query/transaction.py b/contrib/python/ydb/py3/ydb/aio/query/transaction.py index f0547e5f01f..5b63a32b489 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/aio/query/transaction.py @@ -16,28 +16,6 @@ logger = logging.getLogger(__name__) class QueryTxContext(BaseQueryTxContext): - def __init__(self, driver, session_state, session, tx_mode): - """ - An object that provides a simple transaction context manager that allows statements execution - in a transaction. You don't have to open transaction explicitly, because context manager encapsulates - transaction control logic, and opens new transaction if: - - 1) By explicit .begin() method; - 2) On execution of a first statement, which is strictly recommended method, because that avoids useless round trip - - This context manager is not thread-safe, so you should not manipulate on it concurrently. - - :param driver: A driver instance - :param session_state: A state of session - :param tx_mode: Transaction mode, which is a one from the following choises: - 1) QuerySerializableReadWrite() which is default mode; - 2) QueryOnlineReadOnly(allow_inconsistent_reads=False); - 3) QuerySnapshotReadOnly(); - 4) QueryStaleReadOnly(). - """ - super().__init__(driver, session_state, session, tx_mode) - self._init_callback_handler(base.CallbackHandlerMode.ASYNC) - async def __aenter__(self) -> "QueryTxContext": """ Enters a context manager and returns a transaction @@ -52,7 +30,7 @@ class QueryTxContext(BaseQueryTxContext): it is not finished explicitly """ await self._ensure_prev_stream_finished() - if self._tx_state._state == QueryTxStateEnum.BEGINED and self._external_error is None: + if self._tx_state._state == QueryTxStateEnum.BEGINED: # It's strictly recommended to close transactions directly # by using commit_tx=True flag while executing statement or by # .commit() or .rollback() methods, but here we trying to do best @@ -87,9 +65,7 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - self._check_external_error_set() - - if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): + if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -98,13 +74,7 @@ class QueryTxContext(BaseQueryTxContext): await self._ensure_prev_stream_finished() - try: - await self._execute_callbacks_async(base.TxEvent.BEFORE_COMMIT) - await self._commit_call(settings) - await self._execute_callbacks_async(base.TxEvent.AFTER_COMMIT, exc=None) - except BaseException as e: - await self._execute_callbacks_async(base.TxEvent.AFTER_COMMIT, exc=e) - raise e + await self._commit_call(settings) async def rollback(self, settings: Optional[BaseRequestSettings] = None) -> None: """Calls rollback on a transaction if it is open otherwise is no-op. If transaction execution @@ -114,9 +84,7 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - self._check_external_error_set() - - if self._tx_state._should_skip(QueryTxStateEnum.ROLLBACKED): + if self._tx_state._already_in(QueryTxStateEnum.ROLLBACKED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -125,13 +93,7 @@ class QueryTxContext(BaseQueryTxContext): await self._ensure_prev_stream_finished() - try: - await self._execute_callbacks_async(base.TxEvent.BEFORE_ROLLBACK) - await self._rollback_call(settings) - await self._execute_callbacks_async(base.TxEvent.AFTER_ROLLBACK, exc=None) - except BaseException as e: - await self._execute_callbacks_async(base.TxEvent.AFTER_ROLLBACK, exc=e) - raise e + await self._rollback_call(settings) async def execute( self, diff --git a/contrib/python/ydb/py3/ydb/driver.py b/contrib/python/ydb/py3/ydb/driver.py index 3998aeeef5f..49bd223c901 100644 --- a/contrib/python/ydb/py3/ydb/driver.py +++ b/contrib/python/ydb/py3/ydb/driver.py @@ -288,5 +288,4 @@ class Driver(pool.ConnectionPool): def stop(self, timeout=10): self.table_client._stop_pool_if_needed(timeout=timeout) - self.topic_client.close() super().stop(timeout=timeout) diff --git a/contrib/python/ydb/py3/ydb/issues.py b/contrib/python/ydb/py3/ydb/issues.py index 4e76f5ed2b0..f38f99f9257 100644 --- a/contrib/python/ydb/py3/ydb/issues.py +++ b/contrib/python/ydb/py3/ydb/issues.py @@ -178,10 +178,6 @@ class SessionPoolEmpty(Error, queue.Empty): status = StatusCode.SESSION_POOL_EMPTY -class ClientInternalError(Error): - status = StatusCode.CLIENT_INTERNAL_ERROR - - class UnexpectedGrpcMessage(Error): def __init__(self, message: str): super().__init__(message) diff --git a/contrib/python/ydb/py3/ydb/query/base.py b/contrib/python/ydb/py3/ydb/query/base.py index a5ebedd95b3..57a769bb1a1 100644 --- a/contrib/python/ydb/py3/ydb/query/base.py +++ b/contrib/python/ydb/py3/ydb/query/base.py @@ -1,8 +1,6 @@ import abc -import asyncio import enum import functools -from collections import defaultdict import typing from typing import ( @@ -19,10 +17,6 @@ from .. import issues from .. import _utilities from .. import _apis -from ydb._topic_common.common import CallFromSyncToAsync, _get_shared_event_loop -from ydb._grpc.grpcwrapper.common_utils import to_thread - - if typing.TYPE_CHECKING: from .transaction import BaseQueryTxContext @@ -202,64 +196,3 @@ def wrap_execute_query_response( return convert.ResultSet.from_message(response_pb.result_set, settings) return None - - -class TxEvent(enum.Enum): - BEFORE_COMMIT = "BEFORE_COMMIT" - AFTER_COMMIT = "AFTER_COMMIT" - BEFORE_ROLLBACK = "BEFORE_ROLLBACK" - AFTER_ROLLBACK = "AFTER_ROLLBACK" - - -class CallbackHandlerMode(enum.Enum): - SYNC = "SYNC" - ASYNC = "ASYNC" - - -def _get_sync_callback(method: typing.Callable, loop: Optional[asyncio.AbstractEventLoop]): - if asyncio.iscoroutinefunction(method): - if loop is None: - loop = _get_shared_event_loop() - - def async_to_sync_callback(*args, **kwargs): - caller = CallFromSyncToAsync(loop) - return caller.safe_call_with_result(method(*args, **kwargs), 10) - - return async_to_sync_callback - return method - - -def _get_async_callback(method: typing.Callable): - if asyncio.iscoroutinefunction(method): - return method - - async def sync_to_async_callback(*args, **kwargs): - return await to_thread(method, *args, **kwargs, executor=None) - - return sync_to_async_callback - - -class CallbackHandler: - def _init_callback_handler(self, mode: CallbackHandlerMode) -> None: - self._callbacks = defaultdict(list) - self._callback_mode = mode - - def _execute_callbacks_sync(self, event_name: str, *args, **kwargs) -> None: - for callback in self._callbacks[event_name]: - callback(self, *args, **kwargs) - - async def _execute_callbacks_async(self, event_name: str, *args, **kwargs) -> None: - tasks = [asyncio.create_task(callback(self, *args, **kwargs)) for callback in self._callbacks[event_name]] - if not tasks: - return - await asyncio.gather(*tasks) - - def _prepare_callback( - self, callback: typing.Callable, loop: Optional[asyncio.AbstractEventLoop] - ) -> typing.Callable: - if self._callback_mode == CallbackHandlerMode.SYNC: - return _get_sync_callback(callback, loop) - return _get_async_callback(callback) - - def _add_callback(self, event_name: str, callback: typing.Callable, loop: Optional[asyncio.AbstractEventLoop]): - self._callbacks[event_name].append(self._prepare_callback(callback, loop)) diff --git a/contrib/python/ydb/py3/ydb/query/pool.py b/contrib/python/ydb/py3/ydb/query/pool.py index b25f7db855c..e3775c4dd12 100644 --- a/contrib/python/ydb/py3/ydb/query/pool.py +++ b/contrib/python/ydb/py3/ydb/query/pool.py @@ -167,8 +167,6 @@ class QuerySessionPool: def wrapped_callee(): with self.checkout(timeout=retry_settings.max_session_acquire_timeout) as session: with session.transaction(tx_mode=tx_mode) as tx: - if tx_mode.name in ["serializable_read_write", "snapshot_read_only"]: - tx.begin() result = callee(tx, *args, **kwargs) tx.commit() return result @@ -226,6 +224,9 @@ class QuerySessionPool: def __exit__(self, exc_type, exc_val, exc_tb): self.stop() + def __del__(self): + self.stop() + class SimpleQuerySessionCheckout: def __init__(self, pool: QuerySessionPool, timeout: Optional[float]): diff --git a/contrib/python/ydb/py3/ydb/query/transaction.py b/contrib/python/ydb/py3/ydb/query/transaction.py index ae7642dbe21..414401da4d1 100644 --- a/contrib/python/ydb/py3/ydb/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/query/transaction.py @@ -11,7 +11,6 @@ from .. import ( _apis, issues, ) -from .._grpc.grpcwrapper import ydb_topic as _ydb_topic from .._grpc.grpcwrapper import ydb_query as _ydb_query from ..connection import _RpcState as RpcState @@ -43,23 +42,11 @@ class QueryTxStateHelper(abc.ABC): QueryTxStateEnum.DEAD: [], } - _SKIP_TRANSITIONS = { - QueryTxStateEnum.NOT_INITIALIZED: [], - QueryTxStateEnum.BEGINED: [], - QueryTxStateEnum.COMMITTED: [QueryTxStateEnum.COMMITTED, QueryTxStateEnum.ROLLBACKED], - QueryTxStateEnum.ROLLBACKED: [QueryTxStateEnum.COMMITTED, QueryTxStateEnum.ROLLBACKED], - QueryTxStateEnum.DEAD: [], - } - @classmethod def valid_transition(cls, before: QueryTxStateEnum, after: QueryTxStateEnum) -> bool: return after in cls._VALID_TRANSITIONS[before] @classmethod - def should_skip(cls, before: QueryTxStateEnum, after: QueryTxStateEnum) -> bool: - return after in cls._SKIP_TRANSITIONS[before] - - @classmethod def terminal(cls, state: QueryTxStateEnum) -> bool: return len(cls._VALID_TRANSITIONS[state]) == 0 @@ -101,8 +88,8 @@ class QueryTxState: if QueryTxStateHelper.terminal(self._state): raise RuntimeError(f"Transaction is in terminal state: {self._state.value}") - def _should_skip(self, target: QueryTxStateEnum) -> bool: - return QueryTxStateHelper.should_skip(self._state, target) + def _already_in(self, target: QueryTxStateEnum) -> bool: + return self._state == target def _construct_tx_settings(tx_state: QueryTxState) -> _ydb_query.TransactionSettings: @@ -183,7 +170,7 @@ def wrap_tx_rollback_response( return tx -class BaseQueryTxContext(base.CallbackHandler): +class BaseQueryTxContext: def __init__(self, driver, session_state, session, tx_mode): """ An object that provides a simple transaction context manager that allows statements execution @@ -209,7 +196,6 @@ class BaseQueryTxContext(base.CallbackHandler): self._session_state = session_state self.session = session self._prev_stream = None - self._external_error = None @property def session_id(self) -> str: @@ -229,19 +215,6 @@ class BaseQueryTxContext(base.CallbackHandler): """ return self._tx_state.tx_id - def _tx_identity(self) -> _ydb_topic.TransactionIdentity: - if not self.tx_id: - raise RuntimeError("Unable to get tx identity without started tx.") - return _ydb_topic.TransactionIdentity(self.tx_id, self.session_id) - - def _set_external_error(self, exc: BaseException) -> None: - self._external_error = exc - - def _check_external_error_set(self): - if self._external_error is None: - return - raise issues.ClientInternalError("Transaction was failed by external error.") from self._external_error - def _begin_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": self._tx_state._check_invalid_transition(QueryTxStateEnum.BEGINED) @@ -255,7 +228,6 @@ class BaseQueryTxContext(base.CallbackHandler): ) def _commit_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": - self._check_external_error_set() self._tx_state._check_invalid_transition(QueryTxStateEnum.COMMITTED) return self._driver( @@ -268,7 +240,6 @@ class BaseQueryTxContext(base.CallbackHandler): ) def _rollback_call(self, settings: Optional[BaseRequestSettings]) -> "BaseQueryTxContext": - self._check_external_error_set() self._tx_state._check_invalid_transition(QueryTxStateEnum.ROLLBACKED) return self._driver( @@ -291,7 +262,6 @@ class BaseQueryTxContext(base.CallbackHandler): settings: Optional[BaseRequestSettings], ) -> Iterable[_apis.ydb_query.ExecuteQueryResponsePart]: self._tx_state._check_tx_ready_to_use() - self._check_external_error_set() request = base.create_execute_query_request( query=query, @@ -313,41 +283,18 @@ class BaseQueryTxContext(base.CallbackHandler): ) def _move_to_beginned(self, tx_id: str) -> None: - if self._tx_state._should_skip(QueryTxStateEnum.BEGINED) or not tx_id: + if self._tx_state._already_in(QueryTxStateEnum.BEGINED) or not tx_id: return self._tx_state._change_state(QueryTxStateEnum.BEGINED) self._tx_state.tx_id = tx_id def _move_to_commited(self) -> None: - if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): + if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): return self._tx_state._change_state(QueryTxStateEnum.COMMITTED) class QueryTxContext(BaseQueryTxContext): - def __init__(self, driver, session_state, session, tx_mode): - """ - An object that provides a simple transaction context manager that allows statements execution - in a transaction. You don't have to open transaction explicitly, because context manager encapsulates - transaction control logic, and opens new transaction if: - - 1) By explicit .begin() method; - 2) On execution of a first statement, which is strictly recommended method, because that avoids useless round trip - - This context manager is not thread-safe, so you should not manipulate on it concurrently. - - :param driver: A driver instance - :param session_state: A state of session - :param tx_mode: Transaction mode, which is a one from the following choises: - 1) QuerySerializableReadWrite() which is default mode; - 2) QueryOnlineReadOnly(allow_inconsistent_reads=False); - 3) QuerySnapshotReadOnly(); - 4) QueryStaleReadOnly(). - """ - - super().__init__(driver, session_state, session, tx_mode) - self._init_callback_handler(base.CallbackHandlerMode.SYNC) - def __enter__(self) -> "BaseQueryTxContext": """ Enters a context manager and returns a transaction @@ -362,7 +309,7 @@ class QueryTxContext(BaseQueryTxContext): it is not finished explicitly """ self._ensure_prev_stream_finished() - if self._tx_state._state == QueryTxStateEnum.BEGINED and self._external_error is None: + if self._tx_state._state == QueryTxStateEnum.BEGINED: # It's strictly recommended to close transactions directly # by using commit_tx=True flag while executing statement or by # .commit() or .rollback() methods, but here we trying to do best @@ -398,8 +345,7 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - self._check_external_error_set() - if self._tx_state._should_skip(QueryTxStateEnum.COMMITTED): + if self._tx_state._already_in(QueryTxStateEnum.COMMITTED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -408,13 +354,7 @@ class QueryTxContext(BaseQueryTxContext): self._ensure_prev_stream_finished() - try: - self._execute_callbacks_sync(base.TxEvent.BEFORE_COMMIT) - self._commit_call(settings) - self._execute_callbacks_sync(base.TxEvent.AFTER_COMMIT, exc=None) - except BaseException as e: # TODO: probably should be less wide - self._execute_callbacks_sync(base.TxEvent.AFTER_COMMIT, exc=e) - raise e + self._commit_call(settings) def rollback(self, settings: Optional[BaseRequestSettings] = None) -> None: """Calls rollback on a transaction if it is open otherwise is no-op. If transaction execution @@ -424,8 +364,7 @@ class QueryTxContext(BaseQueryTxContext): :return: A committed transaction or exception if commit is failed """ - self._check_external_error_set() - if self._tx_state._should_skip(QueryTxStateEnum.ROLLBACKED): + if self._tx_state._already_in(QueryTxStateEnum.ROLLBACKED): return if self._tx_state._state == QueryTxStateEnum.NOT_INITIALIZED: @@ -434,13 +373,7 @@ class QueryTxContext(BaseQueryTxContext): self._ensure_prev_stream_finished() - try: - self._execute_callbacks_sync(base.TxEvent.BEFORE_ROLLBACK) - self._rollback_call(settings) - self._execute_callbacks_sync(base.TxEvent.AFTER_ROLLBACK, exc=None) - except BaseException as e: # TODO: probably should be less wide - self._execute_callbacks_sync(base.TxEvent.AFTER_ROLLBACK, exc=e) - raise e + self._rollback_call(settings) def execute( self, diff --git a/contrib/python/ydb/py3/ydb/topic.py b/contrib/python/ydb/py3/ydb/topic.py index 52f98e61d85..55f4ea04c5c 100644 --- a/contrib/python/ydb/py3/ydb/topic.py +++ b/contrib/python/ydb/py3/ydb/topic.py @@ -25,8 +25,6 @@ __all__ = [ "TopicWriteResult", "TopicWriter", "TopicWriterAsyncIO", - "TopicTxWriter", - "TopicTxWriterAsyncIO", "TopicWriterInitInfo", "TopicWriterMessage", "TopicWriterSettings", @@ -35,7 +33,6 @@ __all__ = [ import concurrent.futures import datetime from dataclasses import dataclass -import logging from typing import List, Union, Mapping, Optional, Dict, Callable from . import aio, Credentials, _apis, issues @@ -68,10 +65,8 @@ from ._topic_writer.topic_writer import ( # noqa: F401 PublicWriteResult as TopicWriteResult, ) -from ydb._topic_writer.topic_writer_asyncio import TxWriterAsyncIO as TopicTxWriterAsyncIO from ydb._topic_writer.topic_writer_asyncio import WriterAsyncIO as TopicWriterAsyncIO from ._topic_writer.topic_writer_sync import WriterSync as TopicWriter -from ._topic_writer.topic_writer_sync import TxWriterSync as TopicTxWriter from ._topic_common.common import ( wrap_operation as _wrap_operation, @@ -93,8 +88,6 @@ from ._grpc.grpcwrapper.ydb_topic_public_types import ( # noqa: F401 PublicAlterAutoPartitioningSettings as TopicAlterAutoPartitioningSettings, ) -logger = logging.getLogger(__name__) - class TopicClientAsyncIO: _closed: bool @@ -115,8 +108,7 @@ class TopicClientAsyncIO: ) def __del__(self): - if not self._closed: - logger.warning("Topic client was not closed properly. Consider using method close().") + self.close() async def create_topic( self, @@ -284,35 +276,6 @@ class TopicClientAsyncIO: return TopicWriterAsyncIO(self._driver, settings, _client=self) - def tx_writer( - self, - tx, - topic, - *, - producer_id: Optional[str] = None, # default - random - session_metadata: Mapping[str, str] = None, - partition_id: Union[int, None] = None, - auto_seqno: bool = True, - auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select - # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes - # the func will be called from multiply threads in parallel. - encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, - # custom encoder executor for call builtin and custom decoders. If None - use shared executor pool. - # If max_worker in the executor is 1 - then encoders will be called from the thread without parallel. - encoder_executor: Optional[concurrent.futures.Executor] = None, - ) -> TopicTxWriterAsyncIO: - args = locals().copy() - del args["self"] - del args["tx"] - - settings = TopicWriterSettings(**args) - - if not settings.encoder_executor: - settings.encoder_executor = self._executor - - return TopicTxWriterAsyncIO(tx=tx, driver=self._driver, settings=settings, _client=self) - def close(self): if self._closed: return @@ -324,7 +287,7 @@ class TopicClientAsyncIO: if not self._closed: return - raise issues.Error("Topic client closed") + raise RuntimeError("Topic client closed") class TopicClient: @@ -347,8 +310,7 @@ class TopicClient: ) def __del__(self): - if not self._closed: - logger.warning("Topic client was not closed properly. Consider using method close().") + self.close() def create_topic( self, @@ -525,36 +487,6 @@ class TopicClient: return TopicWriter(self._driver, settings, _parent=self) - def tx_writer( - self, - tx, - topic, - *, - producer_id: Optional[str] = None, # default - random - session_metadata: Mapping[str, str] = None, - partition_id: Union[int, None] = None, - auto_seqno: bool = True, - auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select - # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes - # the func will be called from multiply threads in parallel. - encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, - # custom encoder executor for call builtin and custom decoders. If None - use shared executor pool. - # If max_worker in the executor is 1 - then encoders will be called from the thread without parallel. - encoder_executor: Optional[concurrent.futures.Executor] = None, # default shared client executor pool - ) -> TopicWriter: - args = locals().copy() - del args["self"] - del args["tx"] - self._check_closed() - - settings = TopicWriterSettings(**args) - - if not settings.encoder_executor: - settings.encoder_executor = self._executor - - return TopicTxWriter(tx, self._driver, settings, _parent=self) - def close(self): if self._closed: return @@ -566,7 +498,7 @@ class TopicClient: if not self._closed: return - raise issues.Error("Topic client closed") + raise RuntimeError("Topic client closed") @dataclass diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index 070a2455ef3..8bd658d49e4 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.20.0" +VERSION = "3.19.3" |