diff options
author | shadchin <shadchin@yandex-team.com> | 2024-02-12 07:53:52 +0300 |
---|---|---|
committer | shadchin <shadchin@yandex-team.com> | 2024-02-12 08:07:36 +0300 |
commit | ce1b7ca3171f9158180640c6a02a74b4afffedea (patch) | |
tree | e47c1e8391b1b0128262c1e9b1e6ed4c8fff2348 /contrib/tools/python3/src/Lib/typing.py | |
parent | 57350d96f030db90f220ce50ee591d5c5d403df7 (diff) | |
download | ydb-ce1b7ca3171f9158180640c6a02a74b4afffedea.tar.gz |
Update Python from 3.11.8 to 3.12.2
Diffstat (limited to 'contrib/tools/python3/src/Lib/typing.py')
-rw-r--r-- | contrib/tools/python3/src/Lib/typing.py | 1001 |
1 files changed, 460 insertions, 541 deletions
diff --git a/contrib/tools/python3/src/Lib/typing.py b/contrib/tools/python3/src/Lib/typing.py index 5289818950..ffe7ce8d8a 100644 --- a/contrib/tools/python3/src/Lib/typing.py +++ b/contrib/tools/python3/src/Lib/typing.py @@ -23,6 +23,7 @@ from abc import abstractmethod, ABCMeta import collections from collections import defaultdict import collections.abc +import copyreg import contextlib import functools import operator @@ -32,12 +33,16 @@ import types import warnings from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias - -try: - from _typing import _idfunc -except ImportError: - def _idfunc(_, x): - return x +from _typing import ( + _idfunc, + TypeVar, + ParamSpec, + TypeVarTuple, + ParamSpecArgs, + ParamSpecKwargs, + TypeAliasType, + Generic, +) # Please keep __all__ alphabetized within each category. __all__ = [ @@ -138,6 +143,7 @@ __all__ = [ 'NoReturn', 'NotRequired', 'overload', + 'override', 'ParamSpecArgs', 'ParamSpecKwargs', 'Required', @@ -148,6 +154,7 @@ __all__ = [ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeAliasType', 'Unpack', ] @@ -235,16 +242,20 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - if isinstance(obj, types.GenericAlias): - return repr(obj) + # When changing this function, don't forget about + # `_collections_abc._type_repr`, which does the same thing + # and must be consistent with this one. if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: - return('...') + return '...' if isinstance(obj, types.FunctionType): return obj.__name__ + if isinstance(obj, tuple): + # Special case for `repr` of types with `ParamSpec`: + return '[' + ', '.join(_type_repr(t) for t in obj) + ']' return repr(obj) @@ -345,6 +356,7 @@ def _flatten_literal_params(parameters): _cleanups = [] +_caches = {} def _tp_cache(func=None, /, *, typed=False): @@ -353,13 +365,20 @@ def _tp_cache(func=None, /, *, typed=False): For non-hashable arguments, the original function is used as a fallback. """ def decorator(func): - cached = functools.lru_cache(typed=typed)(func) - _cleanups.append(cached.cache_clear) + # The callback 'inner' references the newly created lru_cache + # indirectly by performing a lookup in the global '_caches' dictionary. + # This breaks a reference that can be problematic when combined with + # C API extensions that leak references to types. See GH-98253. + + cache = functools.lru_cache(typed=typed)(func) + _caches[func] = cache + _cleanups.append(cache.cache_clear) + del cache @functools.wraps(func) def inner(*args, **kwds): try: - return cached(*args, **kwds) + return _caches[func](*args, **kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(*args, **kwds) @@ -413,17 +432,6 @@ class _Final: if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") -class _Immutable: - """Mixin to indicate that object should not be copied.""" - - __slots__ = () - - def __copy__(self): - return self - - def __deepcopy__(self, memo): - return self - class _NotIterable: """Mixin to prevent iteration, without being compatible with Iterable. @@ -706,6 +714,15 @@ def Union(self, parameters): return _UnionGenericAlias(self, parameters, name="Optional") return _UnionGenericAlias(self, parameters) +def _make_union(left, right): + """Used from the C implementation of TypeVar. + + TypeVar.__or__ calls this instead of returning types.UnionType + because we want to allow unions between TypeVars and strings + (forward references). + """ + return Union[left, right] + @_SpecialForm def Optional(self, parameters): """Optional[X] is equivalent to Union[X, None].""" @@ -787,8 +804,7 @@ def Concatenate(self, parameters): "ParamSpec variable or ellipsis.") msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) - return _ConcatenateGenericAlias(self, parameters, - _paramspec_tvars=True) + return _ConcatenateGenericAlias(self, parameters) @_SpecialForm @@ -941,313 +957,162 @@ class _PickleUsingNameMixin: return self.__name__ -class _BoundVarianceMixin: - """Mixin giving __init__ bound and variance arguments. - - This is used by TypeVar and ParamSpec, which both employ the notions of - a type 'bound' (restricting type arguments to be a subtype of some - specified type) and type 'variance' (determining subtype relations between - generic types). - """ - def __init__(self, bound, covariant, contravariant): - """Used to setup TypeVars and ParamSpec's bound, covariant and - contravariant attributes. - """ - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None - - def __or__(self, right): - return Union[self, right] - - def __ror__(self, left): - return Union[left, self] - - def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - - -class TypeVar(_Final, _Immutable, _BoundVarianceMixin, _PickleUsingNameMixin, - _root=True): - """Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows: - - def repeat(x: T, n: int) -> List[T]: - '''Return a list containing n references to x.''' - return [x]*n - - def longest(x: A, y: A) -> A: - '''Return the longest of two strings.''' - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of (str, str) -> str and (bytes, bytes) -> bytes. Also note - that if the arguments are instances of some subclass of str, - the return type is still plain str. - - At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. - - Type variables defined with covariant=True or contravariant=True - can be used to declare covariant or contravariant generic types. - See PEP 484 for more details. By default generic types are invariant - in all type variables. - - Type variables can be introspected. e.g.: - - T.__name__ == 'T' - T.__constraints__ == () - T.__covariant__ == False - T.__contravariant__ = False - A.__constraints__ == (str, bytes) - - Note that only type variables defined in global scope can be pickled. - """ - - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - if constraints and bound is not None: - raise TypeError("Constraints cannot be combined with bound=...") - if constraints and len(constraints) == 1: - raise TypeError("A single constraint is not allowed") - msg = "TypeVar(name, constraint, ...): constraints must be types." - self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - def_mod = _caller() - if def_mod != 'typing': - self.__module__ = def_mod +def _typevar_subst(self, arg): + msg = "Parameters to generic types must be types." + arg = _type_check(arg, msg, is_argument=True) + if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or + (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): + raise TypeError(f"{arg} is not valid as type argument") + return arg - def __typing_subst__(self, arg): - msg = "Parameters to generic types must be types." - arg = _type_check(arg, msg, is_argument=True) - if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or - (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): - raise TypeError(f"{arg} is not valid as type argument") - return arg +def _typevartuple_prepare_subst(self, alias, args): + params = alias.__parameters__ + typevartuple_index = params.index(self) + for param in params[typevartuple_index + 1:]: + if isinstance(param, TypeVarTuple): + raise TypeError(f"More than one TypeVarTuple parameter in {alias}") + + alen = len(args) + plen = len(params) + left = typevartuple_index + right = plen - typevartuple_index - 1 + var_tuple_index = None + fillarg = None + for k, arg in enumerate(args): + if not isinstance(arg, type): + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs and len(subargs) == 2 and subargs[-1] is ...: + if var_tuple_index is not None: + raise TypeError("More than one unpacked arbitrary-length tuple argument") + var_tuple_index = k + fillarg = subargs[0] + if var_tuple_index is not None: + left = min(left, var_tuple_index) + right = min(right, alen - var_tuple_index - 1) + elif left + right > alen: + raise TypeError(f"Too few arguments for {alias};" + f" actual {alen}, expected at least {plen-1}") -class TypeVarTuple(_Final, _Immutable, _PickleUsingNameMixin, _root=True): - """Type variable tuple. + return ( + *args[:left], + *([fillarg]*(typevartuple_index - left)), + tuple(args[left: alen - right]), + *([fillarg]*(plen - right - left - typevartuple_index - 1)), + *args[alen - right:], + ) - Usage: - Ts = TypeVarTuple('Ts') # Can be given any name +def _paramspec_subst(self, arg): + if isinstance(arg, (list, tuple)): + arg = tuple(_type_check(a, "Expected a type.") for a in arg) + elif not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + return arg - Just as a TypeVar (type variable) is a placeholder for a single type, - a TypeVarTuple is a placeholder for an *arbitrary* number of types. For - example, if we define a generic class using a TypeVarTuple: - class C(Generic[*Ts]): ... +def _paramspec_prepare_subst(self, alias, args): + params = alias.__parameters__ + i = params.index(self) + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not _is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i+1:]) + return args - Then we can parameterize that class with an arbitrary number of type - arguments: - C[int] # Fine - C[int, str] # Also fine - C[()] # Even this is fine +@_tp_cache +def _generic_class_getitem(cls, params): + """Parameterizes a generic class. - For more details, see PEP 646. + At least, parameterizing a generic class is the *main* thing this method + does. For example, for some generic class `Foo`, this is called when we + do `Foo[int]` - there, with `cls=Foo` and `params=int`. - Note that only TypeVarTuples defined in global scope can be pickled. + However, note that this method is also called when defining generic + classes in the first place with `class Foo(Generic[T]): ...`. """ + if not isinstance(params, tuple): + params = (params,) - def __init__(self, name): - self.__name__ = name - - # Used for pickling. - def_mod = _caller() - if def_mod != 'typing': - self.__module__ = def_mod - - def __iter__(self): - yield Unpack[self] - - def __repr__(self): - return self.__name__ + params = tuple(_type_convert(p) for p in params) + is_generic_or_protocol = cls in (Generic, Protocol) - def __typing_subst__(self, arg): - raise TypeError("Substitution of bare TypeVarTuple is not supported") + if is_generic_or_protocol: + # Generic and Protocol can only be subscripted with unique type variables. + if not params: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty" + ) + if not all(_is_typevar_like(p) for p in params): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be type variables " + f"or parameter specification variables.") + if len(set(params)) != len(params): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be unique") + else: + # Subscripting a regular Generic subclass. + for param in cls.__parameters__: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + params = prepare(cls, params) + _check_generic(cls, params, len(cls.__parameters__)) - def __typing_prepare_subst__(self, alias, args): - params = alias.__parameters__ - typevartuple_index = params.index(self) - for param in params[typevartuple_index + 1:]: + new_args = [] + for param, new_arg in zip(cls.__parameters__, params): if isinstance(param, TypeVarTuple): - raise TypeError(f"More than one TypeVarTuple parameter in {alias}") - - alen = len(args) - plen = len(params) - left = typevartuple_index - right = plen - typevartuple_index - 1 - var_tuple_index = None - fillarg = None - for k, arg in enumerate(args): - if not isinstance(arg, type): - subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) - if subargs and len(subargs) == 2 and subargs[-1] is ...: - if var_tuple_index is not None: - raise TypeError("More than one unpacked arbitrary-length tuple argument") - var_tuple_index = k - fillarg = subargs[0] - if var_tuple_index is not None: - left = min(left, var_tuple_index) - right = min(right, alen - var_tuple_index - 1) - elif left + right > alen: - raise TypeError(f"Too few arguments for {alias};" - f" actual {alen}, expected at least {plen-1}") - - return ( - *args[:left], - *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), - *([fillarg]*(plen - right - left - typevartuple_index - 1)), - *args[alen - right:], - ) - - -class ParamSpecArgs(_Final, _Immutable, _root=True): - """The args for a ParamSpec object. - - Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. - - ParamSpecArgs objects have a reference back to their ParamSpec: - - P.args.__origin__ is P - - This type is meant for runtime introspection and has no special meaning to - static type checkers. - """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.args" - - def __eq__(self, other): - if not isinstance(other, ParamSpecArgs): - return NotImplemented - return self.__origin__ == other.__origin__ - - -class ParamSpecKwargs(_Final, _Immutable, _root=True): - """The kwargs for a ParamSpec object. - - Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. - - ParamSpecKwargs objects have a reference back to their ParamSpec: - - P.kwargs.__origin__ is P - - This type is meant for runtime introspection and has no special meaning to - static type checkers. - """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.kwargs" - - def __eq__(self, other): - if not isinstance(other, ParamSpecKwargs): - return NotImplemented - return self.__origin__ == other.__origin__ - - -class ParamSpec(_Final, _Immutable, _BoundVarianceMixin, _PickleUsingNameMixin, - _root=True): - """Parameter specification variable. - - Usage:: - - P = ParamSpec('P') - - Parameter specification variables exist primarily for the benefit of static - type checkers. They are used to forward the parameter types of one - callable to another callable, a pattern commonly found in higher order - functions and decorators. They are only valid when used in ``Concatenate``, - or as the first argument to ``Callable``, or as parameters for user-defined - Generics. See class Generic for more information on generic types. An - example for annotating a decorator:: - - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: - '''A type-safe decorator to add logging to a function.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> T: - logging.info(f'{f.__name__} was called') - return f(*args, **kwargs) - return inner - - @add_logging - def add_two(x: float, y: float) -> float: - '''Add two numbers together.''' - return x + y - - Parameter specification variables can be introspected. e.g.: - - P.__name__ == 'P' - - Note that only parameter specification variables defined in global scope can - be pickled. - """ - - @property - def args(self): - return ParamSpecArgs(self) + new_args.extend(new_arg) + else: + new_args.append(new_arg) + params = tuple(new_args) - @property - def kwargs(self): - return ParamSpecKwargs(self) + return _GenericAlias(cls, params) - def __init__(self, name, *, bound=None, covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - def_mod = _caller() - if def_mod != 'typing': - self.__module__ = def_mod - def __typing_subst__(self, arg): - if isinstance(arg, (list, tuple)): - arg = tuple(_type_check(a, "Expected a type.") for a in arg) - elif not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") - return arg +def _generic_init_subclass(cls, *args, **kwargs): + super(Generic, cls).__init_subclass__(*args, **kwargs) + tvars = [] + if '__orig_bases__' in cls.__dict__: + error = Generic in cls.__orig_bases__ + else: + error = (Generic in cls.__bases__ and + cls.__name__ != 'Protocol' and + type(cls) != _TypedDictMeta) + if error: + raise TypeError("Cannot inherit from plain Generic") + if '__orig_bases__' in cls.__dict__: + tvars = _collect_parameters(cls.__orig_bases__) + # Look for Generic[T1, ..., Tn]. + # If found, tvars must be a subset of it. + # If not found, tvars is it. + # Also check for and reject plain Generic, + # and reject multiple Generic[...]. + gvars = None + for base in cls.__orig_bases__: + if (isinstance(base, _GenericAlias) and + base.__origin__ is Generic): + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...] multiple times.") + gvars = base.__parameters__ + if gvars is not None: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) + s_args = ', '.join(str(g) for g in gvars) + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in Generic[{s_args}]") + tvars = gvars + cls.__parameters__ = tuple(tvars) - def __typing_prepare_subst__(self, alias, args): - params = alias.__parameters__ - i = params.index(self) - if i >= len(args): - raise TypeError(f"Too few arguments for {alias}") - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if len(params) == 1 and not _is_param_expr(args[0]): - assert i == 0 - args = (args,) - # Convert lists to tuples to help other libraries cache the results. - elif isinstance(args[i], list): - args = (*args[:i], tuple(args[i]), *args[i+1:]) - return args def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -1302,8 +1167,7 @@ class _BaseGenericAlias(_Final, _root=True): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_paramspec_tvars'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1341,8 +1205,7 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # * Note that native container types, e.g. `tuple`, `list`, use # `types.GenericAlias` instead. # * Parameterized classes: - # T = TypeVar('T') - # class C(Generic[T]): pass + # class C[T]: pass # # C[int] is a _GenericAlias # * `Callable` aliases, generic `Callable` aliases, and # parameterized `Callable` aliases: @@ -1357,15 +1220,13 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # ClassVar[float] # TypeVar[bool] - def __init__(self, origin, args, *, inst=True, name=None, - _paramspec_tvars=False): + def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) self.__parameters__ = _collect_parameters(args) - self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -1508,8 +1369,7 @@ class _GenericAlias(_BaseGenericAlias, _root=True): return new_args def copy_with(self, args): - return self.__class__(self.__origin__, args, name=self._name, inst=self._inst, - _paramspec_tvars=self._paramspec_tvars) + return self.__class__(self.__origin__, args, name=self._name, inst=self._inst) def __repr__(self): if self._name: @@ -1599,6 +1459,22 @@ class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): def __ror__(self, left): return Union[left, self] + +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1619,8 +1495,7 @@ class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst, - _paramspec_tvars=True) + name=self._name, inst=self._inst) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -1748,10 +1623,26 @@ def Unpack(self, parameters): Foo[*tuple[int, str]] class Bar(Generic[*Ts]): ... + And from Python 3.12, it can be done using built-in syntax for generics:: + + Foo[*tuple[int, str]] + class Bar[*Ts]: ... + + The operator can also be used along with a `TypedDict` to annotate + `**kwargs` in a function signature:: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + Note that there is only some runtime checking of this operator. Not everything the runtime allows may be accepted by static type checkers. - For more information, see PEP 646. + For more information, see PEPs 646 and 692. """ item = _type_check(parameters, f'{self} accepts only single type.') return _UnpackGenericAlias(origin=self, args=(item,)) @@ -1761,7 +1652,7 @@ class _UnpackGenericAlias(_GenericAlias, _root=True): def __repr__(self): # `Unpack` only takes one argument, so __args__ should contain only # a single item. - return '*' + repr(self.__args__[0]) + return f'typing.Unpack[{_type_repr(self.__args__[0])}]' def __getitem__(self, args): if self.__typing_is_unpacked_typevartuple__: @@ -1785,127 +1676,24 @@ class _UnpackGenericAlias(_GenericAlias, _root=True): return isinstance(self.__args__[0], TypeVarTuple) -class Generic: - """Abstract base class for generic types. - - A generic type is typically declared by inheriting from - this class parameterized with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: - try: - return mapping[key] - except KeyError: - return default - """ - __slots__ = () - _is_protocol = False - - @_tp_cache - def __class_getitem__(cls, params): - """Parameterizes a generic class. - - At least, parameterizing a generic class is the *main* thing this method - does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. - - However, note that this method is also called when defining generic - classes in the first place with `class Foo(Generic[T]): ...`. - """ - if not isinstance(params, tuple): - params = (params,) - - params = tuple(_type_convert(p) for p in params) - if cls in (Generic, Protocol): - # Generic and Protocol can only be subscripted with unique type variables. - if not params: - raise TypeError( - f"Parameter list to {cls.__qualname__}[...] cannot be empty" - ) - if not all(_is_typevar_like(p) for p in params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be type variables " - f"or parameter specification variables.") - if len(set(params)) != len(params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be unique") - else: - # Subscripting a regular Generic subclass. - for param in cls.__parameters__: - prepare = getattr(param, '__typing_prepare_subst__', None) - if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) - - new_args = [] - for param, new_arg in zip(cls.__parameters__, params): - if isinstance(param, TypeVarTuple): - new_args.extend(new_arg) - else: - new_args.append(new_arg) - params = tuple(new_args) - - return _GenericAlias(cls, params, - _paramspec_tvars=True) - - def __init_subclass__(cls, *args, **kwargs): - super().__init_subclass__(*args, **kwargs) - tvars = [] - if '__orig_bases__' in cls.__dict__: - error = Generic in cls.__orig_bases__ - else: - error = (Generic in cls.__bases__ and - cls.__name__ != 'Protocol' and - type(cls) != _TypedDictMeta) - if error: - raise TypeError("Cannot inherit from plain Generic") - if '__orig_bases__' in cls.__dict__: - tvars = _collect_parameters(cls.__orig_bases__) - # Look for Generic[T1, ..., Tn]. - # If found, tvars must be a subset of it. - # If not found, tvars is it. - # Also check for and reject plain Generic, - # and reject multiple Generic[...]. - gvars = None - for base in cls.__orig_bases__: - if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): - if gvars is not None: - raise TypeError( - "Cannot inherit from Generic[...] multiple times.") - gvars = base.__parameters__ - if gvars is not None: - tvarset = set(tvars) - gvarset = set(gvars) - if not tvarset <= gvarset: - s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) - s_args = ', '.join(str(g) for g in gvars) - raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") - tvars = gvars - cls.__parameters__ = tuple(tvars) - - class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -_TYPING_INTERNALS = ['__parameters__', '__orig_bases__', '__orig_class__', - '_is_protocol', '_is_runtime_protocol', '__final__'] +_TYPING_INTERNALS = frozenset({ + '__parameters__', '__orig_bases__', '__orig_class__', + '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', + '__non_callable_proto_members__', '__type_params__', +}) -_SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__', - '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__', '__class_getitem__'] +_SPECIAL_NAMES = frozenset({ + '__abstractmethods__', '__annotations__', '__dict__', '__doc__', + '__init__', '__module__', '__new__', '__slots__', + '__subclasshook__', '__weakref__', '__class_getitem__' +}) # These special attributes will be not collected as protocol members. -EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker'] +EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS | _SPECIAL_NAMES | {'_MutableMapping__marker'} def _get_protocol_attrs(cls): @@ -1916,20 +1704,15 @@ def _get_protocol_attrs(cls): """ attrs = set() for base in cls.__mro__[:-1]: # without object - if base.__name__ in ('Protocol', 'Generic'): + if base.__name__ in {'Protocol', 'Generic'}: continue annotations = getattr(base, '__annotations__', {}) - for attr in list(base.__dict__.keys()) + list(annotations.keys()): + for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: attrs.add(attr) return attrs -def _is_callable_members_only(cls): - # PEP 544 prohibits using issubclass() with protocols that have non-method members. - return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls)) - - def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) @@ -1961,12 +1744,16 @@ def _no_init_or_replace_init(self, *args, **kwargs): def _caller(depth=1, default='__main__'): try: + return sys._getframemodulename(depth + 1) or default + except AttributeError: # For platforms without _getframemodulename() + pass + try: return sys._getframe(depth + 1).f_globals.get('__name__', default) except (AttributeError, ValueError): # For platforms without _getframe() - return None - + pass + return None -def _allow_reckless_class_checks(depth=3): +def _allow_reckless_class_checks(depth=2): """Allow instance and class checks for special stdlib modules. The abc and functools modules indiscriminately call isinstance() and @@ -1978,38 +1765,143 @@ def _allow_reckless_class_checks(depth=3): _PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', + 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', 'Buffer', ], 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], } +@functools.cache +def _lazy_load_getattr_static(): + # Import getattr_static lazily so as not to slow down the import of typing.py + # Cache the result so we don't slow down _ProtocolMeta.__instancecheck__ unnecessarily + from inspect import getattr_static + return getattr_static + + +_cleanups.append(_lazy_load_getattr_static.cache_clear) + +def _pickle_psargs(psargs): + return ParamSpecArgs, (psargs.__origin__,) + +copyreg.pickle(ParamSpecArgs, _pickle_psargs) + +def _pickle_pskwargs(pskwargs): + return ParamSpecKwargs, (pskwargs.__origin__,) + +copyreg.pickle(ParamSpecKwargs, _pickle_pskwargs) + +del _pickle_psargs, _pickle_pskwargs + + class _ProtocolMeta(ABCMeta): - # This metaclass is really unfortunate and exists only because of - # the lack of __instancehook__. + # This metaclass is somewhat unfortunate, + # but is necessary for several reasons... + def __new__(mcls, name, bases, namespace, /, **kwargs): + if name == "Protocol" and bases == (Generic,): + pass + elif Protocol in bases: + for base in bases: + if not ( + base in {object, Generic} + or base.__name__ in _PROTO_ALLOWLIST.get(base.__module__, []) + or ( + issubclass(base, Generic) + and getattr(base, "_is_protocol", False) + ) + ): + raise TypeError( + f"Protocols can only inherit from other protocols, " + f"got {base!r}" + ) + return super().__new__(mcls, name, bases, namespace, **kwargs) + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if getattr(cls, "_is_protocol", False): + cls.__protocol_attrs__ = _get_protocol_attrs(cls) + + def __subclasscheck__(cls, other): + if cls is Protocol: + return type.__subclasscheck__(cls, other) + if ( + getattr(cls, '_is_protocol', False) + and not _allow_reckless_class_checks() + ): + if not isinstance(other, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') + if not getattr(cls, '_is_runtime_protocol', False): + raise TypeError( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) + if ( + # this attribute is set by @runtime_checkable: + cls.__non_callable_proto_members__ + and cls.__dict__.get("__subclasshook__") is _proto_hook + ): + raise TypeError( + "Protocols with non-method members don't support issubclass()" + ) + return super().__subclasscheck__(other) + def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. + if cls is Protocol: + return type.__instancecheck__(cls, instance) + if not getattr(cls, "_is_protocol", False): + # i.e., it's a concrete subclass of a protocol + return super().__instancecheck__(instance) + if ( - getattr(cls, '_is_protocol', False) and not getattr(cls, '_is_runtime_protocol', False) and - not _allow_reckless_class_checks(depth=2) + not _allow_reckless_class_checks() ): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if ((not getattr(cls, '_is_protocol', False) or - _is_callable_members_only(cls)) and - issubclass(instance.__class__, cls)): + if super().__instancecheck__(instance): return True - if cls._is_protocol: - if all(hasattr(instance, attr) and - # All *methods* can be blocked by setting them to None. - (not callable(getattr(cls, attr, None)) or - getattr(instance, attr) is not None) - for attr in _get_protocol_attrs(cls)): - return True - return super().__instancecheck__(instance) + + getattr_static = _lazy_load_getattr_static() + for attr in cls.__protocol_attrs__: + try: + val = getattr_static(instance, attr) + except AttributeError: + break + # this attribute is set by @runtime_checkable: + if val is None and attr not in cls.__non_callable_proto_members__: + break + else: + return True + + return False + + +@classmethod +def _proto_hook(cls, other): + if not cls.__dict__.get('_is_protocol', False): + return NotImplemented + + for attr in cls.__protocol_attrs__: + for base in other.__mro__: + # Check if the members appears in the class dictionary... + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + + # ...or in annotations, if it is a sub-protocol. + annotations = getattr(base, '__annotations__', {}) + if (isinstance(annotations, collections.abc.Mapping) and + attr in annotations and + issubclass(other, Generic) and getattr(other, '_is_protocol', False)): + break + else: + return NotImplemented + return True class Protocol(Generic, metaclass=_ProtocolMeta): @@ -2040,7 +1932,7 @@ class Protocol(Generic, metaclass=_ProtocolMeta): only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as:: - class GenProto(Protocol[T]): + class GenProto[T](Protocol): def meth(self) -> T: ... """ @@ -2057,60 +1949,11 @@ class Protocol(Generic, metaclass=_ProtocolMeta): cls._is_protocol = any(b is Protocol for b in cls.__bases__) # Set (or override) the protocol subclass hook. - def _proto_hook(other): - if not cls.__dict__.get('_is_protocol', False): - return NotImplemented - - # First, perform various sanity checks. - if not getattr(cls, '_is_runtime_protocol', False): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Instance and class checks can only be used with" - " @runtime_checkable protocols") - if not _is_callable_members_only(cls): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Protocols with non-method members" - " don't support issubclass()") - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') - - # Second, perform the actual structural compatibility check. - for attr in _get_protocol_attrs(cls): - for base in other.__mro__: - # Check if the members appears in the class dictionary... - if attr in base.__dict__: - if base.__dict__[attr] is None: - return NotImplemented - break - - # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if (isinstance(annotations, collections.abc.Mapping) and - attr in annotations and - issubclass(other, Generic) and other._is_protocol): - break - else: - return NotImplemented - return True - if '__subclasshook__' not in cls.__dict__: cls.__subclasshook__ = _proto_hook - # We have nothing more to do for non-protocols... - if not cls._is_protocol: - return - - # ... otherwise check consistency of bases, and prohibit instantiation. - for base in cls.__bases__: - if not (base in (object, Generic) or - base.__module__ in _PROTO_ALLOWLIST and - base.__name__ in _PROTO_ALLOWLIST[base.__module__] or - issubclass(base, Generic) and base._is_protocol): - raise TypeError('Protocols can only inherit from other' - ' protocols, got %r' % base) - if cls.__init__ is Protocol.__init__: + # Prohibit instantiation for protocol classes + if cls._is_protocol and cls.__init__ is Protocol.__init__: cls.__init__ = _no_init_or_replace_init @@ -2129,7 +1972,7 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): if isinstance(origin, _AnnotatedAlias): metadata = origin.__metadata__ + metadata origin = origin.__origin__ - super().__init__(origin, origin) + super().__init__(origin, origin, name='Annotated') self.__metadata__ = metadata def copy_with(self, params): @@ -2162,6 +2005,9 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): return 'Annotated' return super().__getattr__(attr) + def __mro_entries__(self, bases): + return (self.__origin__,) + class Annotated: """Add context-specific metadata to a type. @@ -2191,15 +2037,17 @@ class Annotated: - Annotated can be used as a generic type alias:: - Optimized: TypeAlias = Annotated[T, runtime.Optimize()] - assert Optimized[int] == Annotated[int, runtime.Optimize()] + type Optimized[T] = Annotated[T, runtime.Optimize()] + # type checker will treat Optimized[int] + # as equivalent to Annotated[int, runtime.Optimize()] - OptimizedList: TypeAlias = Annotated[list[T], runtime.Optimize()] - assert OptimizedList[int] == Annotated[list[int], runtime.Optimize()] + type OptimizedList[T] = Annotated[list[T], runtime.Optimize()] + # type checker will treat OptimizedList[int] + # as equivalent to Annotated[list[int], runtime.Optimize()] - Annotated cannot be used with an unpacked TypeVarTuple:: - Variadic: TypeAlias = Annotated[*Ts, Ann1] # NOT valid + type Variadic[*Ts] = Annotated[*Ts, Ann1] # NOT valid This would be equivalent to:: @@ -2258,10 +2106,26 @@ def runtime_checkable(cls): Warning: this will check only the presence of the required methods, not their type signatures! """ - if not issubclass(cls, Generic) or not cls._is_protocol: + if not issubclass(cls, Generic) or not getattr(cls, '_is_protocol', False): raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls @@ -2681,8 +2545,9 @@ def final(f): return f -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) +# Some unconstrained type variables. These were initially used by the container types. +# They were never meant for export and are now unused, but we keep them around to +# avoid breaking compatibility with users who import them. T = TypeVar('T') # Any type. KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. @@ -2693,6 +2558,7 @@ T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) + # A useful type variable with constraints. This represents string types. # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) @@ -2734,7 +2600,9 @@ Mapping = _alias(collections.abc.Mapping, 2) MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _alias(collections.abc.ByteString, 0) # Not generic +ByteString = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. +) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -2780,8 +2648,7 @@ Type.__doc__ = \ And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: + def new_user[U](user_class: Type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2848,24 +2715,24 @@ class SupportsIndex(Protocol): @runtime_checkable -class SupportsAbs(Protocol[T_co]): +class SupportsAbs[T](Protocol): """An ABC with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @abstractmethod - def __abs__(self) -> T_co: + def __abs__(self) -> T: pass @runtime_checkable -class SupportsRound(Protocol[T_co]): +class SupportsRound[T](Protocol): """An ABC with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: + def __round__(self, ndigits: int = 0) -> T: pass @@ -2910,7 +2777,7 @@ class NamedTupleMeta(type): module=ns['__module__']) nm_tpl.__bases__ = bases if Generic in bases: - class_getitem = Generic.__class_getitem__.__func__ + class_getitem = _generic_class_getitem nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes for key in ns: @@ -2948,7 +2815,9 @@ def NamedTuple(typename, fields=None, /, **kwargs): elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") - return _make_nmtuple(typename, fields, module=_caller()) + nt = _make_nmtuple(typename, fields, module=_caller()) + nt.__orig_bases__ = (NamedTuple,) + return nt _NamedTuple = type.__new__(NamedTupleMeta, 'NamedTuple', (), {}) @@ -2980,6 +2849,9 @@ class _TypedDictMeta(type): tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) + if not hasattr(tp_dict, '__orig_bases__'): + tp_dict.__orig_bases__ = bases + annotations = {} own_annotations = ns.get('__annotations__', {}) msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" @@ -3111,7 +2983,9 @@ def TypedDict(typename, fields=None, /, *, total=True, **kwargs): # Setting correct module is necessary to make typed dict classes pickleable. ns['__module__'] = module - return _TypedDictMeta(typename, (), ns, total=total) + td = _TypedDictMeta(typename, (), ns, total=total) + td.__orig_bases__ = (TypedDict,) + return td _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) @@ -3418,7 +3292,7 @@ re.__name__ = __name__ + '.re' sys.modules[re.__name__] = re -def reveal_type(obj: T, /) -> T: +def reveal_type[T](obj: T, /) -> T: """Ask a static type checker to reveal the inferred type of an expression. When a static type checker encounters a call to ``reveal_type()``, @@ -3437,24 +3311,28 @@ def reveal_type(obj: T, /) -> T: return obj +class _IdentityCallable(Protocol): + def __call__[T](self, arg: T, /) -> T: + ... + + def dataclass_transform( *, eq_default: bool = True, order_default: bool = False, kw_only_default: bool = False, + frozen_default: bool = False, field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, -) -> Callable[[T], T]: +) -> _IdentityCallable: """Decorator to mark an object as providing dataclass-like behaviour. The decorator can be applied to a function, class, or metaclass. Example usage with a decorator function:: - T = TypeVar("T") - @dataclass_transform() - def create_model(cls: type[T]) -> type[T]: + def create_model[T](cls: type[T]) -> type[T]: ... return cls @@ -3496,6 +3374,8 @@ def dataclass_transform( assumed to be True or False if it is omitted by the caller. - ``kw_only_default`` indicates whether the ``kw_only`` parameter is assumed to be True or False if it is omitted by the caller. + - ``frozen_default`` indicates whether the ``frozen`` parameter is + assumed to be True or False if it is omitted by the caller. - ``field_specifiers`` specifies a static list of supported classes or functions that describe fields, similar to ``dataclasses.field()``. - Arbitrary other keyword arguments are accepted in order to allow for @@ -3512,8 +3392,47 @@ def dataclass_transform( "eq_default": eq_default, "order_default": order_default, "kw_only_default": kw_only_default, + "frozen_default": frozen_default, "field_specifiers": field_specifiers, "kwargs": kwargs, } return cls_or_fn return decorator + + +type _Func = Callable[..., Any] + + +def override[F: _Func](method: F, /) -> F: + """Indicate that a method is intended to override a method in a base class. + + Usage:: + + class Base: + def method(self) -> None: + pass + + class Child(Base): + @override + def method(self) -> None: + super().method() + + When this decorator is applied to a method, the type checker will + validate that it overrides a method or attribute with the same name on a + base class. This helps prevent bugs that may occur when a base class is + changed without an equivalent change to a child class. + + There is no runtime checking of this property. The decorator attempts to + set the ``__override__`` attribute to ``True`` on the decorated object to + allow runtime introspection. + + See PEP 698 for details. + """ + try: + method.__override__ = True + except (AttributeError, TypeError): + # Skip the attribute silently if it is not writable. + # AttributeError happens if the object has __slots__ or a + # read-only property, TypeError if it's a builtin class. + pass + return method |