From d4be68e361f4258cf0848fc70018dfe37a2acc24 Mon Sep 17 00:00:00 2001 From: shadchin Date: Mon, 18 Apr 2022 12:39:32 +0300 Subject: IGNIETFERRO-1816 Update Python 3 from 3.9.12 to 3.10.4 ref:9f96be6d02ee8044fdd6f124b799b270c20ce641 --- contrib/tools/python3/src/Lib/__future__.py | 4 +- contrib/tools/python3/src/Lib/_aix_support.py | 26 +- contrib/tools/python3/src/Lib/_bootlocale.py | 46 - contrib/tools/python3/src/Lib/_collections_abc.py | 150 +- contrib/tools/python3/src/Lib/_compression.py | 12 +- contrib/tools/python3/src/Lib/_markupbase.py | 35 +- contrib/tools/python3/src/Lib/_osx_support.py | 13 +- contrib/tools/python3/src/Lib/_pydecimal.py | 2 +- contrib/tools/python3/src/Lib/_pyio.py | 89 +- contrib/tools/python3/src/Lib/_sitebuiltins.py | 2 +- contrib/tools/python3/src/Lib/abc.py | 38 + contrib/tools/python3/src/Lib/argparse.py | 5 +- contrib/tools/python3/src/Lib/ast.py | 105 +- contrib/tools/python3/src/Lib/asynchat.py | 8 + contrib/tools/python3/src/Lib/asyncio/__init__.py | 4 - .../tools/python3/src/Lib/asyncio/base_events.py | 21 +- contrib/tools/python3/src/Lib/asyncio/events.py | 25 +- contrib/tools/python3/src/Lib/asyncio/futures.py | 6 +- contrib/tools/python3/src/Lib/asyncio/locks.py | 72 +- contrib/tools/python3/src/Lib/asyncio/mixins.py | 31 + .../python3/src/Lib/asyncio/proactor_events.py | 40 +- contrib/tools/python3/src/Lib/asyncio/queues.py | 21 +- contrib/tools/python3/src/Lib/asyncio/runners.py | 3 +- contrib/tools/python3/src/Lib/asyncio/streams.py | 51 +- .../tools/python3/src/Lib/asyncio/subprocess.py | 30 +- contrib/tools/python3/src/Lib/asyncio/tasks.py | 130 +- .../tools/python3/src/Lib/asyncio/unix_events.py | 34 +- contrib/tools/python3/src/Lib/asyncore.py | 23 +- contrib/tools/python3/src/Lib/base64.py | 86 +- contrib/tools/python3/src/Lib/bdb.py | 29 +- contrib/tools/python3/src/Lib/bisect.py | 70 +- contrib/tools/python3/src/Lib/bz2.py | 116 +- contrib/tools/python3/src/Lib/calendar.py | 25 +- contrib/tools/python3/src/Lib/cgi.py | 7 +- contrib/tools/python3/src/Lib/cgitb.py | 4 +- contrib/tools/python3/src/Lib/codecs.py | 2 +- contrib/tools/python3/src/Lib/codeop.py | 73 +- .../tools/python3/src/Lib/collections/__init__.py | 100 +- contrib/tools/python3/src/Lib/colorsys.py | 15 +- contrib/tools/python3/src/Lib/compileall.py | 13 +- .../python3/src/Lib/concurrent/futures/process.py | 8 + contrib/tools/python3/src/Lib/configparser.py | 11 +- contrib/tools/python3/src/Lib/contextlib.py | 64 +- contrib/tools/python3/src/Lib/copy.py | 1 + contrib/tools/python3/src/Lib/copyreg.py | 6 + contrib/tools/python3/src/Lib/csv.py | 12 +- contrib/tools/python3/src/Lib/dataclasses.py | 264 +++- contrib/tools/python3/src/Lib/datetime.py | 51 +- contrib/tools/python3/src/Lib/dis.py | 45 +- .../tools/python3/src/Lib/distutils/__init__.py | 7 + .../python3/src/Lib/distutils/command/__init__.py | 1 - .../python3/src/Lib/distutils/command/bdist.py | 4 +- .../python3/src/Lib/distutils/command/bdist_msi.py | 2 - .../src/Lib/distutils/command/bdist_wininst.py | 377 ----- .../python3/src/Lib/distutils/command/install.py | 98 +- .../tools/python3/src/Lib/distutils/extension.py | 3 +- .../tools/python3/src/Lib/distutils/sysconfig.py | 636 +++----- contrib/tools/python3/src/Lib/distutils/util.py | 3 + contrib/tools/python3/src/Lib/doctest.py | 42 +- contrib/tools/python3/src/Lib/email/_parseaddr.py | 2 +- contrib/tools/python3/src/Lib/email/base64mime.py | 2 +- contrib/tools/python3/src/Lib/email/errors.py | 3 + .../tools/python3/src/Lib/email/headerregistry.py | 13 +- contrib/tools/python3/src/Lib/email/utils.py | 5 +- .../tools/python3/src/Lib/encodings/__init__.py | 3 +- .../tools/python3/src/Lib/ensurepip/__init__.py | 113 +- contrib/tools/python3/src/Lib/enum.py | 25 +- contrib/tools/python3/src/Lib/filecmp.py | 9 +- contrib/tools/python3/src/Lib/fileinput.py | 67 +- contrib/tools/python3/src/Lib/formatter.py | 452 ------ contrib/tools/python3/src/Lib/fractions.py | 125 +- contrib/tools/python3/src/Lib/functools.py | 30 +- contrib/tools/python3/src/Lib/glob.py | 151 +- contrib/tools/python3/src/Lib/gzip.py | 5 +- contrib/tools/python3/src/Lib/hashlib.py | 7 + contrib/tools/python3/src/Lib/hmac.py | 86 +- contrib/tools/python3/src/Lib/http/__init__.py | 1 + contrib/tools/python3/src/Lib/http/client.py | 84 +- contrib/tools/python3/src/Lib/http/cookiejar.py | 40 +- contrib/tools/python3/src/Lib/http/server.py | 5 +- contrib/tools/python3/src/Lib/imaplib.py | 13 +- contrib/tools/python3/src/Lib/imp.py | 3 +- .../tools/python3/src/Lib/importlib/__init__.py | 14 +- contrib/tools/python3/src/Lib/importlib/_abc.py | 54 + .../tools/python3/src/Lib/importlib/_adapters.py | 83 + .../tools/python3/src/Lib/importlib/_bootstrap.py | 80 +- .../src/Lib/importlib/_bootstrap_external.py | 184 +-- contrib/tools/python3/src/Lib/importlib/_common.py | 85 +- contrib/tools/python3/src/Lib/importlib/abc.py | 136 +- .../tools/python3/src/Lib/importlib/machinery.py | 2 - .../tools/python3/src/Lib/importlib/metadata.py | 604 -------- .../python3/src/Lib/importlib/metadata/__init__.py | 1045 +++++++++++++ .../src/Lib/importlib/metadata/_adapters.py | 68 + .../src/Lib/importlib/metadata/_collections.py | 30 + .../src/Lib/importlib/metadata/_functools.py | 85 + .../src/Lib/importlib/metadata/_itertools.py | 19 + .../python3/src/Lib/importlib/metadata/_meta.py | 47 + .../python3/src/Lib/importlib/metadata/_text.py | 99 ++ contrib/tools/python3/src/Lib/importlib/readers.py | 123 ++ .../tools/python3/src/Lib/importlib/resources.py | 220 ++- contrib/tools/python3/src/Lib/importlib/util.py | 14 +- contrib/tools/python3/src/Lib/inspect.py | 161 +- contrib/tools/python3/src/Lib/io.py | 19 +- contrib/tools/python3/src/Lib/ipaddress.py | 11 +- contrib/tools/python3/src/Lib/keyword.py | 7 +- .../tools/python3/src/Lib/lib2to3/pgen2/pgen.py | 2 +- contrib/tools/python3/src/Lib/linecache.py | 11 +- contrib/tools/python3/src/Lib/locale.py | 98 +- contrib/tools/python3/src/Lib/logging/__init__.py | 75 +- contrib/tools/python3/src/Lib/logging/config.py | 5 +- contrib/tools/python3/src/Lib/logging/handlers.py | 11 +- contrib/tools/python3/src/Lib/lzma.py | 1 + contrib/tools/python3/src/Lib/mimetypes.py | 39 +- .../python3/src/Lib/multiprocessing/managers.py | 20 +- .../src/Lib/multiprocessing/resource_tracker.py | 10 +- .../tools/python3/src/Lib/multiprocessing/util.py | 2 +- contrib/tools/python3/src/Lib/netrc.py | 8 +- contrib/tools/python3/src/Lib/ntpath.py | 23 +- contrib/tools/python3/src/Lib/opcode.py | 25 +- contrib/tools/python3/src/Lib/os.py | 100 +- contrib/tools/python3/src/Lib/pathlib.py | 354 ++--- contrib/tools/python3/src/Lib/pickle.py | 1 + contrib/tools/python3/src/Lib/pipes.py | 4 +- contrib/tools/python3/src/Lib/pkgutil.py | 21 +- contrib/tools/python3/src/Lib/platform.py | 72 +- contrib/tools/python3/src/Lib/posixpath.py | 29 +- contrib/tools/python3/src/Lib/pprint.py | 224 +-- contrib/tools/python3/src/Lib/pty.py | 47 +- contrib/tools/python3/src/Lib/py_compile.py | 71 +- contrib/tools/python3/src/Lib/pyclbr.py | 320 ++-- contrib/tools/python3/src/Lib/pydoc.py | 20 +- contrib/tools/python3/src/Lib/pydoc_data/topics.py | 1625 ++++++++++++++++---- contrib/tools/python3/src/Lib/random.py | 85 +- contrib/tools/python3/src/Lib/re.py | 1 - contrib/tools/python3/src/Lib/rlcompleter.py | 9 +- contrib/tools/python3/src/Lib/runpy.py | 3 +- contrib/tools/python3/src/Lib/sched.py | 20 +- contrib/tools/python3/src/Lib/shelve.py | 4 +- contrib/tools/python3/src/Lib/shutil.py | 26 +- contrib/tools/python3/src/Lib/site.py | 53 +- contrib/tools/python3/src/Lib/smtpd.py | 18 +- contrib/tools/python3/src/Lib/smtplib.py | 8 +- contrib/tools/python3/src/Lib/socket.py | 7 +- contrib/tools/python3/src/Lib/sqlite3/__init__.py | 14 + contrib/tools/python3/src/Lib/sqlite3/dbapi2.py | 14 + contrib/tools/python3/src/Lib/ssl.py | 81 +- contrib/tools/python3/src/Lib/statistics.py | 267 +++- contrib/tools/python3/src/Lib/subprocess.py | 53 +- contrib/tools/python3/src/Lib/symbol.py | 122 -- contrib/tools/python3/src/Lib/symtable.py | 74 +- contrib/tools/python3/src/Lib/sysconfig.py | 274 ++-- contrib/tools/python3/src/Lib/tarfile.py | 74 +- contrib/tools/python3/src/Lib/tempfile.py | 60 +- contrib/tools/python3/src/Lib/textwrap.py | 12 +- contrib/tools/python3/src/Lib/threading.py | 112 +- contrib/tools/python3/src/Lib/timeit.py | 1 + contrib/tools/python3/src/Lib/token.py | 11 +- contrib/tools/python3/src/Lib/tokenize.py | 2 + contrib/tools/python3/src/Lib/trace.py | 2 +- contrib/tools/python3/src/Lib/traceback.py | 223 +-- contrib/tools/python3/src/Lib/turtle.py | 18 +- contrib/tools/python3/src/Lib/types.py | 12 +- contrib/tools/python3/src/Lib/typing.py | 564 ++++++- contrib/tools/python3/src/Lib/unittest/_log.py | 29 +- .../tools/python3/src/Lib/unittest/async_case.py | 2 +- contrib/tools/python3/src/Lib/unittest/case.py | 15 +- contrib/tools/python3/src/Lib/unittest/mock.py | 84 +- contrib/tools/python3/src/Lib/unittest/result.py | 3 +- contrib/tools/python3/src/Lib/urllib/parse.py | 3 +- contrib/tools/python3/src/Lib/urllib/request.py | 2 + .../src/Lib/venv/scripts/common/Activate.ps1 | 6 + .../python3/src/Lib/venv/scripts/common/activate | 3 + .../python3/src/Lib/venv/scripts/nt/activate.bat | 1 + .../python3/src/Lib/venv/scripts/nt/deactivate.bat | 1 + .../src/Lib/venv/scripts/posix/activate.csh | 3 +- .../src/Lib/venv/scripts/posix/activate.fish | 2 + contrib/tools/python3/src/Lib/webbrowser.py | 6 +- .../tools/python3/src/Lib/xml/etree/ElementPath.py | 31 +- .../tools/python3/src/Lib/xml/etree/ElementTree.py | 2 - contrib/tools/python3/src/Lib/xml/sax/handler.py | 45 + contrib/tools/python3/src/Lib/zipfile.py | 69 +- contrib/tools/python3/src/Lib/zipimport.py | 170 +- contrib/tools/python3/src/Lib/zoneinfo/_common.py | 3 +- 183 files changed, 8283 insertions(+), 5274 deletions(-) delete mode 100644 contrib/tools/python3/src/Lib/_bootlocale.py create mode 100644 contrib/tools/python3/src/Lib/asyncio/mixins.py delete mode 100644 contrib/tools/python3/src/Lib/distutils/command/bdist_wininst.py delete mode 100644 contrib/tools/python3/src/Lib/formatter.py create mode 100644 contrib/tools/python3/src/Lib/importlib/_abc.py create mode 100644 contrib/tools/python3/src/Lib/importlib/_adapters.py delete mode 100644 contrib/tools/python3/src/Lib/importlib/metadata.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/__init__.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_adapters.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_collections.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_functools.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_itertools.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_meta.py create mode 100644 contrib/tools/python3/src/Lib/importlib/metadata/_text.py create mode 100644 contrib/tools/python3/src/Lib/importlib/readers.py delete mode 100644 contrib/tools/python3/src/Lib/symbol.py (limited to 'contrib/tools/python3/src/Lib') diff --git a/contrib/tools/python3/src/Lib/__future__.py b/contrib/tools/python3/src/Lib/__future__.py index 0e7b5552343..97dc90c6e46 100644 --- a/contrib/tools/python3/src/Lib/__future__.py +++ b/contrib/tools/python3/src/Lib/__future__.py @@ -42,7 +42,7 @@ CompilerFlag is the (bitfield) flag that should be passed in the fourth argument to the builtin function compile() to enable the feature in dynamically compiled code. This flag is stored in the .compiler_flag attribute on _Future instances. These values must match the appropriate -#defines of CO_xxx flags in Include/compile.h. +#defines of CO_xxx flags in Include/cpython/compile.h. No feature line is ever to be deleted from this file. """ @@ -143,5 +143,5 @@ generator_stop = _Feature((3, 5, 0, "beta", 1), CO_FUTURE_GENERATOR_STOP) annotations = _Feature((3, 7, 0, "beta", 1), - (3, 10, 0, "alpha", 0), + (3, 11, 0, "alpha", 0), CO_FUTURE_ANNOTATIONS) diff --git a/contrib/tools/python3/src/Lib/_aix_support.py b/contrib/tools/python3/src/Lib/_aix_support.py index d27a1e8735d..45504934063 100644 --- a/contrib/tools/python3/src/Lib/_aix_support.py +++ b/contrib/tools/python3/src/Lib/_aix_support.py @@ -15,9 +15,8 @@ def _aix_tag(vrtl, bd): # type: (List[int], int) -> str # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default) _sz = 32 if sys.maxsize == (2**31-1) else 64 - _bd = bd if bd != 0 else 9988 # vrtl[version, release, technology_level] - return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz) + return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], bd, _sz) # extract version, release and technology level from a VRMF string @@ -27,20 +26,19 @@ def _aix_vrtl(vrmf): return [int(v[-1]), int(r), int(tl)] -def _aix_bos_rte(): +def _aix_bosmp64(): # type: () -> Tuple[str, int] """ Return a Tuple[str, int] e.g., ['7.1.4.34', 1806] - The fileset bos.rte represents the current AIX run-time level. It's VRMF and - builddate reflect the current ABI levels of the runtime environment. - If no builddate is found give a value that will satisfy pep425 related queries + The fileset bos.mp64 is the AIX kernel. It's VRMF and builddate + reflect the current ABI levels of the runtime environment. """ - # All AIX systems to have lslpp installed in this location - out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"]) + # We expect all AIX systems to have lslpp installed in this location + out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.mp64"]) out = out.decode("utf-8") out = out.strip().split(":") # type: ignore - _bd = int(out[-1]) if out[-1] != '' else 9988 - return (str(out[2]), _bd) + # Use str() and int() to help mypy see types + return (str(out[2]), int(out[-1])) def aix_platform(): @@ -49,11 +47,11 @@ def aix_platform(): AIX filesets are identified by four decimal values: V.R.M.F. V (version) and R (release) can be retreived using ``uname`` Since 2007, starting with AIX 5.3 TL7, the M value has been - included with the fileset bos.rte and represents the Technology + included with the fileset bos.mp64 and represents the Technology Level (TL) of AIX. The F (Fix) value also increases, but is not relevant for comparing releases and binary compatibility. For binary compatibility the so-called builddate is needed. - Again, the builddate of an AIX release is associated with bos.rte. + Again, the builddate of an AIX release is associated with bos.mp64. AIX ABI compatibility is described as guaranteed at: https://www.ibm.com/\ support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html @@ -62,7 +60,7 @@ def aix_platform(): e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit """ - vrmf, bd = _aix_bos_rte() + vrmf, bd = _aix_bosmp64() return _aix_tag(_aix_vrtl(vrmf), bd) @@ -81,7 +79,7 @@ def aix_buildtag(): Return the platform_tag of the system Python was built on. """ # AIX_BUILDDATE is defined by configure with: - # lslpp -Lcq bos.rte | awk -F: '{ print $NF }' + # lslpp -Lcq bos.mp64 | awk -F: '{ print $NF }' build_date = sysconfig.get_config_var("AIX_BUILDDATE") try: build_date = int(build_date) diff --git a/contrib/tools/python3/src/Lib/_bootlocale.py b/contrib/tools/python3/src/Lib/_bootlocale.py deleted file mode 100644 index 3273a3b4225..00000000000 --- a/contrib/tools/python3/src/Lib/_bootlocale.py +++ /dev/null @@ -1,46 +0,0 @@ -"""A minimal subset of the locale module used at interpreter startup -(imported by the _io module), in order to reduce startup time. - -Don't import directly from third-party code; use the `locale` module instead! -""" - -import sys -import _locale - -if sys.platform.startswith("win"): - def getpreferredencoding(do_setlocale=True): - if sys.flags.utf8_mode: - return 'UTF-8' - return _locale._getdefaultlocale()[1] -else: - try: - _locale.CODESET - except AttributeError: - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - def getpreferredencoding(do_setlocale=True): - return 'UTF-8' - else: - def getpreferredencoding(do_setlocale=True): - if sys.flags.utf8_mode: - return 'UTF-8' - # This path for legacy systems needs the more complex - # getdefaultlocale() function, import the full locale module. - import locale - return locale.getpreferredencoding(do_setlocale) - else: - def getpreferredencoding(do_setlocale=True): - assert not do_setlocale - if sys.flags.utf8_mode: - return 'UTF-8' - result = _locale.nl_langinfo(_locale.CODESET) - if not result and sys.platform == 'darwin': - # nl_langinfo can return an empty string - # when the setting has an invalid value. - # Default to UTF-8 in that case because - # UTF-8 is the default charset on OSX and - # returning nothing will crash the - # interpreter. - result = 'UTF-8' - return result diff --git a/contrib/tools/python3/src/Lib/_collections_abc.py b/contrib/tools/python3/src/Lib/_collections_abc.py index 023ac7cf036..40417dc1d31 100644 --- a/contrib/tools/python3/src/Lib/_collections_abc.py +++ b/contrib/tools/python3/src/Lib/_collections_abc.py @@ -416,7 +416,7 @@ class Collection(Sized, Iterable, Container): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened``argtypes`` + This sets ``__args__`` to a tuple containing the flattened ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to @@ -426,32 +426,31 @@ class _CallableGenericAlias(GenericAlias): __slots__ = () def __new__(cls, origin, args): - try: - return cls.__create_ga(origin, args) - except TypeError as exc: - import warnings - warnings.warn(f'{str(exc)} ' - f'(This will raise a TypeError in Python 3.10.)', - DeprecationWarning) - return GenericAlias(origin, args) - - @classmethod - def __create_ga(cls, origin, args): - if not isinstance(args, tuple) or len(args) != 2: + if not (isinstance(args, tuple) and len(args) == 2): raise TypeError( "Callable must be used as Callable[[arg, ...], result].") t_args, t_result = args - if isinstance(t_args, (list, tuple)): - ga_args = tuple(t_args) + (t_result,) - # This relaxes what t_args can be on purpose to allow things like - # PEP 612 ParamSpec. Responsibility for whether a user is using - # Callable[...] properly is deferred to static type checkers. - else: - ga_args = args - return super().__new__(cls, origin, ga_args) + if isinstance(t_args, list): + args = (*t_args, t_result) + elif not _is_param_expr(t_args): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {t_args}") + return super().__new__(cls, origin, args) + + @property + def __parameters__(self): + params = [] + for arg in self.__args__: + # Looks like a genericalias + if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple): + params.extend(arg.__parameters__) + else: + if _is_typevarlike(arg): + params.append(arg) + return tuple(dict.fromkeys(params)) def __repr__(self): - if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: + if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() return (f'collections.abc.Callable' f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' @@ -459,20 +458,78 @@ class _CallableGenericAlias(GenericAlias): def __reduce__(self): args = self.__args__ - if not (len(args) == 2 and args[0] is Ellipsis): + if not (len(args) == 2 and _is_param_expr(args[0])): args = list(args[:-1]), args[-1] return _CallableGenericAlias, (Callable, args) def __getitem__(self, item): # Called during TypeVar substitution, returns the custom subclass - # rather than the default types.GenericAlias object. - ga = super().__getitem__(item) - args = ga.__args__ - t_result = args[-1] - t_args = args[:-1] - args = (t_args, t_result) - return _CallableGenericAlias(Callable, args) - + # rather than the default types.GenericAlias object. Most of the + # code is copied from typing's _GenericAlias and the builtin + # types.GenericAlias. + + # A special case in PEP 612 where if X = Callable[P, int], + # then X[int, str] == X[[int, str]]. + param_len = len(self.__parameters__) + if param_len == 0: + raise TypeError(f'{self} is not a generic class') + if not isinstance(item, tuple): + item = (item,) + if (param_len == 1 and _is_param_expr(self.__parameters__[0]) + and item and not _is_param_expr(item[0])): + item = (list(item),) + item_len = len(item) + if item_len != param_len: + raise TypeError(f'Too {"many" if item_len > param_len else "few"}' + f' arguments for {self};' + f' actual {item_len}, expected {param_len}') + subst = dict(zip(self.__parameters__, item)) + new_args = [] + for arg in self.__args__: + if _is_typevarlike(arg): + if _is_param_expr(arg): + arg = subst[arg] + if not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + else: + arg = subst[arg] + # Looks like a GenericAlias + elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple): + subparams = arg.__parameters__ + if subparams: + subargs = tuple(subst[x] for x in subparams) + arg = arg[subargs] + if isinstance(arg, tuple): + new_args.extend(arg) + else: + new_args.append(arg) + + # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612 + if not isinstance(new_args[0], list): + t_result = new_args[-1] + t_args = new_args[:-1] + new_args = (t_args, t_result) + return _CallableGenericAlias(Callable, tuple(new_args)) + + +def _is_typevarlike(arg): + obj = type(arg) + # looks like a TypeVar/ParamSpec + return (obj.__module__ == 'typing' + and obj.__name__ in {'ParamSpec', 'TypeVar'}) + +def _is_param_expr(obj): + """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or + ``_ConcatenateGenericAlias`` from typing.py + """ + if obj is Ellipsis: + return True + if isinstance(obj, list): + return True + obj = type(obj) + names = ('ParamSpec', '_ConcatenateGenericAlias') + return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). @@ -514,7 +571,6 @@ class Callable(metaclass=ABCMeta): class Set(Collection): - """A set is a finite, iterable container. This class provides concrete generic implementations of all @@ -740,19 +796,19 @@ MutableSet.register(set) ### MAPPINGS ### - class Mapping(Collection): - - __slots__ = () - """A Mapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __iter__, and __len__. - """ + __slots__ = () + + # Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set. + __abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING + @abstractmethod def __getitem__(self, key): raise KeyError @@ -791,7 +847,6 @@ class Mapping(Collection): __reversed__ = None - Mapping.register(mappingproxy) @@ -874,18 +929,16 @@ ValuesView.register(dict_values) class MutableMapping(Mapping): - - __slots__ = () - """A MutableMapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __setitem__, __delitem__, __iter__, and __len__. - """ + __slots__ = () + @abstractmethod def __setitem__(self, key, value): raise KeyError @@ -962,9 +1015,7 @@ MutableMapping.register(dict) ### SEQUENCES ### - class Sequence(Reversible, Collection): - """All the operations on a read-only sequence. Concrete subclasses must override __new__ or __init__, @@ -973,6 +1024,9 @@ class Sequence(Reversible, Collection): __slots__ = () + # Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set. + __abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE + @abstractmethod def __getitem__(self, index): raise IndexError @@ -1024,7 +1078,6 @@ class Sequence(Reversible, Collection): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v is value or v == value) - Sequence.register(tuple) Sequence.register(str) Sequence.register(range) @@ -1032,7 +1085,6 @@ Sequence.register(memoryview) class ByteString(Sequence): - """This unifies bytes and bytearray. XXX Should add all their methods. @@ -1045,16 +1097,14 @@ ByteString.register(bytearray) class MutableSequence(Sequence): - - __slots__ = () - """All the operations on a read-write sequence. Concrete subclasses must provide __new__ or __init__, __getitem__, __setitem__, __delitem__, __len__, and insert(). - """ + __slots__ = () + @abstractmethod def __setitem__(self, index, value): raise IndexError diff --git a/contrib/tools/python3/src/Lib/_compression.py b/contrib/tools/python3/src/Lib/_compression.py index b00f31b400c..e8b70aa0a3e 100644 --- a/contrib/tools/python3/src/Lib/_compression.py +++ b/contrib/tools/python3/src/Lib/_compression.py @@ -1,7 +1,7 @@ """Internal classes used by the gzip, lzma and bz2 modules""" import io - +import sys BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size @@ -110,6 +110,16 @@ class DecompressReader(io.RawIOBase): self._pos += len(data) return data + def readall(self): + chunks = [] + # sys.maxsize means the max length of output buffer is unlimited, + # so that the whole input buffer can be decompressed within one + # .decompress() call. + while data := self.read(sys.maxsize): + chunks.append(data) + + return b"".join(chunks) + # Rewind the file to the beginning of the data stream. def _rewind(self): self._fp.seek(0) diff --git a/contrib/tools/python3/src/Lib/_markupbase.py b/contrib/tools/python3/src/Lib/_markupbase.py index 2af5f1c23b6..3ad7e279960 100644 --- a/contrib/tools/python3/src/Lib/_markupbase.py +++ b/contrib/tools/python3/src/Lib/_markupbase.py @@ -29,10 +29,6 @@ class ParserBase: raise RuntimeError( "_markupbase.ParserBase must be subclassed") - def error(self, message): - raise NotImplementedError( - "subclasses of ParserBase must override error()") - def reset(self): self.lineno = 1 self.offset = 0 @@ -131,12 +127,11 @@ class ParserBase: # also in data attribute specifications of attlist declaration # also link type declaration subsets in linktype declarations # also link attribute specification lists in link declarations - self.error("unsupported '[' char in %s declaration" % decltype) + raise AssertionError("unsupported '[' char in %s declaration" % decltype) else: - self.error("unexpected '[' char in declaration") + raise AssertionError("unexpected '[' char in declaration") else: - self.error( - "unexpected %r char in declaration" % rawdata[j]) + raise AssertionError("unexpected %r char in declaration" % rawdata[j]) if j < 0: return j return -1 # incomplete @@ -156,7 +151,9 @@ class ParserBase: # look for MS Office ]> ending match= _msmarkedsectionclose.search(rawdata, i+3) else: - self.error('unknown status keyword %r in marked section' % rawdata[i+3:j]) + raise AssertionError( + 'unknown status keyword %r in marked section' % rawdata[i+3:j] + ) if not match: return -1 if report: @@ -168,7 +165,7 @@ class ParserBase: def parse_comment(self, i, report=1): rawdata = self.rawdata if rawdata[i:i+4] != ' diff --git a/contrib/tools/python3/src/Lib/codecs.py b/contrib/tools/python3/src/Lib/codecs.py index d2edd148a29..e6ad6e3a052 100644 --- a/contrib/tools/python3/src/Lib/codecs.py +++ b/contrib/tools/python3/src/Lib/codecs.py @@ -83,7 +83,7 @@ BOM64_BE = BOM_UTF32_BE class CodecInfo(tuple): """Codec details when looking up the codec registry""" - # Private API to allow Python 3.4 to blacklist the known non-Unicode + # Private API to allow Python 3.4 to denylist the known non-Unicode # codecs in the standard library. A more general mechanism to # reliably distinguish test encodings from other codecs will hopefully # be defined for Python 3.5 diff --git a/contrib/tools/python3/src/Lib/codeop.py b/contrib/tools/python3/src/Lib/codeop.py index 4c10470aee7..568e9bbc118 100644 --- a/contrib/tools/python3/src/Lib/codeop.py +++ b/contrib/tools/python3/src/Lib/codeop.py @@ -10,30 +10,6 @@ and: syntax error (OverflowError and ValueError can be produced by malformed literals). -Approach: - -First, check if the source consists entirely of blank lines and -comments; if so, replace it with 'pass', because the built-in -parser doesn't always do the right thing for these. - -Compile three times: as is, with \n, and with \n\n appended. If it -compiles as is, it's complete. If it compiles with one \n appended, -we expect more. If it doesn't compile either way, we compare the -error we get when compiling with \n or \n\n appended. If the errors -are the same, the code is broken. But if the errors are different, we -expect more. Not intuitive; not even guaranteed to hold in future -releases; but this matches the compiler's behavior from Python 1.4 -through 2.2, at least. - -Caveat: - -It is possible (but not likely) that the parser stops parsing with a -successful outcome before reaching the end of the source; in this -case, trailing symbols may be ignored instead of causing an error. -For example, a backslash followed by two newlines may be followed by -arbitrary garbage. This will be fixed once the API for the parser is -better. - The two interfaces are: compile_command(source, filename, symbol): @@ -64,24 +40,25 @@ _features = [getattr(__future__, fname) __all__ = ["compile_command", "Compile", "CommandCompiler"] -PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h +# The following flags match the values from Include/cpython/compile.h +# Caveat emptor: These flags are undocumented on purpose and depending +# on their effect outside the standard library is **unsupported**. +PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 def _maybe_compile(compiler, source, filename, symbol): - # Check for source consisting of only blank lines and comments + # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() if line and line[0] != '#': - break # Leave it alone + break # Leave it alone. else: if symbol != "eval": source = "pass" # Replace it with a 'pass' statement - err = err1 = err2 = None - code = code1 = code2 = None - try: - code = compiler(source, filename, symbol) - except SyntaxError: + return compiler(source, filename, symbol) + except SyntaxError: # Let other compile() errors propagate. pass # Catch syntax warnings after the first compile @@ -90,25 +67,23 @@ def _maybe_compile(compiler, source, filename, symbol): warnings.simplefilter("error") try: - code1 = compiler(source + "\n", filename, symbol) + compiler(source + "\n", filename, symbol) except SyntaxError as e: - err1 = e - - try: - code2 = compiler(source + "\n\n", filename, symbol) - except SyntaxError as e: - err2 = e - - try: - if code: - return code - if not code1 and repr(err1) == repr(err2): - raise err1 - finally: - err1 = err2 = None + if "incomplete input" in str(e): + return None + raise + +def _is_syntax_error(err1, err2): + rep1 = repr(err1) + rep2 = repr(err2) + if "was never closed" in rep1 and "was never closed" in rep2: + return False + if rep1 == rep2: + return True + return False def _compile(source, filename, symbol): - return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) + return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT) def compile_command(source, filename="", symbol="single"): r"""Compile a command and determine whether it is incomplete. @@ -137,7 +112,7 @@ class Compile: statement, it "remembers" and compiles all subsequent program texts with the statement in force.""" def __init__(self): - self.flags = PyCF_DONT_IMPLY_DEDENT + self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT def __call__(self, source, filename, symbol): codeob = compile(source, filename, symbol, self.flags, True) diff --git a/contrib/tools/python3/src/Lib/collections/__init__.py b/contrib/tools/python3/src/Lib/collections/__init__.py index 5bdd3b3516d..818588f7403 100644 --- a/contrib/tools/python3/src/Lib/collections/__init__.py +++ b/contrib/tools/python3/src/Lib/collections/__init__.py @@ -27,7 +27,6 @@ __all__ = [ ] import _collections_abc -import heapq as _heapq import sys as _sys from itertools import chain as _chain @@ -52,22 +51,6 @@ except ImportError: pass -def __getattr__(name): - # For backwards compatibility, continue to make the collections ABCs - # through Python 3.6 available through the collections module. - # Note, no new collections ABCs were added in Python 3.7 - if name in _collections_abc.__all__: - obj = getattr(_collections_abc, name) - import warnings - warnings.warn("Using or importing the ABCs from 'collections' instead " - "of from 'collections.abc' is deprecated since Python 3.3, " - "and in 3.10 it will stop working", - DeprecationWarning, stacklevel=2) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - ################################################################################ ### OrderedDict ################################################################################ @@ -489,6 +472,7 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non '__repr__': __repr__, '_asdict': _asdict, '__getnewargs__': __getnewargs__, + '__match_args__': field_names, } for index, name in enumerate(field_names): doc = _sys.intern(f'Alias for field number {index}') @@ -597,6 +581,10 @@ class Counter(dict): # Needed so that self[missing_item] does not raise KeyError return 0 + def total(self): + 'Sum of the counts' + return sum(self.values()) + def most_common(self, n=None): '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. @@ -608,7 +596,10 @@ class Counter(dict): # Emulate Bag.sortedByCount from Smalltalk if n is None: return sorted(self.items(), key=_itemgetter(1), reverse=True) - return _heapq.nlargest(n, self.items(), key=_itemgetter(1)) + + # Lazy import to speedup Python startup time + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. @@ -719,6 +710,42 @@ class Counter(dict): if elem in self: super().__delitem__(elem) + def __eq__(self, other): + 'True if all counts agree. Missing counts are treated as zero.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] == other[e] for c in (self, other) for e in c) + + def __ne__(self, other): + 'True if any counts disagree. Missing counts are treated as zero.' + if not isinstance(other, Counter): + return NotImplemented + return not self == other + + def __le__(self, other): + 'True if all counts in self are a subset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] <= other[e] for c in (self, other) for e in c) + + def __lt__(self, other): + 'True if all counts in self are a proper subset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return self <= other and self != other + + def __ge__(self, other): + 'True if all counts in self are a superset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] >= other[e] for c in (self, other) for e in c) + + def __gt__(self, other): + 'True if all counts in self are a proper superset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return self >= other and self != other + def __repr__(self): if not self: return f'{self.__class__.__name__}()' @@ -739,12 +766,30 @@ class Counter(dict): # To strip negative and zero counts, add-in an empty counter: # c += Counter() # - # Rich comparison operators for multiset subset and superset tests - # are deliberately omitted due to semantic conflicts with the - # existing inherited dict equality method. Subset and superset - # semantics ignore zero counts and require that p≤q ∧ p≥q → p=q; - # however, that would not be the case for p=Counter(a=1, b=0) - # and q=Counter(a=1) where the dictionaries are not equal. + # Results are ordered according to when an element is first + # encountered in the left operand and then by the order + # encountered in the right operand. + # + # When the multiplicities are all zero or one, multiset operations + # are guaranteed to be equivalent to the corresponding operations + # for regular sets. + # Given counter multisets such as: + # cp = Counter(a=1, b=0, c=1) + # cq = Counter(c=1, d=0, e=1) + # The corresponding regular sets would be: + # sp = {'a', 'c'} + # sq = {'c', 'e'} + # All of the following relations would hold: + # set(cp + cq) == sp | sq + # set(cp - cq) == sp - sq + # set(cp | cq) == sp | sq + # set(cp & cq) == sp & sq + # (cp == cq) == (sp == sq) + # (cp != cq) == (sp != sq) + # (cp <= cq) == (sp <= sq) + # (cp < cq) == (sp < sq) + # (cp >= cq) == (sp >= sq) + # (cp > cq) == (sp > sq) def __add__(self, other): '''Add counts from two counters. @@ -973,12 +1018,15 @@ class ChainMap(_collections_abc.MutableMapping): __copy__ = copy - def new_child(self, m=None): # like Django's Context.push() + def new_child(self, m=None, **kwargs): # like Django's Context.push() '''New ChainMap with a new map followed by all previous maps. If no map is provided, an empty dict is used. + Keyword arguments update the map or new empty dict. ''' if m is None: - m = {} + m = kwargs + elif kwargs: + m.update(kwargs) return self.__class__(m, *self.maps) @property diff --git a/contrib/tools/python3/src/Lib/colorsys.py b/contrib/tools/python3/src/Lib/colorsys.py index b93e3844067..0f52512a67d 100644 --- a/contrib/tools/python3/src/Lib/colorsys.py +++ b/contrib/tools/python3/src/Lib/colorsys.py @@ -75,17 +75,18 @@ def yiq_to_rgb(y, i, q): def rgb_to_hls(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) - # XXX Can optimize (maxc+minc) and (maxc-minc) - l = (minc+maxc)/2.0 + sumc = (maxc+minc) + rangec = (maxc-minc) + l = sumc/2.0 if minc == maxc: return 0.0, l, 0.0 if l <= 0.5: - s = (maxc-minc) / (maxc+minc) + s = rangec / sumc else: - s = (maxc-minc) / (2.0-maxc-minc) - rc = (maxc-r) / (maxc-minc) - gc = (maxc-g) / (maxc-minc) - bc = (maxc-b) / (maxc-minc) + s = rangec / (2.0-sumc) + rc = (maxc-r) / rangec + gc = (maxc-g) / rangec + bc = (maxc-b) / rangec if r == maxc: h = bc-gc elif g == maxc: diff --git a/contrib/tools/python3/src/Lib/compileall.py b/contrib/tools/python3/src/Lib/compileall.py index 25ad83c2290..3755e76ba81 100644 --- a/contrib/tools/python3/src/Lib/compileall.py +++ b/contrib/tools/python3/src/Lib/compileall.py @@ -84,12 +84,14 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if workers < 0: raise ValueError('workers must be greater or equal to 0') if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: + _check_system_limits() + except NotImplementedError: workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) @@ -404,7 +406,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: diff --git a/contrib/tools/python3/src/Lib/concurrent/futures/process.py b/contrib/tools/python3/src/Lib/concurrent/futures/process.py index a29e5247ab8..6ee2ce626e4 100644 --- a/contrib/tools/python3/src/Lib/concurrent/futures/process.py +++ b/contrib/tools/python3/src/Lib/concurrent/futures/process.py @@ -532,6 +532,14 @@ def _check_system_limits(): if _system_limited: raise NotImplementedError(_system_limited) _system_limits_checked = True + try: + import multiprocessing.synchronize + except ImportError: + _system_limited = ( + "This Python build lacks multiprocessing.synchronize, usually due " + "to named semaphores being unavailable on this platform." + ) + raise NotImplementedError(_system_limited) try: nsems_max = os.sysconf("SC_SEM_NSEMS_MAX") except (AttributeError, ValueError): diff --git a/contrib/tools/python3/src/Lib/configparser.py b/contrib/tools/python3/src/Lib/configparser.py index 8dd5c13bcc0..3470624e63f 100644 --- a/contrib/tools/python3/src/Lib/configparser.py +++ b/contrib/tools/python3/src/Lib/configparser.py @@ -316,7 +316,7 @@ class ParsingError(Error): def filename(self): """Deprecated, use `source'.""" warnings.warn( - "The 'filename' attribute will be removed in future versions. " + "The 'filename' attribute will be removed in Python 3.12. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) @@ -326,7 +326,7 @@ class ParsingError(Error): def filename(self, value): """Deprecated, user `source'.""" warnings.warn( - "The 'filename' attribute will be removed in future versions. " + "The 'filename' attribute will be removed in Python 3.12. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) @@ -563,7 +563,7 @@ class RawConfigParser(MutableMapping): # Regular expressions for parsing section headers and options _SECT_TMPL = r""" \[ # [ - (?P
[^]]+) # very permissive! + (?P
.+) # very permissive! \] # ] """ _OPT_TMPL = r""" @@ -690,6 +690,7 @@ class RawConfigParser(MutableMapping): """ if isinstance(filenames, (str, bytes, os.PathLike)): filenames = [filenames] + encoding = io.text_encoding(encoding) read_ok = [] for filename in filenames: try: @@ -756,7 +757,7 @@ class RawConfigParser(MutableMapping): def readfp(self, fp, filename=None): """Deprecated, use read_file instead.""" warnings.warn( - "This method will be removed in future versions. " + "This method will be removed in Python 3.12. " "Use 'parser.read_file()' instead.", DeprecationWarning, stacklevel=2 ) @@ -1231,7 +1232,7 @@ class SafeConfigParser(ConfigParser): super().__init__(*args, **kwargs) warnings.warn( "The SafeConfigParser class has been renamed to ConfigParser " - "in Python 3.2. This alias will be removed in future versions." + "in Python 3.2. This alias will be removed in Python 3.12." " Use ConfigParser directly instead.", DeprecationWarning, stacklevel=2 ) diff --git a/contrib/tools/python3/src/Lib/contextlib.py b/contrib/tools/python3/src/Lib/contextlib.py index 4e8f5f75939..c63a8492e2d 100644 --- a/contrib/tools/python3/src/Lib/contextlib.py +++ b/contrib/tools/python3/src/Lib/contextlib.py @@ -9,7 +9,7 @@ from types import MethodType, GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", "AsyncExitStack", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress"] + "redirect_stdout", "redirect_stderr", "suppress", "aclosing"] class AbstractContextManager(abc.ABC): @@ -80,6 +80,22 @@ class ContextDecorator(object): return inner +class AsyncContextDecorator(object): + "A base class or mixin that enables async context managers to work as decorators." + + def _recreate_cm(self): + """Return a recreated instance of self. + """ + return self + + def __call__(self, func): + @wraps(func) + async def inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + return inner + + class _GeneratorContextManagerBase: """Shared functionality for @contextmanager and @asynccontextmanager.""" @@ -168,9 +184,11 @@ class _GeneratorContextManager( return False raise RuntimeError("generator didn't stop after throw()") - -class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, - AbstractAsyncContextManager): +class _AsyncGeneratorContextManager( + _GeneratorContextManagerBase, + AbstractAsyncContextManager, + AsyncContextDecorator, +): """Helper for @asynccontextmanager decorator.""" async def __aenter__(self): @@ -178,14 +196,14 @@ class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, # they are only needed for recreation, which is not possible anymore del self.args, self.kwds, self.func try: - return await self.gen.__anext__() + return await anext(self.gen) except StopAsyncIteration: raise RuntimeError("generator didn't yield") from None async def __aexit__(self, typ, value, traceback): if typ is None: try: - await self.gen.__anext__() + await anext(self.gen) except StopAsyncIteration: return False else: @@ -322,6 +340,32 @@ class closing(AbstractContextManager): self.thing.close() +class aclosing(AbstractAsyncContextManager): + """Async context manager for safely finalizing an asynchronously cleaned-up + resource such as an async generator, calling its ``aclose()`` method. + + Code like this: + + async with aclosing(.fetch()) as agen: + + + is equivalent to this: + + agen = .fetch() + try: + + finally: + await agen.aclose() + + """ + def __init__(self, thing): + self.thing = thing + async def __aenter__(self): + return self.thing + async def __aexit__(self, *exc_info): + await self.thing.aclose() + + class _RedirectStream(AbstractContextManager): _stream = None @@ -674,7 +718,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): return received_exc and suppressed_exc -class nullcontext(AbstractContextManager): +class nullcontext(AbstractContextManager, AbstractAsyncContextManager): """Context manager that does no additional processing. Used as a stand-in for a normal context manager, when a particular @@ -693,3 +737,9 @@ class nullcontext(AbstractContextManager): def __exit__(self, *excinfo): pass + + async def __aenter__(self): + return self.enter_result + + async def __aexit__(self, *excinfo): + pass diff --git a/contrib/tools/python3/src/Lib/copy.py b/contrib/tools/python3/src/Lib/copy.py index 1081d43952e..69bac980be2 100644 --- a/contrib/tools/python3/src/Lib/copy.py +++ b/contrib/tools/python3/src/Lib/copy.py @@ -192,6 +192,7 @@ d[bytes] = _deepcopy_atomic d[str] = _deepcopy_atomic d[types.CodeType] = _deepcopy_atomic d[type] = _deepcopy_atomic +d[range] = _deepcopy_atomic d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[weakref.ref] = _deepcopy_atomic diff --git a/contrib/tools/python3/src/Lib/copyreg.py b/contrib/tools/python3/src/Lib/copyreg.py index 7ab8c128eb0..356db6f083e 100644 --- a/contrib/tools/python3/src/Lib/copyreg.py +++ b/contrib/tools/python3/src/Lib/copyreg.py @@ -36,6 +36,12 @@ else: pickle(complex, pickle_complex, complex) +def pickle_union(obj): + import functools, operator + return functools.reduce, (operator.or_, obj.__args__) + +pickle(type(int | str), pickle_union) + # Support for pickling new-style objects def _reconstructor(cls, base, state): diff --git a/contrib/tools/python3/src/Lib/csv.py b/contrib/tools/python3/src/Lib/csv.py index dc85077f3ec..bb3ee269ae7 100644 --- a/contrib/tools/python3/src/Lib/csv.py +++ b/contrib/tools/python3/src/Lib/csv.py @@ -409,14 +409,10 @@ class Sniffer: continue # skip rows that have irregular number of columns for col in list(columnTypes.keys()): - - for thisType in [int, float, complex]: - try: - thisType(row[col]) - break - except (ValueError, OverflowError): - pass - else: + thisType = complex + try: + thisType(row[col]) + except (ValueError, OverflowError): # fallback to length of string thisType = len(row[col]) diff --git a/contrib/tools/python3/src/Lib/dataclasses.py b/contrib/tools/python3/src/Lib/dataclasses.py index 5ff67ad2ea9..105a95b9554 100644 --- a/contrib/tools/python3/src/Lib/dataclasses.py +++ b/contrib/tools/python3/src/Lib/dataclasses.py @@ -6,8 +6,9 @@ import inspect import keyword import builtins import functools +import abc import _thread -from types import GenericAlias +from types import FunctionType, GenericAlias __all__ = ['dataclass', @@ -15,6 +16,7 @@ __all__ = ['dataclass', 'Field', 'FrozenInstanceError', 'InitVar', + 'KW_ONLY', 'MISSING', # Helper functions. @@ -151,6 +153,20 @@ __all__ = ['dataclass', # # See _hash_action (below) for a coded version of this table. +# __match_args__ +# +# +--- match_args= parameter +# | +# v | | | +# | no | yes | <--- class has __match_args__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ +# __match_args__ is always added unless the class already defines it. It is a +# tuple of __init__ parameter names; non-init fields must be matched by keyword. + # Raised when an attempt is made to modify a frozen class. class FrozenInstanceError(AttributeError): pass @@ -169,6 +185,12 @@ class _MISSING_TYPE: pass MISSING = _MISSING_TYPE() +# A sentinel object to indicate that following fields are keyword-only by +# default. Use a class to give it a better repr. +class _KW_ONLY_TYPE: + pass +KW_ONLY = _KW_ONLY_TYPE() + # Since most per-field metadata will be unused, create an empty # read-only proxy that can be shared among all fields. _EMPTY_METADATA = types.MappingProxyType({}) @@ -217,7 +239,6 @@ class InitVar: def __class_getitem__(cls, type): return InitVar(type) - # Instances of Field are only ever created from within this module, # and only from the field() function, although Field instances are # exposed externally as (conceptually) read-only objects. @@ -238,11 +259,12 @@ class Field: 'init', 'compare', 'metadata', + 'kw_only', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata): + metadata, kw_only): self.name = None self.type = None self.default = default @@ -254,6 +276,7 @@ class Field: self.metadata = (_EMPTY_METADATA if metadata is None else types.MappingProxyType(metadata)) + self.kw_only = kw_only self._field_type = None def __repr__(self): @@ -267,6 +290,7 @@ class Field: f'hash={self.hash!r},' f'compare={self.compare!r},' f'metadata={self.metadata!r},' + f'kw_only={self.kw_only!r},' f'_field_type={self._field_type}' ')') @@ -320,17 +344,19 @@ class _DataclassParams: # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None): + hash=None, compare=True, metadata=None, kw_only=MISSING): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a 0-argument function called to initialize a field's value. If init - is True, the field will be a parameter to the class's __init__() - function. If repr is True, the field will be included in the - object's repr(). If hash is True, the field will be included in - the object's hash(). If compare is True, the field will be used - in comparison functions. metadata, if specified, must be a - mapping which is stored but not otherwise examined by dataclass. + is true, the field will be a parameter to the class's __init__() + function. If repr is true, the field will be included in the + object's repr(). If hash is true, the field will be included in the + object's hash(). If compare is true, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. If kw_only + is true, the field will become a keyword-only parameter to + __init__(). It is an error to specify both default and default_factory. """ @@ -338,7 +364,16 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata) + metadata, kw_only) + + +def _fields_in_init_order(fields): + # Returns the fields as __init__ will output them. It returns 2 tuples: + # the first for normal args, and the second for keyword args. + + return (tuple(f for f in fields if f.init and not f.kw_only), + tuple(f for f in fields if f.init and f.kw_only) + ) def _tuple_str(obj_name, fields): @@ -395,7 +430,6 @@ def _create_fn(name, args, body, *, globals=None, locals=None, local_vars = ', '.join(locals.keys()) txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} exec(txt, globals, ns) return ns['__create_fn__'](**locals) @@ -413,7 +447,7 @@ def _field_assign(frozen, name, value, self_name): return f'{self_name}.{name}={value}' -def _field_init(f, frozen, globals, self_name): +def _field_init(f, frozen, globals, self_name, slots): # Return the text of the line in the body of __init__ that will # initialize this field. @@ -453,9 +487,15 @@ def _field_init(f, frozen, globals, self_name): globals[default_name] = f.default value = f.name else: - # This field does not need initialization. Signify that - # to the caller by returning None. - return None + # If the class has slots, then initialize this field. + if slots and f.default is not MISSING: + globals[default_name] = f.default + value = default_name + else: + # This field does not need initialization: reading from it will + # just use the class attribute that contains the default. + # Signify that to the caller by returning None. + return None # Only test this now, so that we can create variables for the # default. However, return None to signify that we're not going @@ -486,7 +526,8 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name, globals): +def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, + self_name, globals, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -494,9 +535,10 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): # function source code, but catching it here gives a better error # message, and future-proofs us in case we build up the function # using ast. + seen_default = False - for f in fields: - # Only consider fields in the __init__ call. + for f in std_fields: + # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): seen_default = True @@ -512,7 +554,7 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): body_lines = [] for f in fields: - line = _field_init(f, frozen, locals, self_name) + line = _field_init(f, frozen, locals, self_name, slots) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -528,8 +570,15 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): if not body_lines: body_lines = ['pass'] + _init_params = [_init_param(f) for f in std_fields] + if kw_only_fields: + # Add the keyword-only args. Because the * can only be added if + # there's at least one keyword-only arg, there needs to be a test here + # (instead of just concatenting the lists together). + _init_params += ['*'] + _init_params += [_init_param(f) for f in kw_only_fields] return _create_fn('__init__', - [self_name] + [_init_param(f) for f in fields if f.init], + [self_name] + _init_params, body_lines, locals=locals, globals=globals, @@ -608,6 +657,9 @@ def _is_initvar(a_type, dataclasses): return (a_type is dataclasses.InitVar or type(a_type) is dataclasses.InitVar) +def _is_kw_only(a_type, dataclasses): + return a_type is dataclasses.KW_ONLY + def _is_type(annotation, cls, a_module, a_type, is_type_predicate): # Given a type annotation string, does it refer to a_type in @@ -668,10 +720,11 @@ def _is_type(annotation, cls, a_module, a_type, is_type_predicate): return False -def _get_field(cls, a_name, a_type): - # Return a Field object for this field name and type. ClassVars - # and InitVars are also returned, but marked as such (see - # f._field_type). +def _get_field(cls, a_name, a_type, default_kw_only): + # Return a Field object for this field name and type. ClassVars and + # InitVars are also returned, but marked as such (see f._field_type). + # default_kw_only is the value of kw_only to use if there isn't a field() + # that defines it. # If the default value isn't derived from Field, then it's only a # normal default value. Convert it to a Field(). @@ -742,6 +795,19 @@ def _get_field(cls, a_name, a_type): # init=)? It makes no sense for # ClassVar and InitVar to specify init=. + # kw_only validation and assignment. + if f._field_type in (_FIELD, _FIELD_INITVAR): + # For real and InitVar fields, if kw_only wasn't specified use the + # default value. + if f.kw_only is MISSING: + f.kw_only = default_kw_only + else: + # Make sure kw_only isn't set for ClassVars + assert f._field_type is _FIELD_CLASSVAR + if f.kw_only is not MISSING: + raise TypeError(f'field {f.name} is a ClassVar but specifies ' + 'kw_only') + # For real fields, disallow mutable defaults for known types. if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): raise ValueError(f'mutable default {type(f.default)} for field ' @@ -749,12 +815,19 @@ def _get_field(cls, a_name, a_type): return f +def _set_qualname(cls, value): + # Ensure that the functions returned from _create_fn uses the proper + # __qualname__ (the class they belong to). + if isinstance(value, FunctionType): + value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" + return value def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True + _set_qualname(cls, value) setattr(cls, name, value) return False @@ -769,7 +842,7 @@ def _hash_set_none(cls, fields, globals): def _hash_add(cls, fields, globals): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _hash_fn(flds, globals) + return _set_qualname(cls, _hash_fn(flds, globals)) def _hash_exception(cls, fields, globals): # Raise an exception. @@ -806,7 +879,8 @@ _hash_action = {(False, False, False, False): None, # version of this table. -def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order @@ -860,8 +934,27 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where # we can. - cls_fields = [_get_field(cls, name, type) - for name, type in cls_annotations.items()] + cls_fields = [] + # Get a reference to this module for the _is_kw_only() test. + KW_ONLY_seen = False + dataclasses = sys.modules[__name__] + for name, type in cls_annotations.items(): + # See if this is a marker to change the value of kw_only. + if (_is_kw_only(type, dataclasses) + or (isinstance(type, str) + and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY, + _is_kw_only))): + # Switch the default to kw_only=True, and ignore this + # annotation: it's not a real field. + if KW_ONLY_seen: + raise TypeError(f'{name!r} is KW_ONLY, but KW_ONLY ' + 'has already been specified') + KW_ONLY_seen = True + kw_only = True + else: + # Otherwise it's a field of some type. + cls_fields.append(_get_field(cls, name, type, kw_only)) + for f in cls_fields: fields[f.name] = f @@ -916,15 +1009,22 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if order and not eq: raise ValueError('eq must be true if order is true') + # Include InitVars and regular fields (so, not ClassVars). This is + # initialized here, outside of the "if init:" test, because std_init_fields + # is used with match_args, below. + all_init_fields = [f for f in fields.values() + if f._field_type in (_FIELD, _FIELD_INITVAR)] + (std_init_fields, + kw_only_init_fields) = _fields_in_init_order(all_init_fields) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - # Include InitVars and regular fields (so, not ClassVars). - flds = [f for f in fields.values() - if f._field_type in (_FIELD, _FIELD_INITVAR)] _set_new_attribute(cls, '__init__', - _init_fn(flds, + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, frozen, has_post_init, # The name to use for the "self" @@ -933,6 +1033,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): '__dataclass_self__' if 'self' in fields else 'self', globals, + slots, )) # Get the fields as a list, and include only real fields. This is @@ -992,11 +1093,70 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): cls.__doc__ = (cls.__name__ + str(inspect.signature(cls)).replace(' -> None', '')) + if match_args: + # I could probably compute this once + _set_new_attribute(cls, '__match_args__', + tuple(f.name for f in std_init_fields)) + + if slots: + cls = _add_slots(cls, frozen) + + abc.update_abstractmethods(cls) + + return cls + + +# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen +# classes with slots. These could be slighly more performant if we generated +# the code instead of iterating over fields. But that can be a project for +# another day, if performance becomes an issue. +def _dataclass_getstate(self): + return [getattr(self, f.name) for f in fields(self)] + + +def _dataclass_setstate(self, state): + for field, value in zip(fields(self), state): + # use setattr because dataclass may be frozen + object.__setattr__(self, field.name, value) + + +def _add_slots(cls, is_frozen): + # Need to create a new class, since we can't set __slots__ + # after a class has been created. + + # Make sure __slots__ isn't already set. + if '__slots__' in cls.__dict__: + raise TypeError(f'{cls.__name__} already specifies __slots__') + + # Create a new dict for our new class. + cls_dict = dict(cls.__dict__) + field_names = tuple(f.name for f in fields(cls)) + cls_dict['__slots__'] = field_names + for field_name in field_names: + # Remove our attributes, if present. They'll still be + # available in _MARKER. + cls_dict.pop(field_name, None) + + # Remove __dict__ itself. + cls_dict.pop('__dict__', None) + + # And finally create the class. + qualname = getattr(cls, '__qualname__', None) + cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + if qualname is not None: + cls.__qualname__ = qualname + + if is_frozen: + # Need this for pickling frozen classes with slots. + cls.__getstate__ = _dataclass_getstate + cls.__setstate__ = _dataclass_setstate + return cls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=False, frozen=False): + unsafe_hash=False, frozen=False, match_args=True, + kw_only=False, slots=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -1006,11 +1166,15 @@ def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, repr is true, a __repr__() method is added. If order is true, rich comparison dunder methods are added. If unsafe_hash is true, a __hash__() method function is added. If frozen is true, fields may - not be assigned to after instance creation. + not be assigned to after instance creation. If match_args is true, + the __match_args__ tuple is added. If kw_only is true, then by + default all fields are keyword-only. If slots is true, an + __slots__ attribute is added. """ def wrap(cls): - return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, + frozen, match_args, kw_only, slots) # See if we're being called as @dataclass or @dataclass(). if cls is None: @@ -1169,7 +1333,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, - frozen=False): + frozen=False, match_args=True, kw_only=False, slots=False): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1195,14 +1359,12 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, if namespace is None: namespace = {} - else: - # Copy namespace since we're going to mutate it. - namespace = namespace.copy() # While we're looking through the field names, validate that they # are identifiers, are not keywords, and not duplicates. seen = set() - anns = {} + annotations = {} + defaults = {} for item in fields: if isinstance(item, str): name = item @@ -1211,7 +1373,7 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, name, tp, = item elif len(item) == 3: name, tp, spec = item - namespace[name] = spec + defaults[name] = spec else: raise TypeError(f'Invalid field: {item!r}') @@ -1223,14 +1385,22 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, raise TypeError(f'Field name duplicated: {name!r}') seen.add(name) - anns[name] = tp + annotations[name] = tp + + # Update 'ns' with the user-supplied namespace plus our calculated values. + def exec_body_callback(ns): + ns.update(namespace) + ns.update(defaults) + ns['__annotations__'] = annotations - namespace['__annotations__'] = anns # We use `types.new_class()` instead of simply `type()` to allow dynamic creation - # of generic dataclassses. - cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace)) + # of generic dataclasses. + cls = types.new_class(cls_name, bases, {}, exec_body_callback) + + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen) + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots) def replace(obj, /, **changes): diff --git a/contrib/tools/python3/src/Lib/datetime.py b/contrib/tools/python3/src/Lib/datetime.py index 23d2bf09181..6bf37ccfab7 100644 --- a/contrib/tools/python3/src/Lib/datetime.py +++ b/contrib/tools/python3/src/Lib/datetime.py @@ -11,6 +11,7 @@ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", import time as _time import math as _math import sys +from operator import index as _index def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 @@ -380,42 +381,10 @@ def _check_utc_offset(name, offset): "-timedelta(hours=24) and timedelta(hours=24)" % (name, offset)) -def _check_int_field(value): - if isinstance(value, int): - return value - if isinstance(value, float): - raise TypeError('integer argument expected, got float') - try: - value = value.__index__() - except AttributeError: - pass - else: - if not isinstance(value, int): - raise TypeError('__index__ returned non-int (type %s)' % - type(value).__name__) - return value - orig = value - try: - value = value.__int__() - except AttributeError: - pass - else: - if not isinstance(value, int): - raise TypeError('__int__ returned non-int (type %s)' % - type(value).__name__) - import warnings - warnings.warn("an integer is required (got type %s)" % - type(orig).__name__, - DeprecationWarning, - stacklevel=2) - return value - raise TypeError('an integer is required (got type %s)' % - type(value).__name__) - def _check_date_fields(year, month, day): - year = _check_int_field(year) - month = _check_int_field(month) - day = _check_int_field(day) + year = _index(year) + month = _index(month) + day = _index(day) if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -426,10 +395,10 @@ def _check_date_fields(year, month, day): return year, month, day def _check_time_fields(hour, minute, second, microsecond, fold): - hour = _check_int_field(hour) - minute = _check_int_field(minute) - second = _check_int_field(second) - microsecond = _check_int_field(microsecond) + hour = _index(hour) + minute = _index(minute) + second = _index(second) + microsecond = _index(microsecond) if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -2541,10 +2510,10 @@ else: # Clean up unused names del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, - _check_date_fields, _check_int_field, _check_time_fields, + _check_date_fields, _check_time_fields, _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _format_offset, _is_leap, _isoweek1monday, _math, + _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math, _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, _parse_hh_mm_ss_ff, _IsoCalendarDate) diff --git a/contrib/tools/python3/src/Lib/dis.py b/contrib/tools/python3/src/Lib/dis.py index e289e176c78..fe5d24e8805 100644 --- a/contrib/tools/python3/src/Lib/dis.py +++ b/contrib/tools/python3/src/Lib/dis.py @@ -338,8 +338,11 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None, argval, argrepr = _get_const_info(arg, constants) elif op in hasname: argval, argrepr = _get_name_info(arg, names) + elif op in hasjabs: + argval = arg*2 + argrepr = "to " + repr(argval) elif op in hasjrel: - argval = offset + 2 + arg + argval = offset + 2 + arg*2 argrepr = "to " + repr(argval) elif op in haslocal: argval, argrepr = _get_name_info(arg, varnames) @@ -384,7 +387,7 @@ def _disassemble_bytes(code, lasti=-1, varnames=None, names=None, constants=None, cells=None, linestarts=None, *, file=None, line_offset=0): # Omit the line number column entirely if we have no line number info - show_lineno = linestarts is not None + show_lineno = bool(linestarts) if show_lineno: maxlineno = max(linestarts.values()) + line_offset if maxlineno >= 1000: @@ -425,6 +428,7 @@ def _unpack_opargs(code): extended_arg = (arg << 8) if op == EXTENDED_ARG else 0 else: arg = None + extended_arg = 0 yield (i, op, arg) def findlabels(code): @@ -437,9 +441,9 @@ def findlabels(code): for offset, op, arg in _unpack_opargs(code): if arg is not None: if op in hasjrel: - label = offset + 2 + arg + label = offset + 2 + arg*2 elif op in hasjabs: - label = arg + label = arg*2 else: continue if label not in labels: @@ -449,32 +453,15 @@ def findlabels(code): def findlinestarts(code): """Find the offsets in a byte code which are start of lines in the source. - Generate pairs (offset, lineno) as described in Python/compile.c. - + Generate pairs (offset, lineno) """ - byte_increments = code.co_lnotab[0::2] - line_increments = code.co_lnotab[1::2] - bytecode_len = len(code.co_code) - - lastlineno = None - lineno = code.co_firstlineno - addr = 0 - for byte_incr, line_incr in zip(byte_increments, line_increments): - if byte_incr: - if lineno != lastlineno: - yield (addr, lineno) - lastlineno = lineno - addr += byte_incr - if addr >= bytecode_len: - # The rest of the lnotab byte offsets are past the end of - # the bytecode, so the lines were optimized away. - return - if line_incr >= 0x80: - # line_increments is an array of 8-bit signed integers - line_incr -= 0x100 - lineno += line_incr - if lineno != lastlineno: - yield (addr, lineno) + lastline = None + for start, end, line in code.co_lines(): + if line is not None and line != lastline: + lastline = line + yield start, line + return + class Bytecode: """The bytecode operations of a piece of code diff --git a/contrib/tools/python3/src/Lib/distutils/__init__.py b/contrib/tools/python3/src/Lib/distutils/__init__.py index d823d040a1c..fdad6f65a78 100644 --- a/contrib/tools/python3/src/Lib/distutils/__init__.py +++ b/contrib/tools/python3/src/Lib/distutils/__init__.py @@ -9,5 +9,12 @@ used from a setup script as """ import sys +import warnings __version__ = sys.version[:sys.version.index(' ')] + +_DEPRECATION_MESSAGE = ("The distutils package is deprecated and slated for " + "removal in Python 3.12. Use setuptools or check " + "PEP 632 for potential alternatives") +warnings.warn(_DEPRECATION_MESSAGE, + DeprecationWarning, 2) diff --git a/contrib/tools/python3/src/Lib/distutils/command/__init__.py b/contrib/tools/python3/src/Lib/distutils/command/__init__.py index 481eea9fd4b..fd0bfae7ade 100644 --- a/contrib/tools/python3/src/Lib/distutils/command/__init__.py +++ b/contrib/tools/python3/src/Lib/distutils/command/__init__.py @@ -19,7 +19,6 @@ __all__ = ['build', 'bdist', 'bdist_dumb', 'bdist_rpm', - 'bdist_wininst', 'check', 'upload', # These two are reserved for future use: diff --git a/contrib/tools/python3/src/Lib/distutils/command/bdist.py b/contrib/tools/python3/src/Lib/distutils/command/bdist.py index 014871d280e..d580a8090bd 100644 --- a/contrib/tools/python3/src/Lib/distutils/command/bdist.py +++ b/contrib/tools/python3/src/Lib/distutils/command/bdist.py @@ -62,7 +62,7 @@ class bdist(Command): # Establish the preferred order (for the --help-formats option). format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'wininst', 'zip', 'msi'] + 'zip', 'msi'] # And the real information. format_command = {'rpm': ('bdist_rpm', "RPM distribution"), @@ -71,8 +71,6 @@ class bdist(Command): 'xztar': ('bdist_dumb', "xz'ed tar file"), 'ztar': ('bdist_dumb', "compressed tar file"), 'tar': ('bdist_dumb', "tar file"), - 'wininst': ('bdist_wininst', - "Windows executable installer"), 'zip': ('bdist_dumb', "ZIP file"), 'msi': ('bdist_msi', "Microsoft Installer") } diff --git a/contrib/tools/python3/src/Lib/distutils/command/bdist_msi.py b/contrib/tools/python3/src/Lib/distutils/command/bdist_msi.py index 0863a1883e7..2ed017b4d66 100644 --- a/contrib/tools/python3/src/Lib/distutils/command/bdist_msi.py +++ b/contrib/tools/python3/src/Lib/distutils/command/bdist_msi.py @@ -1,7 +1,5 @@ # Copyright (C) 2005, 2006 Martin von Löwis # Licensed to PSF under a Contributor Agreement. -# The bdist_wininst command proper -# based on bdist_wininst """ Implements the bdist_msi command. """ diff --git a/contrib/tools/python3/src/Lib/distutils/command/bdist_wininst.py b/contrib/tools/python3/src/Lib/distutils/command/bdist_wininst.py deleted file mode 100644 index 0e9ddaa2141..00000000000 --- a/contrib/tools/python3/src/Lib/distutils/command/bdist_wininst.py +++ /dev/null @@ -1,377 +0,0 @@ -"""distutils.command.bdist_wininst - -Implements the Distutils 'bdist_wininst' command: create a windows installer -exe-program.""" - -import os -import sys -import warnings -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import remove_tree -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_wininst(Command): - - description = "create an executable installer for MS Windows" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('bitmap=', 'b', - "bitmap to use for the installer instead of python-powered logo"), - ('title=', 't', - "title to display on the installer background instead of default"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ('user-access-control=', None, - "specify Vista's UAC handling - 'none'/default=no " - "handling, 'auto'=use UAC if target Python installed for " - "all users, 'force'=always use UAC"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - # bpo-10945: bdist_wininst requires mbcs encoding only available on Windows - _unsupported = (sys.platform != "win32") - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - warnings.warn("bdist_wininst command is deprecated since Python 3.8, " - "use bdist_wheel (wheel packages) instead", - DeprecationWarning, 2) - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.bitmap = None - self.title = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.user_access_control = None - - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - if self.skip_build and self.plat_name: - # If build is skipped and plat_name is overridden, bdist will - # not see the correct 'plat_name' - so set that up manually. - bdist = self.distribution.get_command_obj('bdist') - bdist.plat_name = self.plat_name - # next the command will be initialized using that name - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'wininst') - - if not self.target_version: - self.target_version = "" - - if not self.skip_build and self.distribution.has_ext_modules(): - short_version = get_python_version() - if self.target_version and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" \ - " option must be specified" % (short_version,)) - self.target_version = short_version - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - - def run(self): - if (sys.platform != "win32" and - (self.distribution.has_ext_modules() or - self.distribution.has_c_libraries())): - raise DistutilsPlatformError \ - ("distribution contains extensions and/or C libraries; " - "must be compiled on a Windows 32 platform") - - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - install.plat_name = self.plat_name - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - # Use a custom scheme for the zip-file, because we have to decide - # at installation time which scheme to use. - for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): - value = key.upper() - if key == 'headers': - value = value + '/Include/$dist_name' - setattr(install, - 'install_' + key, - value) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - # And make an archive relative to the root of the - # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() - fullname = self.distribution.get_fullname() - arcname = self.make_archive(archive_basename, "zip", - root_dir=self.bdist_dir) - # create an exe containing the zip-file - self.create_exe(arcname, fullname, self.bitmap) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_wininst', pyversion, - self.get_installer_filename(fullname))) - # remove the zip-file again - log.debug("removing temporary file '%s'", arcname) - os.remove(arcname) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def get_inidata(self): - # Return data describing the installation. - lines = [] - metadata = self.distribution.metadata - - # Write the [metadata] section. - lines.append("[metadata]") - - # 'info' will be displayed in the installer's dialog box, - # describing the items to be installed. - info = (metadata.long_description or '') + '\n' - - # Escape newline characters - def escape(s): - return s.replace("\n", "\\n") - - for name in ["author", "author_email", "description", "maintainer", - "maintainer_email", "name", "url", "version"]: - data = getattr(metadata, name, "") - if data: - info = info + ("\n %s: %s" % \ - (name.capitalize(), escape(data))) - lines.append("%s=%s" % (name, escape(data))) - - # The [setup] section contains entries controlling - # the installer runtime. - lines.append("\n[Setup]") - if self.install_script: - lines.append("install_script=%s" % self.install_script) - lines.append("info=%s" % escape(info)) - lines.append("target_compile=%d" % (not self.no_target_compile)) - lines.append("target_optimize=%d" % (not self.no_target_optimize)) - if self.target_version: - lines.append("target_version=%s" % self.target_version) - if self.user_access_control: - lines.append("user_access_control=%s" % self.user_access_control) - - title = self.title or self.distribution.get_fullname() - lines.append("title=%s" % escape(title)) - import time - import distutils - build_info = "Built %s with distutils-%s" % \ - (time.ctime(time.time()), distutils.__version__) - lines.append("build_info=%s" % build_info) - return "\n".join(lines) - - def create_exe(self, arcname, fullname, bitmap=None): - import struct - - self.mkpath(self.dist_dir) - - cfgdata = self.get_inidata() - - installer_name = self.get_installer_filename(fullname) - self.announce("creating %s" % installer_name) - - if bitmap: - with open(bitmap, "rb") as f: - bitmapdata = f.read() - bitmaplen = len(bitmapdata) - else: - bitmaplen = 0 - - with open(installer_name, "wb") as file: - file.write(self.get_exe_bytes()) - if bitmap: - file.write(bitmapdata) - - # Convert cfgdata from unicode to ascii, mbcs encoded - if isinstance(cfgdata, str): - cfgdata = cfgdata.encode("mbcs") - - # Append the pre-install script - cfgdata = cfgdata + b"\0" - if self.pre_install_script: - # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin-1" simply avoids any possible - # failures. - with open(self.pre_install_script, "r", - encoding="latin-1") as script: - script_data = script.read().encode("latin-1") - cfgdata = cfgdata + script_data + b"\n\0" - else: - # empty pre-install script - cfgdata = cfgdata + b"\0" - file.write(cfgdata) - - # The 'magic number' 0x1234567B is used to make sure that the - # binary layout of 'cfgdata' is what the wininst.exe binary - # expects. If the layout changes, increment that number, make - # the corresponding changes to the wininst.exe sources, and - # recompile them. - header = struct.pack("", value) + value = value.replace("$installed_base", "$base") + value = value.replace("$py_version_nodot_plat", "$py_version_nodot") + if key == "headers": + value += "/$dist_name" + if sys.version_info >= (3, 9) and key == "platlib": + # platlibdir is available since 3.9: bpo-1294959 + value = value.replace("/lib/", "/$platlibdir/") + INSTALL_SCHEMES[main_key][key] = value + +# The following part of INSTALL_SCHEMES has a different definition +# than the one in sysconfig, but because both depend on the site module, +# the outcomes should be the same. if HAS_USER_SITE: INSTALL_SCHEMES['nt_user'] = { 'purelib': '$usersite', @@ -64,11 +86,6 @@ if HAS_USER_SITE: 'data' : '$userbase', } -# The keys to an installation scheme; if any new types of files are to be -# installed, be sure to add an entry to every installation scheme above, -# and to SCHEME_KEYS here. -SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') - class install(Command): @@ -169,8 +186,9 @@ class install(Command): self.install_lib = None # set to either purelib or platlib self.install_scripts = None self.install_data = None - self.install_userbase = USER_BASE - self.install_usersite = USER_SITE + if HAS_USER_SITE: + self.install_userbase = USER_BASE + self.install_usersite = USER_SITE self.compile = None self.optimize = None @@ -305,6 +323,9 @@ class install(Command): self.config_vars['userbase'] = self.install_userbase self.config_vars['usersite'] = self.install_usersite + if sysconfig.is_python_build(True): + self.config_vars['srcdir'] = sysconfig.get_config_var('srcdir') + self.expand_basedirs() self.dump_dirs("post-expand_basedirs()") @@ -343,8 +364,9 @@ class install(Command): # Convert directories from Unix /-separated syntax to the local # convention. self.convert_paths('lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers', - 'userbase', 'usersite') + 'scripts', 'data', 'headers') + if HAS_USER_SITE: + self.convert_paths('userbase', 'usersite') # Deprecated # Well, we're not actually fully completely finalized yet: we still diff --git a/contrib/tools/python3/src/Lib/distutils/extension.py b/contrib/tools/python3/src/Lib/distutils/extension.py index c507da360aa..e85032ece89 100644 --- a/contrib/tools/python3/src/Lib/distutils/extension.py +++ b/contrib/tools/python3/src/Lib/distutils/extension.py @@ -4,6 +4,7 @@ Provides the Extension class, used to describe C/C++ extension modules in setup scripts.""" import os +import re import warnings # This class is really only used by the "build_ext" command, so it might @@ -161,7 +162,7 @@ def read_setup_file(filename): line = file.readline() if line is None: # eof break - if _variable_rx.match(line): # VAR=VALUE, handled in first pass + if re.match(_variable_rx, line): # VAR=VALUE, handled in first pass continue if line[0] == line[-1] == "*": diff --git a/contrib/tools/python3/src/Lib/distutils/sysconfig.py b/contrib/tools/python3/src/Lib/distutils/sysconfig.py index 4e5464705b8..3414a761e76 100644 --- a/contrib/tools/python3/src/Lib/distutils/sysconfig.py +++ b/contrib/tools/python3/src/Lib/distutils/sysconfig.py @@ -13,298 +13,69 @@ import _imp import os import re import sys +import warnings -from .errors import DistutilsPlatformError - -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - -# Path to the base directory of the project. On Windows the binary may -# live in project/PCbuild/win32 or project/PCbuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - if sys.executable: - project_base = os.path.dirname(os.path.abspath(sys.executable)) - else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - project_base = os.getcwd() - - -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(PREFIX, "PCbuild"))): - return PREFIX - return d - project_base = _fix_pcbuild(project_base) - _sys_home = _fix_pcbuild(_sys_home) - -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) - -python_build = _python_build() - - -# Calculate the build qualifier flags if they are defined. Adding the flags -# to the include and lib directories only makes sense for an installation, not -# an in-source build. -build_flags = '' -try: - if not python_build: - build_flags = sys.abiflags -except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass - -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. - """ - return '%d.%d' % sys.version_info[:2] - - -def get_python_inc(plat_specific=0, prefix=None): - """Return the directory containing installed Python header files. - - If 'plat_specific' is false (the default), this is the path to the - non-platform-specific header files, i.e. Python.h and so on; - otherwise, this is the path to platform-specific header files - (namely pyconfig.h). - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if os.name == "posix": - if python_build: - # Assume the executable is in the build directory. The - # pyconfig.h file should be in the same directory. Since - # the build directory may not be the source directory, we - # must use "srcdir" from the makefile to find the "Include" - # directory. - if plat_specific: - return _sys_home or project_base - else: - incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) - python_dir = 'python' + get_python_version() + build_flags - return os.path.join(prefix, "include", python_dir) - elif os.name == "nt": - if python_build: - # Include both the include and PC dir to ensure we can find - # pyconfig.h - return (os.path.join(prefix, "include") + os.path.pathsep + - os.path.join(prefix, "PC")) - return os.path.join(prefix, "include") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its C header files " - "on platform '%s'" % os.name) - - -def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - """Return the directory containing the Python library (standard or - site additions). - - If 'plat_specific' is true, return the directory containing - platform-specific modules, i.e. any module from a non-pure-Python - module distribution; otherwise, return the platform-shared library - directory. If 'standard_lib' is true, return the directory - containing standard Python library modules; otherwise, return the - directory for site-specific modules. - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - if standard_lib: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - else: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - if plat_specific or standard_lib: - # Platform-specific modules (any module from a non-pure-Python - # module distribution) or standard Python library modules. - libdir = sys.platlibdir - else: - # Pure Python - libdir = "lib" - libpython = os.path.join(prefix, libdir, - "python" + get_python_version()) - if standard_lib: - return libpython - else: - return os.path.join(libpython, "site-packages") - elif os.name == "nt": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - return os.path.join(prefix, "Lib", "site-packages") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its library " - "on platform '%s'" % os.name) - - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - - (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - - if 'CC' in os.environ: - newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - ldshared = newcc + ldshared[len(cc):] - cc = newcc - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = cflags + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - if 'AR' in os.environ: - ar = os.environ['AR'] - if 'ARFLAGS' in os.environ: - archiver = ar + ' ' + os.environ['ARFLAGS'] - else: - archiver = ar + ' ' + ar_flags - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc, - archiver=archiver) - - compiler.shared_lib_extension = shlib_suffix - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") - else: - inc_dir = _sys_home or project_base - else: - inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(_sys_home or project_base, "Makefile") - lib_dir = get_python_lib(plat_specific=0, standard_lib=1) - config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch - return os.path.join(lib_dir, config_file, 'Makefile') +from functools import partial +from .errors import DistutilsPlatformError +from sysconfig import ( + _PREFIX as PREFIX, + _BASE_PREFIX as BASE_PREFIX, + _EXEC_PREFIX as EXEC_PREFIX, + _BASE_EXEC_PREFIX as BASE_EXEC_PREFIX, + _PROJECT_BASE as project_base, + _PYTHON_BUILD as python_build, + _init_posix as sysconfig_init_posix, + parse_config_h as sysconfig_parse_config_h, + + _init_non_posix, + _is_python_source_dir, + _sys_home, + + _variable_rx, + _findvar1_rx, + _findvar2_rx, + + expand_makefile_vars, + is_python_build, + get_config_h_filename, + get_config_var, + get_config_vars, + get_makefile_filename, + get_python_version, +) + +# This is better than +# from sysconfig import _CONFIG_VARS as _config_vars +# because it makes sure that the global dictionary is initialized +# which might not be true in the time of import. +_config_vars = get_config_vars() + +if os.name == "nt": + from sysconfig import _fix_pcbuild + +warnings.warn( + 'The distutils.sysconfig module is deprecated, use sysconfig instead', + DeprecationWarning, + stacklevel=2 +) + + +# Following functions are the same as in sysconfig but with different API def parse_config_h(fp, g=None): - """Parse a config.h-style file. + return sysconfig_parse_config_h(fp, vars=g) - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g +_python_build = partial(is_python_build, check_home=True) +_init_posix = partial(sysconfig_init_posix, _config_vars) +_init_nt = partial(_init_non_posix, _config_vars) -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") +# Similar function is also implemented in sysconfig as _parse_makefile +# but without the parsing capabilities of distutils.text_file.TextFile. def parse_makefile(fn, g=None): """Parse a Makefile-style file. - A dictionary containing name/value pairs is returned. If an optional dictionary is passed in as the second argument, it is used instead of a new dictionary. @@ -321,7 +92,7 @@ def parse_makefile(fn, g=None): line = fp.readline() if line is None: # eof break - m = _variable_rx.match(line) + m = re.match(_variable_rx, line) if m: n, v = m.group(1, 2) v = v.strip() @@ -349,7 +120,7 @@ def parse_makefile(fn, g=None): while notdone: for name in list(notdone): value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) + m = re.search(_findvar1_rx, value) or re.search(_findvar2_rx, value) if m: n = m.group(1) found = True @@ -408,148 +179,173 @@ def parse_makefile(fn, g=None): return g -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. +# Following functions are deprecated together with this module and they +# have no direct replacement + +# Calculate the build qualifier flags if they are defined. Adding the flags +# to the include and lib directories only makes sense for an installation, not +# an in-source build. +build_flags = '' +try: + if not python_build: + build_flags = sys.abiflags +except AttributeError: + # It's not a configure-based build, so the sys module doesn't have + # this attribute, which is fine. + pass + + +def customize_compiler(compiler): + """Do any platform-specific customization of a CCompiler instance. + + Mainly needed on Unix, so we can plug in the information that + varies across Unices and is stored in Python's Makefile. """ + if compiler.compiler_type == "unix": + if sys.platform == "darwin": + # Perform first-time customization of compiler-related + # config vars on OS X now that we know we need a compiler. + # This is primarily to support Pythons from binary + # installers. The kind and paths to build tools on + # the user system may vary significantly from the system + # that Python itself was built on. Also the user OS + # version and build tools may not support the same set + # of CPU architectures for universal builds. + if not _config_vars.get('CUSTOMIZED_OSX_COMPILER'): + import _osx_support + _osx_support.customize_compiler(_config_vars) + _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. + (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ + get_config_vars('CC', 'CXX', 'CFLAGS', + 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - while True: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] + if 'CC' in os.environ: + newcc = os.environ['CC'] + if (sys.platform == 'darwin' + and 'LDSHARED' not in os.environ + and ldshared.startswith(cc)): + # On OS X, if CC is overridden, use that as the default + # command for LDSHARED as well + ldshared = newcc + ldshared[len(cc):] + cc = newcc + if 'CXX' in os.environ: + cxx = os.environ['CXX'] + if 'LDSHARED' in os.environ: + ldshared = os.environ['LDSHARED'] + if 'CPP' in os.environ: + cpp = os.environ['CPP'] else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - #name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - # '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - # abi=sys.abiflags, - # platform=sys.platform, - # multiarch=getattr(sys.implementation, '_multiarch', ''), - #)) - #_temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - #build_time_vars = _temp.build_time_vars - global _config_vars - _config_vars = {'SO': '.so'} - #_config_vars.update(build_time_vars) - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() + cpp = cc + " -E" # not always + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: + cflags = cflags + ' ' + os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] + cflags = cflags + ' ' + os.environ['CPPFLAGS'] + ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] + if 'AR' in os.environ: + ar = os.environ['AR'] + if 'ARFLAGS' in os.environ: + archiver = ar + ' ' + os.environ['ARFLAGS'] else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) + archiver = ar + ' ' + ar_flags + + cc_cmd = cc + ' ' + cflags + compiler.set_executables( + preprocessor=cpp, + compiler=cc_cmd, + compiler_so=cc_cmd + ' ' + ccshared, + compiler_cxx=cxx, + linker_so=ldshared, + linker_exe=cc, + archiver=archiver) + + compiler.shared_lib_extension = shlib_suffix + + +def get_python_inc(plat_specific=0, prefix=None): + """Return the directory containing installed Python header files. + + If 'plat_specific' is false (the default), this is the path to the + non-platform-specific header files, i.e. Python.h and so on; + otherwise, this is the path to platform-specific header files + (namely pyconfig.h). + + If 'prefix' is supplied, use it instead of sys.base_prefix or + sys.base_exec_prefix -- i.e., ignore 'plat_specific'. + """ + if prefix is None: + prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX + if os.name == "posix": + if python_build: + # Assume the executable is in the build directory. The + # pyconfig.h file should be in the same directory. Since + # the build directory may not be the source directory, we + # must use "srcdir" from the makefile to find the "Include" + # directory. + if plat_specific: + return _sys_home or project_base else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - - # Convert srcdir into an absolute path if it appears necessary. - # Normally it is relative to the build directory. However, during - # testing, for example, we might be running a non-installed python - # from a different directory. - if python_build and os.name == "posix": - base = project_base - if (not os.path.isabs(_config_vars['srcdir']) and - base != os.getcwd()): - # srcdir is relative and we are not in the same directory - # as the executable. Assume executable is in the build - # directory and make srcdir absolute. - srcdir = os.path.join(base, _config_vars['srcdir']) - _config_vars['srcdir'] = os.path.normpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals + incdir = os.path.join(get_config_var('srcdir'), 'Include') + return os.path.normpath(incdir) + python_dir = 'python' + get_python_version() + build_flags + return os.path.join(prefix, "include", python_dir) + elif os.name == "nt": + if python_build: + # Include both the include and PC dir to ensure we can find + # pyconfig.h + return (os.path.join(prefix, "include") + os.path.pathsep + + os.path.join(prefix, "PC")) + return os.path.join(prefix, "include") else: - return _config_vars + raise DistutilsPlatformError( + "I don't know where Python installs its C header files " + "on platform '%s'" % os.name) + + +def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): + """Return the directory containing the Python library (standard or + site additions). -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) + If 'plat_specific' is true, return the directory containing + platform-specific modules, i.e. any module from a non-pure-Python + module distribution; otherwise, return the platform-shared library + directory. If 'standard_lib' is true, return the directory + containing standard Python library modules; otherwise, return the + directory for site-specific modules. + + If 'prefix' is supplied, use it instead of sys.base_prefix or + sys.base_exec_prefix -- i.e., ignore 'plat_specific'. """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) - return get_config_vars().get(name) + if prefix is None: + if standard_lib: + prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX + else: + prefix = plat_specific and EXEC_PREFIX or PREFIX + + if os.name == "posix": + if plat_specific or standard_lib: + # Platform-specific modules (any module from a non-pure-Python + # module distribution) or standard Python library modules. + libdir = sys.platlibdir + else: + # Pure Python + libdir = "lib" + libpython = os.path.join(prefix, libdir, + "python" + get_python_version()) + if standard_lib: + return libpython + else: + return os.path.join(libpython, "site-packages") + elif os.name == "nt": + if standard_lib: + return os.path.join(prefix, "Lib") + else: + return os.path.join(prefix, "Lib", "site-packages") + else: + raise DistutilsPlatformError( + "I don't know where Python installs its library " + "on platform '%s'" % os.name) diff --git a/contrib/tools/python3/src/Lib/distutils/util.py b/contrib/tools/python3/src/Lib/distutils/util.py index 4b002ecef1d..2ce5c5b64d6 100644 --- a/contrib/tools/python3/src/Lib/distutils/util.py +++ b/contrib/tools/python3/src/Lib/distutils/util.py @@ -9,6 +9,7 @@ import re import importlib.util import string import sys +import distutils from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -419,8 +420,10 @@ byte_compile(files, optimize=%r, force=%r, direct=1) """ % (optimize, force, prefix, base_dir, verbose)) + msg = distutils._DEPRECATION_MESSAGE cmd = [sys.executable] cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.append(f'-Wignore:{msg}:DeprecationWarning') cmd.append(script_name) spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, diff --git a/contrib/tools/python3/src/Lib/doctest.py b/contrib/tools/python3/src/Lib/doctest.py index 54ae93bcf19..80a14bccfbf 100644 --- a/contrib/tools/python3/src/Lib/doctest.py +++ b/contrib/tools/python3/src/Lib/doctest.py @@ -102,7 +102,7 @@ import re import sys import traceback import unittest -from io import StringIO +from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple TestResults = namedtuple('TestResults', 'failed attempted') @@ -212,23 +212,24 @@ def _normalize_module(module, depth=2): raise TypeError("Expected a module, string, or None") def _newline_convert(data): - # We have two cases to cover and we need to make sure we do - # them in the right order - for newline in ('\r\n', '\r'): - data = data.replace(newline, '\n') - return data + # The IO module provides a handy decoder for universal newline conversion + return IncrementalNewlineDecoder(None, True).decode(data, True) def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) - if getattr(package, '__loader__', None) is not None: - if hasattr(package.__loader__, 'get_data'): - file_contents = package.__loader__.get_data(filename) - file_contents = file_contents.decode(encoding) - # get_data() opens files as 'rb', so one must do the equivalent - # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + if (loader := getattr(package, '__loader__', None)) is None: + try: + loader = package.__spec__.loader + except AttributeError: + pass + if hasattr(loader, 'get_data'): + file_contents = loader.get_data(filename) + file_contents = file_contents.decode(encoding) + # get_data() opens files as 'rb', so one must do the equivalent + # conversion as universal newlines would do. + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -975,6 +976,17 @@ class DocTestFinder: else: raise ValueError("object must be a class or function") + def _is_routine(self, obj): + """ + Safely unwrap objects and determine if they are functions. + """ + maybe_routine = obj + try: + maybe_routine = inspect.unwrap(maybe_routine) + except ValueError: + pass + return inspect.isroutine(maybe_routine) + def _find(self, tests, obj, name, module, source_lines, globs, seen): """ Find tests for the given object and any contained objects, and @@ -997,9 +1009,9 @@ class DocTestFinder: if inspect.ismodule(obj) and self._recurse: for valname, val in obj.__dict__.items(): valname = '%s.%s' % (name, valname) + # Recurse to functions & classes. - if ((inspect.isroutine(inspect.unwrap(val)) - or inspect.isclass(val)) and + if ((self._is_routine(val) or inspect.isclass(val)) and self._from_module(module, val)): self._find(tests, val, valname, module, source_lines, globs, seen) diff --git a/contrib/tools/python3/src/Lib/email/_parseaddr.py b/contrib/tools/python3/src/Lib/email/_parseaddr.py index c5a7b23193e..ba5ad5a36d0 100644 --- a/contrib/tools/python3/src/Lib/email/_parseaddr.py +++ b/contrib/tools/python3/src/Lib/email/_parseaddr.py @@ -65,7 +65,7 @@ def _parsedate_tz(data): """ if not data: - return + return None data = data.split() if not data: # This happens for whitespace-only input. return None diff --git a/contrib/tools/python3/src/Lib/email/base64mime.py b/contrib/tools/python3/src/Lib/email/base64mime.py index 17f0818f6ca..a7cc37365c6 100644 --- a/contrib/tools/python3/src/Lib/email/base64mime.py +++ b/contrib/tools/python3/src/Lib/email/base64mime.py @@ -84,7 +84,7 @@ def body_encode(s, maxlinelen=76, eol=NL): in an email. """ if not s: - return s + return "" encvec = [] max_unencoded = maxlinelen * 3 // 4 diff --git a/contrib/tools/python3/src/Lib/email/errors.py b/contrib/tools/python3/src/Lib/email/errors.py index d28a6800104..3ad00565549 100644 --- a/contrib/tools/python3/src/Lib/email/errors.py +++ b/contrib/tools/python3/src/Lib/email/errors.py @@ -108,3 +108,6 @@ class NonASCIILocalPartDefect(HeaderDefect): """local_part contains non-ASCII characters""" # This defect only occurs during unicode parsing, not when # parsing messages decoded from binary. + +class InvalidDateDefect(HeaderDefect): + """Header has unparsable or invalid date""" diff --git a/contrib/tools/python3/src/Lib/email/headerregistry.py b/contrib/tools/python3/src/Lib/email/headerregistry.py index 5d84fc0d82d..b590d69e8b7 100644 --- a/contrib/tools/python3/src/Lib/email/headerregistry.py +++ b/contrib/tools/python3/src/Lib/email/headerregistry.py @@ -2,10 +2,6 @@ This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. - -Eventually HeaderRegistry will be a public API, but it isn't yet, -and will probably change some before that happens. - """ from types import MappingProxyType @@ -302,7 +298,14 @@ class DateHeader: kwds['parse_tree'] = parser.TokenList() return if isinstance(value, str): - value = utils.parsedate_to_datetime(value) + kwds['decoded'] = value + try: + value = utils.parsedate_to_datetime(value) + except ValueError: + kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format')) + kwds['datetime'] = None + kwds['parse_tree'] = parser.TokenList() + return kwds['datetime'] = value kwds['decoded'] = utils.format_datetime(kwds['datetime']) kwds['parse_tree'] = cls.value_parser(kwds['decoded']) diff --git a/contrib/tools/python3/src/Lib/email/utils.py b/contrib/tools/python3/src/Lib/email/utils.py index 48d30160aa6..cfdfeb3f1a8 100644 --- a/contrib/tools/python3/src/Lib/email/utils.py +++ b/contrib/tools/python3/src/Lib/email/utils.py @@ -195,7 +195,10 @@ def make_msgid(idstring=None, domain=None): def parsedate_to_datetime(data): - *dtuple, tz = _parsedate_tz(data) + parsed_date_tz = _parsedate_tz(data) + if parsed_date_tz is None: + raise ValueError('Invalid date value or format "%s"' % str(data)) + *dtuple, tz = parsed_date_tz if tz is None: return datetime.datetime(*dtuple[:6]) return datetime.datetime(*dtuple[:6], diff --git a/contrib/tools/python3/src/Lib/encodings/__init__.py b/contrib/tools/python3/src/Lib/encodings/__init__.py index ddd5afdcf2d..4b37d3321c9 100644 --- a/contrib/tools/python3/src/Lib/encodings/__init__.py +++ b/contrib/tools/python3/src/Lib/encodings/__init__.py @@ -61,7 +61,8 @@ def normalize_encoding(encoding): if c.isalnum() or c == '.': if punct and chars: chars.append('_') - chars.append(c) + if c.isascii(): + chars.append(c) punct = False else: punct = True diff --git a/contrib/tools/python3/src/Lib/ensurepip/__init__.py b/contrib/tools/python3/src/Lib/ensurepip/__init__.py index e510cc7fb2b..3fbe8b2a5b1 100644 --- a/contrib/tools/python3/src/Lib/ensurepip/__init__.py +++ b/contrib/tools/python3/src/Lib/ensurepip/__init__.py @@ -1,16 +1,16 @@ +import collections import os import os.path +import subprocess import sys -import runpy +import sysconfig import tempfile -import subprocess from importlib import resources -from . import _bundled - __all__ = ["version", "bootstrap"] +_PACKAGE_NAMES = ('setuptools', 'pip') _SETUPTOOLS_VERSION = "58.1.0" _PIP_VERSION = "22.0.4" _PROJECTS = [ @@ -18,6 +18,65 @@ _PROJECTS = [ ("pip", _PIP_VERSION, "py3"), ] +# Packages bundled in ensurepip._bundled have wheel_name set. +# Packages from WHEEL_PKG_DIR have wheel_path set. +_Package = collections.namedtuple('Package', + ('version', 'wheel_name', 'wheel_path')) + +# Directory of system wheel packages. Some Linux distribution packaging +# policies recommend against bundling dependencies. For example, Fedora +# installs wheel packages in the /usr/share/python-wheels/ directory and don't +# install the ensurepip._bundled package. +_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') + + +def _find_packages(path): + packages = {} + try: + filenames = os.listdir(path) + except OSError: + # Ignore: path doesn't exist or permission error + filenames = () + # Make the code deterministic if a directory contains multiple wheel files + # of the same package, but don't attempt to implement correct version + # comparison since this case should not happen. + filenames = sorted(filenames) + for filename in filenames: + # filename is like 'pip-21.2.4-py3-none-any.whl' + if not filename.endswith(".whl"): + continue + for name in _PACKAGE_NAMES: + prefix = name + '-' + if filename.startswith(prefix): + break + else: + continue + + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + version = filename.removeprefix(prefix).partition('-')[0] + wheel_path = os.path.join(path, filename) + packages[name] = _Package(version, None, wheel_path) + return packages + + +def _get_packages(): + global _PACKAGES, _WHEEL_PKG_DIR + if _PACKAGES is not None: + return _PACKAGES + + packages = {} + for name, version, py_tag in _PROJECTS: + wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" + packages[name] = _Package(version, wheel_name, None) + if _WHEEL_PKG_DIR: + dir_packages = _find_packages(_WHEEL_PKG_DIR) + # only used the wheel package directory if all packages are found there + if all(name in dir_packages for name in _PACKAGE_NAMES): + packages = dir_packages + _PACKAGES = packages + return packages +_PACKAGES = None + def _run_pip(args, additional_paths=None): # Run the bootstraping in a subprocess to avoid leaking any state that happens @@ -31,14 +90,16 @@ sys.path = {additional_paths or []} + sys.path sys.argv[1:] = {args} runpy.run_module("pip", run_name="__main__", alter_sys=True) """ - return subprocess.run([sys.executable, "-c", code], check=True).returncode + return subprocess.run([sys.executable, '-W', 'ignore::DeprecationWarning', + "-c", code], check=True).returncode def version(): """ Returns a string specifying the bundled version of pip. """ - return _PIP_VERSION + return _get_packages()['pip'].version + def _disable_pip_configuration_settings(): # We deliberately ignore all pip environment variables @@ -100,16 +161,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version, py_tag in _PROJECTS: - wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) - whl = resources.read_binary( - _bundled, - wheel_name, - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + for name, package in _get_packages().items(): + if package.wheel_name: + # Use bundled wheel package + from ensurepip import _bundled + wheel_name = package.wheel_name + whl = resources.read_binary(_bundled, wheel_name) + else: + # Use the wheel package directory + with open(package.wheel_path, "rb") as fp: + whl = fp.read() + wheel_name = os.path.basename(package.wheel_path) + + filename = os.path.join(tmpdir, wheel_name) + with open(filename, "wb") as fp: fp.write(whl) - additional_paths.append(os.path.join(tmpdir, wheel_name)) + additional_paths.append(filename) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -122,7 +190,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -135,11 +203,14 @@ def _uninstall_helper(*, verbosity=0): except ImportError: return - # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + # If the installed pip version doesn't match the available one, + # leave it alone + available_version = version() + if pip.__version__ != available_version: + print(f"ensurepip will only uninstall a matching version " + f"({pip.__version__!r} installed, " + f"{available_version!r} available)", + file=sys.stderr) return _disable_pip_configuration_settings() @@ -149,7 +220,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) def _main(argv=None): diff --git a/contrib/tools/python3/src/Lib/enum.py b/contrib/tools/python3/src/Lib/enum.py index ee4c4c04f98..f5657a6eba2 100644 --- a/contrib/tools/python3/src/Lib/enum.py +++ b/contrib/tools/python3/src/Lib/enum.py @@ -44,10 +44,11 @@ def _is_sunder(name): def _is_private(cls_name, name): # do not use `re` as `re` imports `enum` pattern = '_%s__' % (cls_name, ) + pat_len = len(pattern) if ( - len(name) >= 5 + len(name) > pat_len and name.startswith(pattern) - and name[len(pattern)] != '_' + and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -97,7 +98,7 @@ class _EnumDict(dict): if _is_private(self._cls_name, key): import warnings warnings.warn( - "private variables, such as %r, will be normal attributes in 3.10" + "private variables, such as %r, will be normal attributes in 3.11" % (key, ), DeprecationWarning, stacklevel=2, @@ -392,12 +393,19 @@ class EnumMeta(type): start=start, ) - def __contains__(cls, member): - if not isinstance(member, Enum): + def __contains__(cls, obj): + if not isinstance(obj, Enum): + import warnings + warnings.warn( + "in 3.12 __contains__ will no longer raise TypeError, but will return True if\n" + "obj is a member or a member's value", + DeprecationWarning, + stacklevel=2, + ) raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( - type(member).__qualname__, cls.__class__.__qualname__)) - return isinstance(member, cls) and member._name_ in cls._member_map_ + type(obj).__qualname__, cls.__class__.__qualname__)) + return isinstance(obj, cls) and obj._name_ in cls._member_map_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -705,7 +713,8 @@ class Enum(metaclass=EnumMeta): 'error in %s._missing_: returned %r instead of None or a valid member' % (cls.__name__, result) ) - exc.__context__ = ve_exc + if not isinstance(exc, ValueError): + exc.__context__ = ve_exc raise exc finally: # ensure all variables that could hold an exception are destroyed diff --git a/contrib/tools/python3/src/Lib/filecmp.py b/contrib/tools/python3/src/Lib/filecmp.py index 1893f909de8..70a4b23c982 100644 --- a/contrib/tools/python3/src/Lib/filecmp.py +++ b/contrib/tools/python3/src/Lib/filecmp.py @@ -116,7 +116,9 @@ class dircmp: same_files: list of identical files. diff_files: list of filenames which differ. funny_files: list of files which could not be compared. - subdirs: a dictionary of dircmp objects, keyed by names in common_dirs. + subdirs: a dictionary of dircmp instances (or MyDirCmp instances if this + object is of type MyDirCmp, a subclass of dircmp), keyed by names + in common_dirs. """ def __init__(self, a, b, ignore=None, hide=None): # Initialize @@ -186,14 +188,15 @@ class dircmp: self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories - # A new dircmp object is created for each common subdirectory, + # A new dircmp (or MyDirCmp if dircmp was subclassed) object is created + # for each common subdirectory, # these are stored in a dictionary indexed by filename. # The hide and ignore properties are inherited from the parent self.subdirs = {} for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() diff --git a/contrib/tools/python3/src/Lib/fileinput.py b/contrib/tools/python3/src/Lib/fileinput.py index 0c31f93ed8f..35347185da0 100644 --- a/contrib/tools/python3/src/Lib/fileinput.py +++ b/contrib/tools/python3/src/Lib/fileinput.py @@ -3,7 +3,7 @@ Typical use is: import fileinput - for line in fileinput.input(): + for line in fileinput.input(encoding="utf-8"): process(line) This iterates over the lines of all files listed in sys.argv[1:], @@ -63,15 +63,9 @@ file remains around; by default, the extension is ".bak" and it is deleted when the output file is closed. In-place filtering is disabled when standard input is read. XXX The current implementation does not work for MS-DOS 8+3 filesystems. - -XXX Possible additions: - -- optional getopt argument processing -- isatty() -- read(), read(size), even readlines() - """ +import io import sys, os from types import GenericAlias @@ -81,7 +75,8 @@ __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", _state = None -def input(files=None, inplace=False, backup="", *, mode="r", openhook=None): +def input(files=None, inplace=False, backup="", *, mode="r", openhook=None, + encoding=None, errors=None): """Return an instance of the FileInput class, which can be iterated. The parameters are passed to the constructor of the FileInput class. @@ -91,7 +86,8 @@ def input(files=None, inplace=False, backup="", *, mode="r", openhook=None): global _state if _state and _state._file: raise RuntimeError("input() already active") - _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook) + _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook, + encoding=encoding, errors=errors) return _state def close(): @@ -186,7 +182,7 @@ class FileInput: """ def __init__(self, files=None, inplace=False, backup="", *, - mode="r", openhook=None): + mode="r", openhook=None, encoding=None, errors=None): if isinstance(files, str): files = (files,) elif isinstance(files, os.PathLike): @@ -209,6 +205,17 @@ class FileInput: self._file = None self._isstdin = False self._backupfilename = None + self._encoding = encoding + self._errors = errors + + # We can not use io.text_encoding() here because old openhook doesn't + # take encoding parameter. + if (sys.flags.warn_default_encoding and + "b" not in mode and encoding is None and openhook is None): + import warnings + warnings.warn("'encoding' argument not specified.", + EncodingWarning, 2) + # restrict mode argument to reading modes if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " @@ -324,6 +331,13 @@ class FileInput: self._file = None self._isstdin = False self._backupfilename = 0 + + # EncodingWarning is emitted in __init__() already + if "b" not in self._mode: + encoding = self._encoding or "locale" + else: + encoding = None + if self._filename == '-': self._filename = '' if 'b' in self._mode: @@ -341,18 +355,18 @@ class FileInput: pass # The next few lines may raise OSError os.rename(self._filename, self._backupfilename) - self._file = open(self._backupfilename, self._mode) + self._file = open(self._backupfilename, self._mode, encoding=encoding) try: perm = os.fstat(self._file.fileno()).st_mode except OSError: - self._output = open(self._filename, self._write_mode) + self._output = open(self._filename, self._write_mode, encoding=encoding) else: mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC if hasattr(os, 'O_BINARY'): mode |= os.O_BINARY fd = os.open(self._filename, mode, perm) - self._output = os.fdopen(fd, self._write_mode) + self._output = os.fdopen(fd, self._write_mode, encoding=encoding) try: os.chmod(self._filename, perm) except OSError: @@ -362,9 +376,15 @@ class FileInput: else: # This may raise OSError if self._openhook: - self._file = self._openhook(self._filename, self._mode) + # Custom hooks made previous to Python 3.10 didn't have + # encoding argument + if self._encoding is None: + self._file = self._openhook(self._filename, self._mode) + else: + self._file = self._openhook( + self._filename, self._mode, encoding=self._encoding, errors=self._errors) else: - self._file = open(self._filename, self._mode) + self._file = open(self._filename, self._mode, encoding=encoding, errors=self._errors) self._readline = self._file.readline # hide FileInput._readline return self._readline() @@ -395,16 +415,23 @@ class FileInput: __class_getitem__ = classmethod(GenericAlias) -def hook_compressed(filename, mode): +def hook_compressed(filename, mode, *, encoding=None, errors=None): + if encoding is None: # EncodingWarning is emitted in FileInput() already. + encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': import gzip - return gzip.open(filename, mode) + stream = gzip.open(filename, mode) elif ext == '.bz2': import bz2 - return bz2.BZ2File(filename, mode) + stream = bz2.BZ2File(filename, mode) else: - return open(filename, mode) + return open(filename, mode, encoding=encoding, errors=errors) + + # gzip and bz2 are binary mode by default. + if "b" not in mode: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors) + return stream def hook_encoded(encoding, errors=None): diff --git a/contrib/tools/python3/src/Lib/formatter.py b/contrib/tools/python3/src/Lib/formatter.py deleted file mode 100644 index e2394de8c29..00000000000 --- a/contrib/tools/python3/src/Lib/formatter.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Generic output formatting. - -Formatter objects transform an abstract flow of formatting events into -specific output events on writer objects. Formatters manage several stack -structures to allow various properties of a writer object to be changed and -restored; writers need not be able to handle relative changes nor any sort -of ``change back'' operation. Specific writer properties which may be -controlled via formatter objects are horizontal alignment, font, and left -margin indentations. A mechanism is provided which supports providing -arbitrary, non-exclusive style settings to a writer as well. Additional -interfaces facilitate formatting events which are not reversible, such as -paragraph separation. - -Writer objects encapsulate device interfaces. Abstract devices, such as -file formats, are supported as well as physical devices. The provided -implementations all work with abstract devices. The interface makes -available mechanisms for setting the properties which formatter objects -manage and inserting data into the output. -""" - -import sys -import warnings -warnings.warn('the formatter module is deprecated', DeprecationWarning, - stacklevel=2) - - -AS_IS = None - - -class NullFormatter: - """A formatter which does nothing. - - If the writer parameter is omitted, a NullWriter instance is created. - No methods of the writer are called by NullFormatter instances. - - Implementations should inherit from this class if implementing a writer - interface but don't need to inherit any implementation. - - """ - - def __init__(self, writer=None): - if writer is None: - writer = NullWriter() - self.writer = writer - def end_paragraph(self, blankline): pass - def add_line_break(self): pass - def add_hor_rule(self, *args, **kw): pass - def add_label_data(self, format, counter, blankline=None): pass - def add_flowing_data(self, data): pass - def add_literal_data(self, data): pass - def flush_softspace(self): pass - def push_alignment(self, align): pass - def pop_alignment(self): pass - def push_font(self, x): pass - def pop_font(self): pass - def push_margin(self, margin): pass - def pop_margin(self): pass - def set_spacing(self, spacing): pass - def push_style(self, *styles): pass - def pop_style(self, n=1): pass - def assert_line_data(self, flag=1): pass - - -class AbstractFormatter: - """The standard formatter. - - This implementation has demonstrated wide applicability to many writers, - and may be used directly in most circumstances. It has been used to - implement a full-featured World Wide Web browser. - - """ - - # Space handling policy: blank spaces at the boundary between elements - # are handled by the outermost context. "Literal" data is not checked - # to determine context, so spaces in literal data are handled directly - # in all circumstances. - - def __init__(self, writer): - self.writer = writer # Output device - self.align = None # Current alignment - self.align_stack = [] # Alignment stack - self.font_stack = [] # Font state - self.margin_stack = [] # Margin state - self.spacing = None # Vertical spacing state - self.style_stack = [] # Other state, e.g. color - self.nospace = 1 # Should leading space be suppressed - self.softspace = 0 # Should a space be inserted - self.para_end = 1 # Just ended a paragraph - self.parskip = 0 # Skipped space between paragraphs? - self.hard_break = 1 # Have a hard break - self.have_label = 0 - - def end_paragraph(self, blankline): - if not self.hard_break: - self.writer.send_line_break() - self.have_label = 0 - if self.parskip < blankline and not self.have_label: - self.writer.send_paragraph(blankline - self.parskip) - self.parskip = blankline - self.have_label = 0 - self.hard_break = self.nospace = self.para_end = 1 - self.softspace = 0 - - def add_line_break(self): - if not (self.hard_break or self.para_end): - self.writer.send_line_break() - self.have_label = self.parskip = 0 - self.hard_break = self.nospace = 1 - self.softspace = 0 - - def add_hor_rule(self, *args, **kw): - if not self.hard_break: - self.writer.send_line_break() - self.writer.send_hor_rule(*args, **kw) - self.hard_break = self.nospace = 1 - self.have_label = self.para_end = self.softspace = self.parskip = 0 - - def add_label_data(self, format, counter, blankline = None): - if self.have_label or not self.hard_break: - self.writer.send_line_break() - if not self.para_end: - self.writer.send_paragraph((blankline and 1) or 0) - if isinstance(format, str): - self.writer.send_label_data(self.format_counter(format, counter)) - else: - self.writer.send_label_data(format) - self.nospace = self.have_label = self.hard_break = self.para_end = 1 - self.softspace = self.parskip = 0 - - def format_counter(self, format, counter): - label = '' - for c in format: - if c == '1': - label = label + ('%d' % counter) - elif c in 'aA': - if counter > 0: - label = label + self.format_letter(c, counter) - elif c in 'iI': - if counter > 0: - label = label + self.format_roman(c, counter) - else: - label = label + c - return label - - def format_letter(self, case, counter): - label = '' - while counter > 0: - counter, x = divmod(counter-1, 26) - # This makes a strong assumption that lowercase letters - # and uppercase letters form two contiguous blocks, with - # letters in order! - s = chr(ord(case) + x) - label = s + label - return label - - def format_roman(self, case, counter): - ones = ['i', 'x', 'c', 'm'] - fives = ['v', 'l', 'd'] - label, index = '', 0 - # This will die of IndexError when counter is too big - while counter > 0: - counter, x = divmod(counter, 10) - if x == 9: - label = ones[index] + ones[index+1] + label - elif x == 4: - label = ones[index] + fives[index] + label - else: - if x >= 5: - s = fives[index] - x = x-5 - else: - s = '' - s = s + ones[index]*x - label = s + label - index = index + 1 - if case == 'I': - return label.upper() - return label - - def add_flowing_data(self, data): - if not data: return - prespace = data[:1].isspace() - postspace = data[-1:].isspace() - data = " ".join(data.split()) - if self.nospace and not data: - return - elif prespace or self.softspace: - if not data: - if not self.nospace: - self.softspace = 1 - self.parskip = 0 - return - if not self.nospace: - data = ' ' + data - self.hard_break = self.nospace = self.para_end = \ - self.parskip = self.have_label = 0 - self.softspace = postspace - self.writer.send_flowing_data(data) - - def add_literal_data(self, data): - if not data: return - if self.softspace: - self.writer.send_flowing_data(" ") - self.hard_break = data[-1:] == '\n' - self.nospace = self.para_end = self.softspace = \ - self.parskip = self.have_label = 0 - self.writer.send_literal_data(data) - - def flush_softspace(self): - if self.softspace: - self.hard_break = self.para_end = self.parskip = \ - self.have_label = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - - def push_alignment(self, align): - if align and align != self.align: - self.writer.new_alignment(align) - self.align = align - self.align_stack.append(align) - else: - self.align_stack.append(self.align) - - def pop_alignment(self): - if self.align_stack: - del self.align_stack[-1] - if self.align_stack: - self.align = align = self.align_stack[-1] - self.writer.new_alignment(align) - else: - self.align = None - self.writer.new_alignment(None) - - def push_font(self, font): - size, i, b, tt = font - if self.softspace: - self.hard_break = self.para_end = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - if self.font_stack: - csize, ci, cb, ctt = self.font_stack[-1] - if size is AS_IS: size = csize - if i is AS_IS: i = ci - if b is AS_IS: b = cb - if tt is AS_IS: tt = ctt - font = (size, i, b, tt) - self.font_stack.append(font) - self.writer.new_font(font) - - def pop_font(self): - if self.font_stack: - del self.font_stack[-1] - if self.font_stack: - font = self.font_stack[-1] - else: - font = None - self.writer.new_font(font) - - def push_margin(self, margin): - self.margin_stack.append(margin) - fstack = [m for m in self.margin_stack if m] - if not margin and fstack: - margin = fstack[-1] - self.writer.new_margin(margin, len(fstack)) - - def pop_margin(self): - if self.margin_stack: - del self.margin_stack[-1] - fstack = [m for m in self.margin_stack if m] - if fstack: - margin = fstack[-1] - else: - margin = None - self.writer.new_margin(margin, len(fstack)) - - def set_spacing(self, spacing): - self.spacing = spacing - self.writer.new_spacing(spacing) - - def push_style(self, *styles): - if self.softspace: - self.hard_break = self.para_end = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - for style in styles: - self.style_stack.append(style) - self.writer.new_styles(tuple(self.style_stack)) - - def pop_style(self, n=1): - del self.style_stack[-n:] - self.writer.new_styles(tuple(self.style_stack)) - - def assert_line_data(self, flag=1): - self.nospace = self.hard_break = not flag - self.para_end = self.parskip = self.have_label = 0 - - -class NullWriter: - """Minimal writer interface to use in testing & inheritance. - - A writer which only provides the interface definition; no actions are - taken on any methods. This should be the base class for all writers - which do not need to inherit any implementation methods. - - """ - def __init__(self): pass - def flush(self): pass - def new_alignment(self, align): pass - def new_font(self, font): pass - def new_margin(self, margin, level): pass - def new_spacing(self, spacing): pass - def new_styles(self, styles): pass - def send_paragraph(self, blankline): pass - def send_line_break(self): pass - def send_hor_rule(self, *args, **kw): pass - def send_label_data(self, data): pass - def send_flowing_data(self, data): pass - def send_literal_data(self, data): pass - - -class AbstractWriter(NullWriter): - """A writer which can be used in debugging formatters, but not much else. - - Each method simply announces itself by printing its name and - arguments on standard output. - - """ - - def new_alignment(self, align): - print("new_alignment(%r)" % (align,)) - - def new_font(self, font): - print("new_font(%r)" % (font,)) - - def new_margin(self, margin, level): - print("new_margin(%r, %d)" % (margin, level)) - - def new_spacing(self, spacing): - print("new_spacing(%r)" % (spacing,)) - - def new_styles(self, styles): - print("new_styles(%r)" % (styles,)) - - def send_paragraph(self, blankline): - print("send_paragraph(%r)" % (blankline,)) - - def send_line_break(self): - print("send_line_break()") - - def send_hor_rule(self, *args, **kw): - print("send_hor_rule()") - - def send_label_data(self, data): - print("send_label_data(%r)" % (data,)) - - def send_flowing_data(self, data): - print("send_flowing_data(%r)" % (data,)) - - def send_literal_data(self, data): - print("send_literal_data(%r)" % (data,)) - - -class DumbWriter(NullWriter): - """Simple writer class which writes output on the file object passed in - as the file parameter or, if file is omitted, on standard output. The - output is simply word-wrapped to the number of columns specified by - the maxcol parameter. This class is suitable for reflowing a sequence - of paragraphs. - - """ - - def __init__(self, file=None, maxcol=72): - self.file = file or sys.stdout - self.maxcol = maxcol - NullWriter.__init__(self) - self.reset() - - def reset(self): - self.col = 0 - self.atbreak = 0 - - def send_paragraph(self, blankline): - self.file.write('\n'*blankline) - self.col = 0 - self.atbreak = 0 - - def send_line_break(self): - self.file.write('\n') - self.col = 0 - self.atbreak = 0 - - def send_hor_rule(self, *args, **kw): - self.file.write('\n') - self.file.write('-'*self.maxcol) - self.file.write('\n') - self.col = 0 - self.atbreak = 0 - - def send_literal_data(self, data): - self.file.write(data) - i = data.rfind('\n') - if i >= 0: - self.col = 0 - data = data[i+1:] - data = data.expandtabs() - self.col = self.col + len(data) - self.atbreak = 0 - - def send_flowing_data(self, data): - if not data: return - atbreak = self.atbreak or data[0].isspace() - col = self.col - maxcol = self.maxcol - write = self.file.write - for word in data.split(): - if atbreak: - if col + len(word) >= maxcol: - write('\n') - col = 0 - else: - write(' ') - col = col + 1 - write(word) - col = col + len(word) - atbreak = 1 - self.col = col - self.atbreak = data[-1].isspace() - - -def test(file = None): - w = DumbWriter() - f = AbstractFormatter(w) - if file is not None: - fp = open(file) - elif sys.argv[1:]: - fp = open(sys.argv[1]) - else: - fp = sys.stdin - try: - for line in fp: - if line == '\n': - f.end_paragraph(1) - else: - f.add_flowing_data(line) - finally: - if fp is not sys.stdin: - fp.close() - f.end_paragraph(0) - - -if __name__ == '__main__': - test() diff --git a/contrib/tools/python3/src/Lib/fractions.py b/contrib/tools/python3/src/Lib/fractions.py index de3e23b7592..96047beb454 100644 --- a/contrib/tools/python3/src/Lib/fractions.py +++ b/contrib/tools/python3/src/Lib/fractions.py @@ -380,32 +380,139 @@ class Fraction(numbers.Rational): return forward, reverse + # Rational arithmetic algorithms: Knuth, TAOCP, Volume 2, 4.5.1. + # + # Assume input fractions a and b are normalized. + # + # 1) Consider addition/subtraction. + # + # Let g = gcd(da, db). Then + # + # na nb na*db ± nb*da + # a ± b == -- ± -- == ------------- == + # da db da*db + # + # na*(db//g) ± nb*(da//g) t + # == ----------------------- == - + # (da*db)//g d + # + # Now, if g > 1, we're working with smaller integers. + # + # Note, that t, (da//g) and (db//g) are pairwise coprime. + # + # Indeed, (da//g) and (db//g) share no common factors (they were + # removed) and da is coprime with na (since input fractions are + # normalized), hence (da//g) and na are coprime. By symmetry, + # (db//g) and nb are coprime too. Then, + # + # gcd(t, da//g) == gcd(na*(db//g), da//g) == 1 + # gcd(t, db//g) == gcd(nb*(da//g), db//g) == 1 + # + # Above allows us optimize reduction of the result to lowest + # terms. Indeed, + # + # g2 = gcd(t, d) == gcd(t, (da//g)*(db//g)*g) == gcd(t, g) + # + # t//g2 t//g2 + # a ± b == ----------------------- == ---------------- + # (da//g)*(db//g)*(g//g2) (da//g)*(db//g2) + # + # is a normalized fraction. This is useful because the unnormalized + # denominator d could be much larger than g. + # + # We should special-case g == 1 (and g2 == 1), since 60.8% of + # randomly-chosen integers are coprime: + # https://en.wikipedia.org/wiki/Coprime_integers#Probability_of_coprimality + # Note, that g2 == 1 always for fractions, obtained from floats: here + # g is a power of 2 and the unnormalized numerator t is an odd integer. + # + # 2) Consider multiplication + # + # Let g1 = gcd(na, db) and g2 = gcd(nb, da), then + # + # na*nb na*nb (na//g1)*(nb//g2) + # a*b == ----- == ----- == ----------------- + # da*db db*da (db//g1)*(da//g2) + # + # Note, that after divisions we're multiplying smaller integers. + # + # Also, the resulting fraction is normalized, because each of + # two factors in the numerator is coprime to each of the two factors + # in the denominator. + # + # Indeed, pick (na//g1). It's coprime with (da//g2), because input + # fractions are normalized. It's also coprime with (db//g1), because + # common factors are removed by g1 == gcd(na, db). + # + # As for addition/subtraction, we should special-case g1 == 1 + # and g2 == 1 for same reason. That happens also for multiplying + # rationals, obtained from floats. + def _add(a, b): """a + b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db + b.numerator * da, - da * db) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g = math.gcd(da, db) + if g == 1: + return Fraction(na * db + da * nb, da * db, _normalize=False) + s = da // g + t = na * (db // g) + nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction(t, s * db, _normalize=False) + return Fraction(t // g2, s * (db // g2), _normalize=False) __add__, __radd__ = _operator_fallbacks(_add, operator.add) def _sub(a, b): """a - b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db - b.numerator * da, - da * db) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g = math.gcd(da, db) + if g == 1: + return Fraction(na * db - da * nb, da * db, _normalize=False) + s = da // g + t = na * (db // g) - nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction(t, s * db, _normalize=False) + return Fraction(t // g2, s * (db // g2), _normalize=False) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) def _mul(a, b): """a * b""" - return Fraction(a.numerator * b.numerator, a.denominator * b.denominator) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g1 = math.gcd(na, db) + if g1 > 1: + na //= g1 + db //= g1 + g2 = math.gcd(nb, da) + if g2 > 1: + nb //= g2 + da //= g2 + return Fraction(na * nb, db * da, _normalize=False) __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) def _div(a, b): """a / b""" - return Fraction(a.numerator * b.denominator, - a.denominator * b.numerator) + # Same as _mul(), with inversed b. + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g1 = math.gcd(na, nb) + if g1 > 1: + na //= g1 + nb //= g1 + g2 = math.gcd(db, da) + if g2 > 1: + da //= g2 + db //= g2 + n, d = na * db, nb * da + if d < 0: + n, d = -n, -d + return Fraction(n, d, _normalize=False) __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) diff --git a/contrib/tools/python3/src/Lib/functools.py b/contrib/tools/python3/src/Lib/functools.py index 77e5035ebcc..305ceb450a7 100644 --- a/contrib/tools/python3/src/Lib/functools.py +++ b/contrib/tools/python3/src/Lib/functools.py @@ -236,14 +236,14 @@ _initial_missing = object() def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, sequence[, initial]) -> value + reduce(function, iterable[, initial]) -> value - Apply a function of two arguments cumulatively to the items of a sequence, - from left to right, so as to reduce the sequence to a single value. - For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates + Apply a function of two arguments cumulatively to the items of a sequence + or iterable, from left to right, so as to reduce the iterable to a single + value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates ((((1+2)+3)+4)+5). If initial is present, it is placed before the items - of the sequence in the calculation, and serves as a default when the - sequence is empty. + of the iterable in the calculation, and serves as a default when the + iterable is empty. """ it = iter(sequence) @@ -252,7 +252,8 @@ def reduce(function, sequence, initial=_initial_missing): try: value = next(it) except StopIteration: - raise TypeError("reduce() of empty sequence with no initial value") from None + raise TypeError( + "reduce() of empty iterable with no initial value") from None else: value = initial @@ -912,24 +913,11 @@ class singledispatchmethod: self.dispatcher = singledispatch(func) self.func = func - # bpo-45678: special-casing for classmethod/staticmethod in Python <=3.9, - # as functools.update_wrapper doesn't work properly in singledispatchmethod.__get__ - # if it is applied to an unbound classmethod/staticmethod - if isinstance(func, (staticmethod, classmethod)): - self._wrapped_func = func.__func__ - else: - self._wrapped_func = func def register(self, cls, method=None): """generic_method.register(cls, func) -> func Registers a new implementation for the given *cls* on a *generic_method*. """ - # bpo-39679: in Python <= 3.9, classmethods and staticmethods don't - # inherit __annotations__ of the wrapped function (fixed in 3.10+ as - # a side-effect of bpo-43682) but we need that for annotation-derived - # singledispatches. So we add that just-in-time here. - if isinstance(cls, (staticmethod, classmethod)): - cls.__annotations__ = getattr(cls.__func__, '__annotations__', {}) return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): @@ -939,7 +927,7 @@ class singledispatchmethod: _method.__isabstractmethod__ = self.__isabstractmethod__ _method.register = self.register - update_wrapper(_method, self._wrapped_func) + update_wrapper(_method, self.func) return _method @property diff --git a/contrib/tools/python3/src/Lib/glob.py b/contrib/tools/python3/src/Lib/glob.py index 12370611309..9fc08f45df1 100644 --- a/contrib/tools/python3/src/Lib/glob.py +++ b/contrib/tools/python3/src/Lib/glob.py @@ -4,11 +4,13 @@ import contextlib import os import re import fnmatch +import itertools +import stat import sys __all__ = ["glob", "iglob", "escape"] -def glob(pathname, *, recursive=False): +def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False): """Return a list of paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la @@ -19,9 +21,9 @@ def glob(pathname, *, recursive=False): If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ - return list(iglob(pathname, recursive=recursive)) + return list(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive)) -def iglob(pathname, *, recursive=False): +def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False): """Return an iterator which yields the paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la @@ -33,35 +35,44 @@ def iglob(pathname, *, recursive=False): zero or more directories and subdirectories. """ sys.audit("glob.glob", pathname, recursive) - it = _iglob(pathname, recursive, False) - if recursive and _isrecursive(pathname): - s = next(it) # skip empty string - assert not s + sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd) + if root_dir is not None: + root_dir = os.fspath(root_dir) + else: + root_dir = pathname[:0] + it = _iglob(pathname, root_dir, dir_fd, recursive, False) + if not pathname or recursive and _isrecursive(pathname[:2]): + try: + s = next(it) # skip empty string + if s: + it = itertools.chain((s,), it) + except StopIteration: + pass return it -def _iglob(pathname, recursive, dironly): +def _iglob(pathname, root_dir, dir_fd, recursive, dironly): dirname, basename = os.path.split(pathname) if not has_magic(pathname): assert not dironly if basename: - if os.path.lexists(pathname): + if _lexists(_join(root_dir, pathname), dir_fd): yield pathname else: # Patterns ending with a slash should match only directories - if os.path.isdir(dirname): + if _isdir(_join(root_dir, dirname), dir_fd): yield pathname return if not dirname: if recursive and _isrecursive(basename): - yield from _glob2(dirname, basename, dironly) + yield from _glob2(root_dir, basename, dir_fd, dironly) else: - yield from _glob1(dirname, basename, dironly) + yield from _glob1(root_dir, basename, dir_fd, dironly) return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path # contains magic characters (i.e. r'\\?\C:'). if dirname != pathname and has_magic(dirname): - dirs = _iglob(dirname, recursive, True) + dirs = _iglob(dirname, root_dir, dir_fd, recursive, True) else: dirs = [dirname] if has_magic(basename): @@ -72,80 +83,125 @@ def _iglob(pathname, recursive, dironly): else: glob_in_dir = _glob0 for dirname in dirs: - for name in glob_in_dir(dirname, basename, dironly): + for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly): yield os.path.join(dirname, name) # These 2 helper functions non-recursively glob inside a literal directory. # They return a list of basenames. _glob1 accepts a pattern while _glob0 # takes a literal basename (so it only has to check for its existence). -def _glob1(dirname, pattern, dironly): - names = _listdir(dirname, dironly) +def _glob1(dirname, pattern, dir_fd, dironly): + names = _listdir(dirname, dir_fd, dironly) if not _ishidden(pattern): names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) -def _glob0(dirname, basename, dironly): - if not basename: - # `os.path.split()` returns an empty basename for paths ending with a - # directory separator. 'q*x/' should match only directories. - if os.path.isdir(dirname): +def _glob0(dirname, basename, dir_fd, dironly): + if basename: + if _lexists(_join(dirname, basename), dir_fd): return [basename] else: - if os.path.lexists(os.path.join(dirname, basename)): + # `os.path.split()` returns an empty basename for paths ending with a + # directory separator. 'q*x/' should match only directories. + if _isdir(dirname, dir_fd): return [basename] return [] # Following functions are not public but can be used by third-party code. def glob0(dirname, pattern): - return _glob0(dirname, pattern, False) + return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): - return _glob1(dirname, pattern, False) + return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal # directory. -def _glob2(dirname, pattern, dironly): +def _glob2(dirname, pattern, dir_fd, dironly): assert _isrecursive(pattern) yield pattern[:0] - yield from _rlistdir(dirname, dironly) + yield from _rlistdir(dirname, dir_fd, dironly) # If dironly is false, yields all file names inside a directory. # If dironly is true, yields only directory names. -def _iterdir(dirname, dironly): - if not dirname: - if isinstance(dirname, bytes): - dirname = bytes(os.curdir, 'ASCII') - else: - dirname = os.curdir +def _iterdir(dirname, dir_fd, dironly): try: - with os.scandir(dirname) as it: - for entry in it: - try: - if not dironly or entry.is_dir(): - yield entry.name - except OSError: - pass + fd = None + fsencode = None + if dir_fd is not None: + if dirname: + fd = arg = os.open(dirname, _dir_open_flags, dir_fd=dir_fd) + else: + arg = dir_fd + if isinstance(dirname, bytes): + fsencode = os.fsencode + elif dirname: + arg = dirname + elif isinstance(dirname, bytes): + arg = bytes(os.curdir, 'ASCII') + else: + arg = os.curdir + try: + with os.scandir(arg) as it: + for entry in it: + try: + if not dironly or entry.is_dir(): + if fsencode is not None: + yield fsencode(entry.name) + else: + yield entry.name + except OSError: + pass + finally: + if fd is not None: + os.close(fd) except OSError: return -def _listdir(dirname, dironly): - with contextlib.closing(_iterdir(dirname, dironly)) as it: +def _listdir(dirname, dir_fd, dironly): + with contextlib.closing(_iterdir(dirname, dir_fd, dironly)) as it: return list(it) # Recursively yields relative pathnames inside a literal directory. -def _rlistdir(dirname, dironly): - names = _listdir(dirname, dironly) +def _rlistdir(dirname, dir_fd, dironly): + names = _listdir(dirname, dir_fd, dironly) for x in names: if not _ishidden(x): yield x - path = os.path.join(dirname, x) if dirname else x - for y in _rlistdir(path, dironly): - yield os.path.join(x, y) + path = _join(dirname, x) if dirname else x + for y in _rlistdir(path, dir_fd, dironly): + yield _join(x, y) +def _lexists(pathname, dir_fd): + # Same as os.path.lexists(), but with dir_fd + if dir_fd is None: + return os.path.lexists(pathname) + try: + os.lstat(pathname, dir_fd=dir_fd) + except (OSError, ValueError): + return False + else: + return True + +def _isdir(pathname, dir_fd): + # Same as os.path.isdir(), but with dir_fd + if dir_fd is None: + return os.path.isdir(pathname) + try: + st = os.stat(pathname, dir_fd=dir_fd) + except (OSError, ValueError): + return False + else: + return stat.S_ISDIR(st.st_mode) + +def _join(dirname, basename): + # It is common if dirname or basename is empty + if not dirname or not basename: + return dirname or basename + return os.path.join(dirname, basename) + magic_check = re.compile('([*?[])') magic_check_bytes = re.compile(b'([*?[])') @@ -176,3 +232,6 @@ def escape(pathname): else: pathname = magic_check.sub(r'[\1]', pathname) return drive + pathname + + +_dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) diff --git a/contrib/tools/python3/src/Lib/gzip.py b/contrib/tools/python3/src/Lib/gzip.py index 11a5f41d566..475ec326c0c 100644 --- a/contrib/tools/python3/src/Lib/gzip.py +++ b/contrib/tools/python3/src/Lib/gzip.py @@ -62,6 +62,7 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, raise TypeError("filename must be a str or bytes object, or a file") if "t" in mode: + encoding = io.text_encoding(encoding) return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file @@ -277,7 +278,7 @@ class GzipFile(_compression.BaseStream): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") - if isinstance(data, bytes): + if isinstance(data, (bytes, bytearray)): length = len(data) else: # accept any data that supports the buffer protocol @@ -595,7 +596,7 @@ def main(): f = builtins.open(arg, "rb") g = open(arg + ".gz", "wb") while True: - chunk = f.read(1024) + chunk = f.read(io.DEFAULT_BUFFER_SIZE) if not chunk: break g.write(chunk) diff --git a/contrib/tools/python3/src/Lib/hashlib.py b/contrib/tools/python3/src/Lib/hashlib.py index 58c340d56e3..21a73f3bf6c 100644 --- a/contrib/tools/python3/src/Lib/hashlib.py +++ b/contrib/tools/python3/src/Lib/hashlib.py @@ -173,6 +173,7 @@ try: algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: + _hashlib = None new = __py_new __get_hash = __get_builtin_constructor @@ -180,6 +181,7 @@ try: # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA from _hashlib import pbkdf2_hmac except ImportError: + from warnings import warn as _warn _trans_5C = bytes((x ^ 0x5C) for x in range(256)) _trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -190,6 +192,11 @@ except ImportError: as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster for long passwords. """ + _warn( + "Python implementation of pbkdf2_hmac() is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) if not isinstance(hash_name, str): raise TypeError(hash_name) diff --git a/contrib/tools/python3/src/Lib/hmac.py b/contrib/tools/python3/src/Lib/hmac.py index 180bc378b52..8b4f920db95 100644 --- a/contrib/tools/python3/src/Lib/hmac.py +++ b/contrib/tools/python3/src/Lib/hmac.py @@ -8,11 +8,12 @@ try: import _hashlib as _hashopenssl except ImportError: _hashopenssl = None - _openssl_md_meths = None + _functype = None from _operator import _compare_digest as compare_digest else: - _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names) compare_digest = _hashopenssl.compare_digest + _functype = type(_hashopenssl.openssl_sha256) # builtin type + import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) @@ -23,7 +24,6 @@ trans_36 = bytes((x ^ 0x36) for x in range(256)) digest_size = None - class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. @@ -32,7 +32,7 @@ class HMAC: blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( - "_digest_cons", "_inner", "_outer", "block_size", "digest_size" + "_hmac", "_inner", "_outer", "block_size", "digest_size" ) def __init__(self, key, msg=None, digestmod=''): @@ -55,15 +55,30 @@ class HMAC: if not digestmod: raise TypeError("Missing required parameter 'digestmod'.") + if _hashopenssl and isinstance(digestmod, (str, _functype)): + try: + self._init_hmac(key, msg, digestmod) + except _hashopenssl.UnsupportedDigestmodError: + self._init_old(key, msg, digestmod) + else: + self._init_old(key, msg, digestmod) + + def _init_hmac(self, key, msg, digestmod): + self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + def _init_old(self, key, msg, digestmod): if callable(digestmod): - self._digest_cons = digestmod + digest_cons = digestmod elif isinstance(digestmod, str): - self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d) + digest_cons = lambda d=b'': _hashlib.new(digestmod, d) else: - self._digest_cons = lambda d=b'': digestmod.new(d) + digest_cons = lambda d=b'': digestmod.new(d) - self._outer = self._digest_cons() - self._inner = self._digest_cons() + self._hmac = None + self._outer = digest_cons() + self._inner = digest_cons() self.digest_size = self._inner.digest_size if hasattr(self._inner, 'block_size'): @@ -79,13 +94,13 @@ class HMAC: RuntimeWarning, 2) blocksize = self.blocksize + if len(key) > blocksize: + key = digest_cons(key).digest() + # self.blocksize is the default blocksize. self.block_size is # effective block size as well as the public API attribute. self.block_size = blocksize - if len(key) > blocksize: - key = self._digest_cons(key).digest() - key = key.ljust(blocksize, b'\0') self._outer.update(key.translate(trans_5C)) self._inner.update(key.translate(trans_36)) @@ -94,23 +109,15 @@ class HMAC: @property def name(self): - return "hmac-" + self._inner.name - - @property - def digest_cons(self): - return self._digest_cons - - @property - def inner(self): - return self._inner - - @property - def outer(self): - return self._outer + if self._hmac: + return self._hmac.name + else: + return f"hmac-{self._inner.name}" def update(self, msg): """Feed data from msg into this hashing object.""" - self._inner.update(msg) + inst = self._hmac or self._inner + inst.update(msg) def copy(self): """Return a separate copy of this hashing object. @@ -119,10 +126,14 @@ class HMAC: """ # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) - other._digest_cons = self._digest_cons other.digest_size = self.digest_size - other._inner = self._inner.copy() - other._outer = self._outer.copy() + if self._hmac: + other._hmac = self._hmac.copy() + other._inner = other._outer = None + else: + other._hmac = None + other._inner = self._inner.copy() + other._outer = self._outer.copy() return other def _current(self): @@ -130,9 +141,12 @@ class HMAC: To be used only internally with digest() and hexdigest(). """ - h = self._outer.copy() - h.update(self._inner.digest()) - return h + if self._hmac: + return self._hmac + else: + h = self._outer.copy() + h.update(self._inner.digest()) + return h def digest(self): """Return the hash value of this hashing object. @@ -179,9 +193,11 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if (_hashopenssl is not None and - isinstance(digest, str) and digest in _openssl_md_meths): - return _hashopenssl.hmac_digest(key, msg, digest) + if _hashopenssl is not None and isinstance(digest, (str, _functype)): + try: + return _hashopenssl.hmac_digest(key, msg, digest) + except _hashopenssl.UnsupportedDigestmodError: + pass if callable(digest): digest_cons = digest diff --git a/contrib/tools/python3/src/Lib/http/__init__.py b/contrib/tools/python3/src/Lib/http/__init__.py index 37be765349e..bf8d7d68868 100644 --- a/contrib/tools/python3/src/Lib/http/__init__.py +++ b/contrib/tools/python3/src/Lib/http/__init__.py @@ -2,6 +2,7 @@ from enum import IntEnum __all__ = ['HTTPStatus'] + class HTTPStatus(IntEnum): """HTTP status codes and reason phrases diff --git a/contrib/tools/python3/src/Lib/http/client.py b/contrib/tools/python3/src/Lib/http/client.py index a98432e5685..a6ab135b2c3 100644 --- a/contrib/tools/python3/src/Lib/http/client.py +++ b/contrib/tools/python3/src/Lib/http/client.py @@ -75,6 +75,7 @@ import http import io import re import socket +import sys import collections.abc from urllib.parse import urlsplit @@ -106,9 +107,6 @@ globals().update(http.HTTPStatus.__members__) # Mapping status codes to official W3C names responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()} -# maximal amount of data to read at one time in _safe_read -MAXAMOUNT = 1048576 - # maximal line length when calling readline(). _MAXLINE = 65536 _MAXHEADERS = 100 @@ -457,18 +455,25 @@ class HTTPResponse(io.BufferedIOBase): self._close_conn() return b"" + if self.chunked: + return self._read_chunked(amt) + if amt is not None: - # Amount is given, implement using readinto - b = bytearray(amt) - n = self.readinto(b) - return memoryview(b)[:n].tobytes() + if self.length is not None and amt > self.length: + # clip the read to the "end of response" + amt = self.length + s = self.fp.read(amt) + if not s and amt: + # Ideally, we would raise IncompleteRead if the content-length + # wasn't satisfied, but it might break compatibility. + self._close_conn() + elif self.length is not None: + self.length -= len(s) + if not self.length: + self._close_conn() + return s else: # Amount is not given (unbounded read) so we must check self.length - # and self.chunked - - if self.chunked: - return self._readall_chunked() - if self.length is None: s = self.fp.read() else: @@ -569,7 +574,7 @@ class HTTPResponse(io.BufferedIOBase): self.chunk_left = chunk_left return chunk_left - def _readall_chunked(self): + def _read_chunked(self, amt=None): assert self.chunked != _UNKNOWN value = [] try: @@ -577,7 +582,15 @@ class HTTPResponse(io.BufferedIOBase): chunk_left = self._get_chunk_left() if chunk_left is None: break + + if amt is not None and amt <= chunk_left: + value.append(self._safe_read(amt)) + self.chunk_left = chunk_left - amt + break + value.append(self._safe_read(chunk_left)) + if amt is not None: + amt -= chunk_left self.chunk_left = 0 return b''.join(value) except IncompleteRead: @@ -608,43 +621,24 @@ class HTTPResponse(io.BufferedIOBase): raise IncompleteRead(bytes(b[0:total_bytes])) def _safe_read(self, amt): - """Read the number of bytes requested, compensating for partial reads. - - Normally, we have a blocking socket, but a read() can be interrupted - by a signal (resulting in a partial read). - - Note that we cannot distinguish between EOF and an interrupt when zero - bytes have been read. IncompleteRead() will be raised in this - situation. + """Read the number of bytes requested. This function should be used when bytes "should" be present for reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - s = [] - while amt > 0: - chunk = self.fp.read(min(amt, MAXAMOUNT)) - if not chunk: - raise IncompleteRead(b''.join(s), amt) - s.append(chunk) - amt -= len(chunk) - return b"".join(s) + data = self.fp.read(amt) + if len(data) < amt: + raise IncompleteRead(data, amt-len(data)) + return data def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" - total_bytes = 0 - mvb = memoryview(b) - while total_bytes < len(b): - if MAXAMOUNT < len(mvb): - temp_mvb = mvb[0:MAXAMOUNT] - n = self.fp.readinto(temp_mvb) - else: - n = self.fp.readinto(mvb) - if not n: - raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b)) - mvb = mvb[n:] - total_bytes += n - return total_bytes + amt = len(b) + n = self.fp.readinto(b) + if n < amt: + raise IncompleteRead(bytes(b[:n]), amt-n) + return n def read1(self, n=-1): """Read with at most one underlying system call. If at least one @@ -943,6 +937,7 @@ class HTTPConnection: def connect(self): """Connect to the host and port specified in __init__.""" + sys.audit("http.client.connect", self, self.host, self.port) self.sock = self._create_connection( (self.host,self.port), self.timeout, self.source_address) # Might fail in OSs that don't implement TCP_NODELAY @@ -995,8 +990,10 @@ class HTTPConnection: break if encode: datablock = datablock.encode("iso-8859-1") + sys.audit("http.client.send", self, datablock) self.sock.sendall(datablock) return + sys.audit("http.client.send", self, data) try: self.sock.sendall(data) except TypeError: @@ -1422,6 +1419,9 @@ else: self.cert_file = cert_file if context is None: context = ssl._create_default_https_context() + # send ALPN extension to indicate HTTP/1.1 protocol + if self._http_vsn == 11: + context.set_alpn_protocols(['http/1.1']) # enable PHA for TLS 1.3 connections if available if context.post_handshake_auth is not None: context.post_handshake_auth = True diff --git a/contrib/tools/python3/src/Lib/http/cookiejar.py b/contrib/tools/python3/src/Lib/http/cookiejar.py index 47ed5c3d64a..eaa76c26b9c 100644 --- a/contrib/tools/python3/src/Lib/http/cookiejar.py +++ b/contrib/tools/python3/src/Lib/http/cookiejar.py @@ -50,10 +50,18 @@ def _debug(*args): logger = logging.getLogger("http.cookiejar") return logger.debug(*args) - +HTTPONLY_ATTR = "HTTPOnly" +HTTPONLY_PREFIX = "#HttpOnly_" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) +NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File") MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") +NETSCAPE_HEADER_TEXT = """\ +# Netscape HTTP Cookie File +# http://curl.haxx.se/rfc/cookie_spec.html +# This is a generated file! Do not edit. + +""" def _warn_unhandled_exception(): # There are a few catch-all except: statements in this module, for @@ -2004,19 +2012,11 @@ class MozillaCookieJar(FileCookieJar): header by default (Mozilla can cope with that). """ - magic_re = re.compile("#( Netscape)? HTTP Cookie File") - header = """\ -# Netscape HTTP Cookie File -# http://curl.haxx.se/rfc/cookie_spec.html -# This is a generated file! Do not edit. - -""" def _really_load(self, f, filename, ignore_discard, ignore_expires): now = time.time() - magic = f.readline() - if not self.magic_re.search(magic): + if not NETSCAPE_MAGIC_RGX.match(f.readline()): raise LoadError( "%r does not look like a Netscape format cookies file" % filename) @@ -2024,8 +2024,17 @@ class MozillaCookieJar(FileCookieJar): try: while 1: line = f.readline() + rest = {} + if line == "": break + # httponly is a cookie flag as defined in rfc6265 + # when encoded in a netscape cookie file, + # the line is prepended with "#HttpOnly_" + if line.startswith(HTTPONLY_PREFIX): + rest[HTTPONLY_ATTR] = "" + line = line[len(HTTPONLY_PREFIX):] + # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] @@ -2063,7 +2072,7 @@ class MozillaCookieJar(FileCookieJar): discard, None, None, - {}) + rest) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): @@ -2083,16 +2092,17 @@ class MozillaCookieJar(FileCookieJar): else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename, "w") as f: - f.write(self.header) + f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: + domain = cookie.domain if not ignore_discard and cookie.discard: continue if not ignore_expires and cookie.is_expired(now): continue if cookie.secure: secure = "TRUE" else: secure = "FALSE" - if cookie.domain.startswith("."): initial_dot = "TRUE" + if domain.startswith("."): initial_dot = "TRUE" else: initial_dot = "FALSE" if cookie.expires is not None: expires = str(cookie.expires) @@ -2107,7 +2117,9 @@ class MozillaCookieJar(FileCookieJar): else: name = cookie.name value = cookie.value + if cookie.has_nonstandard_attr(HTTPONLY_ATTR): + domain = HTTPONLY_PREFIX + domain f.write( - "\t".join([cookie.domain, initial_dot, cookie.path, + "\t".join([domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") diff --git a/contrib/tools/python3/src/Lib/http/server.py b/contrib/tools/python3/src/Lib/http/server.py index 2d2300c2aea..58abadf7377 100644 --- a/contrib/tools/python3/src/Lib/http/server.py +++ b/contrib/tools/python3/src/Lib/http/server.py @@ -412,7 +412,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): method = getattr(self, mname) method() self.wfile.flush() #actually send the response if not already done. - except socket.timeout as e: + except TimeoutError as e: #a read or a write timed out. Discard this connection self.log_error("Request timed out: %r", e) self.close_connection = True @@ -1091,8 +1091,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): env['PATH_INFO'] = uqrest env['PATH_TRANSLATED'] = self.translate_path(uqrest) env['SCRIPT_NAME'] = scriptname - if query: - env['QUERY_STRING'] = query + env['QUERY_STRING'] = query env['REMOTE_ADDR'] = self.client_address[0] authorization = self.headers.get("authorization") if authorization: diff --git a/contrib/tools/python3/src/Lib/imaplib.py b/contrib/tools/python3/src/Lib/imaplib.py index d9720f20c39..73184396d89 100644 --- a/contrib/tools/python3/src/Lib/imaplib.py +++ b/contrib/tools/python3/src/Lib/imaplib.py @@ -1251,13 +1251,12 @@ class IMAP4: sys.stderr.write(' %s.%02d %s\n' % (tm, (secs*100)%100, s)) sys.stderr.flush() - def _dump_ur(self, dict): - # Dump untagged responses (in `dict'). - l = dict.items() - if not l: return - t = '\n\t\t' - l = map(lambda x:'%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or ''), l) - self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) + def _dump_ur(self, untagged_resp_dict): + if not untagged_resp_dict: + return + items = (f'{key}: {value!r}' + for key, value in untagged_resp_dict.items()) + self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) def _log(self, line): # Keep log of last `_cmd_log_len' interactions for debugging. diff --git a/contrib/tools/python3/src/Lib/imp.py b/contrib/tools/python3/src/Lib/imp.py index 31f8c766381..e02aaef344c 100644 --- a/contrib/tools/python3/src/Lib/imp.py +++ b/contrib/tools/python3/src/Lib/imp.py @@ -28,7 +28,8 @@ import tokenize import types import warnings -warnings.warn("the imp module is deprecated in favour of importlib; " +warnings.warn("the imp module is deprecated in favour of importlib and slated " + "for removal in Python 3.12; " "see the module's documentation for alternative uses", DeprecationWarning, stacklevel=2) diff --git a/contrib/tools/python3/src/Lib/importlib/__init__.py b/contrib/tools/python3/src/Lib/importlib/__init__.py index 0c73c505f98..ce61883288a 100644 --- a/contrib/tools/python3/src/Lib/importlib/__init__.py +++ b/contrib/tools/python3/src/Lib/importlib/__init__.py @@ -34,7 +34,7 @@ try: import _frozen_importlib_external as _bootstrap_external except ImportError: from . import _bootstrap_external - _bootstrap_external._setup(_bootstrap) + _bootstrap_external._set_bootstrap_module(_bootstrap) _bootstrap._bootstrap_external = _bootstrap_external else: _bootstrap_external.__name__ = 'importlib._bootstrap_external' @@ -54,7 +54,6 @@ _unpack_uint32 = _bootstrap_external._unpack_uint32 # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import types import warnings @@ -79,8 +78,8 @@ def find_loader(name, path=None): This function is deprecated in favor of importlib.util.find_spec(). """ - warnings.warn('Deprecated since Python 3.4. ' - 'Use importlib.util.find_spec() instead.', + warnings.warn('Deprecated since Python 3.4 and slated for removal in ' + 'Python 3.12; use importlib.util.find_spec() instead', DeprecationWarning, stacklevel=2) try: loader = sys.modules[name].__loader__ @@ -136,12 +135,13 @@ def reload(module): The module must have been successfully imported before. """ - if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: - name = module.__name__ + try: + name = module.__name__ + except AttributeError: + raise TypeError("reload() argument must be a module") if sys.modules.get(name) is not module: msg = "module {} not in sys.modules" diff --git a/contrib/tools/python3/src/Lib/importlib/_abc.py b/contrib/tools/python3/src/Lib/importlib/_abc.py new file mode 100644 index 00000000000..f80348fc7ff --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/_abc.py @@ -0,0 +1,54 @@ +"""Subset of importlib.abc used to reduce importlib.util imports.""" +from . import _bootstrap +import abc +import warnings + + +class Loader(metaclass=abc.ABCMeta): + + """Abstract base class for import loaders.""" + + def create_module(self, spec): + """Return a module to initialize and into which to load. + + This method should raise ImportError if anything prevents it + from creating a new module. It may return None to indicate + that the spec should create the new module. + """ + # By default, defer to default semantics for the new module. + return None + + # We don't define exec_module() here since that would break + # hasattr checks we do to support backward compatibility. + + def load_module(self, fullname): + """Return the loaded module. + + The module must be added to sys.modules and have import-related + attributes set properly. The fullname is a str. + + ImportError is raised on failure. + + This method is deprecated in favor of loader.exec_module(). If + exec_module() exists then it is used to provide a backwards-compatible + functionality for this method. + + """ + if not hasattr(self, 'exec_module'): + raise ImportError + # Warning implemented in _load_module_shim(). + return _bootstrap._load_module_shim(self, fullname) + + def module_repr(self, module): + """Return a module's repr. + + Used by the module type when the method does not raise + NotImplementedError. + + This method is deprecated. + + """ + warnings.warn("importlib.abc.Loader.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) + # The exception will cause ModuleType.__repr__ to ignore this method. + raise NotImplementedError diff --git a/contrib/tools/python3/src/Lib/importlib/_adapters.py b/contrib/tools/python3/src/Lib/importlib/_adapters.py new file mode 100644 index 00000000000..e72edd10705 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/_adapters.py @@ -0,0 +1,83 @@ +from contextlib import suppress + +from . import abc + + +class SpecLoaderAdapter: + """ + Adapt a package spec to adapt the underlying loader. + """ + + def __init__(self, spec, adapter=lambda spec: spec.loader): + self.spec = spec + self.loader = adapter(spec) + + def __getattr__(self, name): + return getattr(self.spec, name) + + +class TraversableResourcesLoader: + """ + Adapt a loader to provide TraversableResources. + """ + + def __init__(self, spec): + self.spec = spec + + def get_resource_reader(self, name): + return DegenerateFiles(self.spec)._native() + + +class DegenerateFiles: + """ + Adapter for an existing or non-existant resource reader + to provide a degenerate .files(). + """ + + class Path(abc.Traversable): + def iterdir(self): + return iter(()) + + def is_dir(self): + return False + + is_file = exists = is_dir # type: ignore + + def joinpath(self, other): + return DegenerateFiles.Path() + + @property + def name(self): + return '' + + def open(self, mode='rb', *args, **kwargs): + raise ValueError() + + def __init__(self, spec): + self.spec = spec + + @property + def _reader(self): + with suppress(AttributeError): + return self.spec.loader.get_resource_reader(self.spec.name) + + def _native(self): + """ + Return the native reader if it supports files(). + """ + reader = self._reader + return reader if hasattr(reader, 'files') else self + + def __getattr__(self, attr): + return getattr(self._reader, attr) + + def files(self): + return DegenerateFiles.Path() + + +def wrap_spec(package): + """ + Construct a package spec with traversable compatibility + on the spec/loader/reader. + """ + return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/tools/python3/src/Lib/importlib/_bootstrap.py b/contrib/tools/python3/src/Lib/importlib/_bootstrap.py index e00b27ece26..527bc9c63c9 100644 --- a/contrib/tools/python3/src/Lib/importlib/_bootstrap.py +++ b/contrib/tools/python3/src/Lib/importlib/_bootstrap.py @@ -20,10 +20,23 @@ work. One should use importlib as the public-facing version of this module. # reference any injected objects! This includes not only global code but also # anything specified at the class level. +def _object_name(obj): + try: + return obj.__qualname__ + except AttributeError: + return type(obj).__qualname__ + # Bootstrap-related code ###################################################### +# Modules injected manually by _setup() +_thread = None +_warnings = None +_weakref = None + +# Import done by _install_external_importers() _bootstrap_external = None + def _wrap(new, old): """Simple substitute for functools.update_wrapper.""" for replace in ['__module__', '__name__', '__qualname__', '__doc__']: @@ -262,9 +275,12 @@ def _requires_frozen(fxn): def _load_module_shim(self, fullname): """Load the specified module into sys.modules and return it. - This method is deprecated. Use loader.exec_module instead. + This method is deprecated. Use loader.exec_module() instead. """ + msg = ("the load_module() method is deprecated and slated for removal in " + "Python 3.12; use exec_module() instead") + _warnings.warn(msg, DeprecationWarning) spec = spec_from_loader(fullname, self) if fullname in sys.modules: module = sys.modules[fullname] @@ -276,26 +292,16 @@ def _load_module_shim(self, fullname): # Module specifications ####################################################### def _module_repr(module): - # The implementation of ModuleType.__repr__(). + """The implementation of ModuleType.__repr__().""" loader = getattr(module, '__loader__', None) - if hasattr(loader, 'module_repr'): - # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader - # drop their implementations for module_repr. we can add a - # deprecation warning here. + if spec := getattr(module, "__spec__", None): + return _module_repr_from_spec(spec) + elif hasattr(loader, 'module_repr'): try: return loader.module_repr(module) except Exception: pass - try: - spec = module.__spec__ - except AttributeError: - pass - else: - if spec is not None: - return _module_repr_from_spec(spec) - - # We could use module.__class__.__name__ instead of 'module' in the - # various repr permutations. + # Fall through to a catch-all which always succeeds. try: name = module.__name__ except AttributeError: @@ -605,9 +611,9 @@ def _exec(spec, module): else: _init_module_attrs(spec, module, override=True) if not hasattr(spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) spec.loader.load_module(name) else: spec.loader.exec_module(module) @@ -620,9 +626,8 @@ def _exec(spec, module): def _load_backward_compatible(spec): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. + # It is assumed that all callers have been warned about using load_module() + # appropriately before calling this function. try: spec.loader.load_module(spec.name) except: @@ -661,6 +666,9 @@ def _load_unlocked(spec): if spec.loader is not None: # Not a namespace package. if not hasattr(spec.loader, 'exec_module'): + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) return _load_backward_compatible(spec) module = module_from_spec(spec) @@ -731,6 +739,8 @@ class BuiltinImporter: The method is deprecated. The import machinery does the job itself. """ + _warnings.warn("BuiltinImporter.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) return f'' @classmethod @@ -751,19 +761,22 @@ class BuiltinImporter: This method is deprecated. Use find_spec() instead. """ + _warnings.warn("BuiltinImporter.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) return spec.loader if spec is not None else None - @classmethod - def create_module(self, spec): + @staticmethod + def create_module(spec): """Create a built-in module""" if spec.name not in sys.builtin_module_names: raise ImportError('{!r} is not a built-in module'.format(spec.name), name=spec.name) return _call_with_frames_removed(_imp.create_builtin, spec) - @classmethod - def exec_module(self, module): + @staticmethod + def exec_module(module): """Exec a built-in module""" _call_with_frames_removed(_imp.exec_builtin, module) @@ -806,6 +819,8 @@ class FrozenImporter: The method is deprecated. The import machinery does the job itself. """ + _warnings.warn("FrozenImporter.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) return ''.format(m.__name__, FrozenImporter._ORIGIN) @classmethod @@ -822,10 +837,13 @@ class FrozenImporter: This method is deprecated. Use find_spec() instead. """ + _warnings.warn("FrozenImporter.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) return cls if _imp.is_frozen(fullname) else None - @classmethod - def create_module(cls, spec): + @staticmethod + def create_module(spec): """Use default semantics for module creation.""" @staticmethod @@ -844,6 +862,7 @@ class FrozenImporter: This method is deprecated. Use exec_module() instead. """ + # Warning about deprecation implemented in _load_module_shim(). return _load_module_shim(cls, fullname) @classmethod @@ -890,8 +909,9 @@ def _resolve_name(name, package, level): def _find_spec_legacy(finder, name, path): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. + msg = (f"{_object_name(finder)}.find_spec() not found; " + "falling back to find_module()") + _warnings.warn(msg, ImportWarning) loader = finder.find_module(name, path) if loader is None: return None diff --git a/contrib/tools/python3/src/Lib/importlib/_bootstrap_external.py b/contrib/tools/python3/src/Lib/importlib/_bootstrap_external.py index f3828b10e1c..49bcaea78d7 100644 --- a/contrib/tools/python3/src/Lib/importlib/_bootstrap_external.py +++ b/contrib/tools/python3/src/Lib/importlib/_bootstrap_external.py @@ -19,6 +19,9 @@ work. One should use importlib as the public-facing version of this module. # reference any injected objects! This includes not only global code but also # anything specified at the class level. +# Module injected manually by _set_bootstrap_module() +_bootstrap = None + # Import builtin modules import _imp import _io @@ -70,6 +73,8 @@ def _make_relax_case(): return False return _relax_case +_relax_case = _make_relax_case() + def _pack_uint32(x): """Convert a 32-bit integer to little-endian.""" @@ -337,6 +342,16 @@ _code_type = type(_write_atomic.__code__) # Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) # Python 3.9a2 3424 (simplify bytecodes for *value unpacking) # Python 3.9a2 3425 (simplify bytecodes for **value unpacking) +# Python 3.10a1 3430 (Make 'annotations' future by default) +# Python 3.10a1 3431 (New line number table format -- PEP 626) +# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) +# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) +# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) +# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). +# Python 3.10b1 3436 (Add GEN_START bytecode #43683) +# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) +# Python 3.10b1 3438 Safer line number table handling. +# Python 3.10b1 3439 (Add ROT_N) # # MAGIC must change whenever the bytecode emitted by the compiler may no @@ -346,13 +361,17 @@ _code_type = type(_write_atomic.__code__) # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3425).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3439).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' _OPT = 'opt-' -SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. +SOURCE_SUFFIXES = ['.py'] +if _MS_WINDOWS: + SOURCE_SUFFIXES.append('.pyw') + +EXTENSION_SUFFIXES = _imp.extension_suffixes() BYTECODE_SUFFIXES = ['.pyc'] # Deprecated. @@ -527,15 +546,18 @@ def _check_name(method): raise ImportError('loader for %s cannot handle %s' % (self.name, name), name=name) return method(self, name, *args, **kwargs) - try: + + # FIXME: @_check_name is used to define class methods before the + # _bootstrap module is set by _set_bootstrap_module(). + if _bootstrap is not None: _wrap = _bootstrap._wrap - except NameError: - # XXX yuck + else: def _wrap(new, old): for replace in ['__module__', '__name__', '__qualname__', '__doc__']: if hasattr(old, replace): setattr(new, replace, getattr(old, replace)) new.__dict__.update(old.__dict__) + _wrap(_check_name_wrapper, method) return _check_name_wrapper @@ -547,6 +569,9 @@ def _find_module_shim(self, fullname): This method is deprecated in favor of finder.find_spec(). """ + _warnings.warn("find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) # Call find_loader(). If it returns a string (indicating this # is a namespace package portion), generate a warning and # return None. @@ -718,6 +743,11 @@ def spec_from_file_location(name, location=None, *, loader=None, pass else: location = _os.fspath(location) + if not _path_isabs(location): + try: + location = _path_join(_os.getcwd(), location) + except OSError: + pass # If the location is on the filesystem, but doesn't actually exist, # we could return None here, indicating that the location is not @@ -771,10 +801,10 @@ class WindowsRegistryFinder: REGISTRY_KEY_DEBUG = ( 'Software\\Python\\PythonCore\\{sys_version}' '\\Modules\\{fullname}\\Debug') - DEBUG_BUILD = False # Changed in _setup() + DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES) - @classmethod - def _open_registry(cls, key): + @staticmethod + def _open_registry(key): try: return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key) except OSError: @@ -815,9 +845,12 @@ class WindowsRegistryFinder: def find_module(cls, fullname, path=None): """Find module named in the registry. - This method is deprecated. Use exec_module() instead. + This method is deprecated. Use find_spec() instead. """ + _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) if spec is not None: return spec.loader @@ -850,7 +883,8 @@ class _LoaderBasics: _bootstrap._call_with_frames_removed(exec, code, module.__dict__) def load_module(self, fullname): - """This module is deprecated.""" + """This method is deprecated.""" + # Warning implemented in _load_module_shim(). return _bootstrap._load_module_shim(self, fullname) @@ -1025,7 +1059,7 @@ class FileLoader: """ # The only reason for this method is for the name check. # Issue #14857: Avoid the zero-argument form of super so the implementation - # of that form can be updated without breaking the frozen module + # of that form can be updated without breaking the frozen module. return super(FileLoader, self).load_module(fullname) @_check_name @@ -1042,32 +1076,10 @@ class FileLoader: with _io.FileIO(path, 'r') as file: return file.read() - # ResourceReader ABC API. - @_check_name def get_resource_reader(self, module): - if self.is_package(module): - return self - return None - - def open_resource(self, resource): - path = _path_join(_path_split(self.path)[0], resource) - return _io.FileIO(path, 'r') - - def resource_path(self, resource): - if not self.is_resource(resource): - raise FileNotFoundError - path = _path_join(_path_split(self.path)[0], resource) - return path - - def is_resource(self, name): - if path_sep in name: - return False - path = _path_join(_path_split(self.path)[0], name) - return _path_isfile(path) - - def contents(self): - return iter(_os.listdir(_path_split(self.path)[0])) + from importlib.readers import FileReader + return FileReader(self) class SourceFileLoader(FileLoader, SourceLoader): @@ -1140,10 +1152,6 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics): return None -# Filled in by _setup(). -EXTENSION_SUFFIXES = [] - - class ExtensionFileLoader(FileLoader, _LoaderBasics): """Loader for extension modules. @@ -1154,11 +1162,6 @@ class ExtensionFileLoader(FileLoader, _LoaderBasics): def __init__(self, name, path): self.name = name - if not _path_isabs(path): - try: - path = _path_join(_os.getcwd(), path) - except OSError: - pass self.path = path def __eq__(self, other): @@ -1275,13 +1278,15 @@ class _NamespaceLoader: def __init__(self, name, path, path_finder): self._path = _NamespacePath(name, path, path_finder) - @classmethod - def module_repr(cls, module): + @staticmethod + def module_repr(module): """Return repr for the module. The method is deprecated. The import machinery does the job itself. """ + _warnings.warn("_NamespaceLoader.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) return ''.format(module.__name__) def is_package(self, fullname): @@ -1308,8 +1313,13 @@ class _NamespaceLoader: # The import system never calls this method. _bootstrap._verbose_message('namespace module loaded with path {!r}', self._path) + # Warning implemented in _load_module_shim(). return _bootstrap._load_module_shim(self, fullname) + def get_resource_reader(self, module): + from importlib.readers import NamespaceReader + return NamespaceReader(self._path) + # Finders ##################################################################### @@ -1317,8 +1327,8 @@ class PathFinder: """Meta path finder for sys.path and package __path__ attributes.""" - @classmethod - def invalidate_caches(cls): + @staticmethod + def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders stored in sys.path_importer_caches (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): @@ -1330,8 +1340,8 @@ class PathFinder: # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 - @classmethod - def _path_hooks(cls, path): + @staticmethod + def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" if sys.path_hooks is not None and not sys.path_hooks: _warnings.warn('sys.path_hooks is empty', ImportWarning) @@ -1370,8 +1380,14 @@ class PathFinder: # This would be a good place for a DeprecationWarning if # we ended up going that route. if hasattr(finder, 'find_loader'): + msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " + "falling back to find_loader()") + _warnings.warn(msg, ImportWarning) loader, portions = finder.find_loader(fullname) else: + msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " + "falling back to find_module()") + _warnings.warn(msg, ImportWarning) loader = finder.find_module(fullname) portions = [] if loader is not None: @@ -1444,13 +1460,16 @@ class PathFinder: This method is deprecated. Use find_spec() instead. """ + _warnings.warn("PathFinder.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) if spec is None: return None return spec.loader - @classmethod - def find_distributions(cls, *args, **kwargs): + @staticmethod + def find_distributions(*args, **kwargs): """ Find distributions. @@ -1501,6 +1520,9 @@ class FileFinder: This method is deprecated. Use find_spec() instead. """ + _warnings.warn("FileFinder.find_loader() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = self.find_spec(fullname) if spec is None: return None, [] @@ -1651,66 +1673,14 @@ def _get_supported_file_loaders(): return [extensions, source, bytecode] -def _setup(_bootstrap_module): - """Setup the path-based importers for importlib by importing needed - built-in modules and injecting them into the global namespace. - - Other components are extracted from the core bootstrap module. - - """ - global sys, _imp, _bootstrap +def _set_bootstrap_module(_bootstrap_module): + global _bootstrap _bootstrap = _bootstrap_module - sys = _bootstrap.sys - _imp = _bootstrap._imp - - self_module = sys.modules[__name__] - - # Directly load the os module (needed during bootstrap). - os_details = ('posix', ['/']), ('nt', ['\\', '/']) - for builtin_os, path_separators in os_details: - # Assumption made in _path_join() - assert all(len(sep) == 1 for sep in path_separators) - path_sep = path_separators[0] - if builtin_os in sys.modules: - os_module = sys.modules[builtin_os] - break - else: - try: - os_module = _bootstrap._builtin_from_name(builtin_os) - break - except ImportError: - continue - else: - raise ImportError('importlib requires posix or nt') - - setattr(self_module, '_os', os_module) - setattr(self_module, 'path_sep', path_sep) - setattr(self_module, 'path_separators', ''.join(path_separators)) - setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators}) - - # Directly load built-in modules needed during bootstrap. - builtin_names = ['_io', '_warnings', 'marshal'] - if builtin_os == 'nt': - builtin_names.append('winreg') - for builtin_name in builtin_names: - if builtin_name not in sys.modules: - builtin_module = _bootstrap._builtin_from_name(builtin_name) - else: - builtin_module = sys.modules[builtin_name] - setattr(self_module, builtin_name, builtin_module) - - # Constants - setattr(self_module, '_relax_case', _make_relax_case()) - EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) - if builtin_os == 'nt': - SOURCE_SUFFIXES.append('.pyw') - if '_d.pyd' in EXTENSION_SUFFIXES: - WindowsRegistryFinder.DEBUG_BUILD = True def _install(_bootstrap_module): """Install the path-based import components.""" - _setup(_bootstrap_module) + _set_bootstrap_module(_bootstrap_module) supported_loaders = _get_supported_file_loaders() sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.append(PathFinder) diff --git a/contrib/tools/python3/src/Lib/importlib/_common.py b/contrib/tools/python3/src/Lib/importlib/_common.py index c1204f0b8f9..549fee379a4 100644 --- a/contrib/tools/python3/src/Lib/importlib/_common.py +++ b/contrib/tools/python3/src/Lib/importlib/_common.py @@ -1,28 +1,82 @@ import os import pathlib -import zipfile import tempfile import functools import contextlib +import types +import importlib +from typing import Union, Any, Optional +from .abc import ResourceReader, Traversable -def from_package(package): +from ._adapters import wrap_spec + +Package = Union[types.ModuleType, str] + + +def files(package): + # type: (Package) -> Traversable """ - Return a Traversable object for the given package. + Get a Traversable resource from a package + """ + return from_package(get_package(package)) + +def normalize_path(path): + # type: (Any) -> str + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. """ - return fallback_resources(package.__spec__) + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError(f'{path!r} must be only a file name') + return file_name -def fallback_resources(spec): - package_directory = pathlib.Path(spec.origin).parent - try: - archive_path = spec.loader.archive - rel_path = package_directory.relative_to(archive_path) - return zipfile.Path(archive_path, str(rel_path) + '/') - except Exception: - pass - return package_directory +def get_resource_reader(package): + # type: (types.ModuleType) -> Optional[ResourceReader] + """ + Return the package's loader if it's a ResourceReader. + """ + # We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + if reader is None: + return None + return reader(spec.name) # type: ignore + + +def resolve(cand): + # type: (Package) -> types.ModuleType + return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand) + + +def get_package(package): + # type: (Package) -> types.ModuleType + """Take a package name or module object and return the module. + + Raise an exception if the resolved module is not a package. + """ + resolved = resolve(package) + if wrap_spec(resolved).submodule_search_locations is None: + raise TypeError(f'{package!r} is not a package') + return resolved + + +def from_package(package): + """ + Return a Traversable object for the given package. + + """ + spec = wrap_spec(package) + reader = spec.loader.get_resource_reader(spec.name) + return reader.files() @contextlib.contextmanager @@ -34,6 +88,7 @@ def _tempfile(reader, suffix=''): try: os.write(fd, reader()) os.close(fd) + del reader yield pathlib.Path(raw_path) finally: try: @@ -43,14 +98,12 @@ def _tempfile(reader, suffix=''): @functools.singledispatch -@contextlib.contextmanager def as_file(path): """ Given a Traversable object, return that object as a path on the local file system in a context manager. """ - with _tempfile(path.read_bytes, suffix=path.name) as local: - yield local + return _tempfile(path.read_bytes, suffix=path.name) @as_file.register(pathlib.Path) diff --git a/contrib/tools/python3/src/Lib/importlib/abc.py b/contrib/tools/python3/src/Lib/importlib/abc.py index b8a9bb1a21e..0b4a3f80717 100644 --- a/contrib/tools/python3/src/Lib/importlib/abc.py +++ b/contrib/tools/python3/src/Lib/importlib/abc.py @@ -1,5 +1,4 @@ """Abstract base classes related to import.""" -from . import _bootstrap from . import _bootstrap_external from . import machinery try: @@ -12,8 +11,10 @@ try: import _frozen_importlib_external except ImportError: _frozen_importlib_external = _bootstrap_external +from ._abc import Loader import abc import warnings +from typing import BinaryIO, Iterable, Text from typing import Protocol, runtime_checkable @@ -40,15 +41,27 @@ class Finder(metaclass=abc.ABCMeta): Deprecated since Python 3.3 """ + def __init__(self): + warnings.warn("the Finder ABC is deprecated and " + "slated for removal in Python 3.12; use MetaPathFinder " + "or PathEntryFinder instead", + DeprecationWarning) + @abc.abstractmethod def find_module(self, fullname, path=None): """An abstract method that should find a module. The fullname is a str and the optional path is a str or None. Returns a Loader object or None. """ + warnings.warn("importlib.abc.Finder along with its find_module() " + "method are deprecated and " + "slated for removal in Python 3.12; use " + "MetaPathFinder.find_spec() or " + "PathEntryFinder.find_spec() instead", + DeprecationWarning) -class MetaPathFinder(Finder): +class MetaPathFinder(metaclass=abc.ABCMeta): """Abstract base class for import finders on sys.meta_path.""" @@ -67,8 +80,8 @@ class MetaPathFinder(Finder): """ warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec() " - "(available since 3.4)", + "3.4 in favor of MetaPathFinder.find_spec() and is " + "slated for removal in Python 3.12", DeprecationWarning, stacklevel=2) if not hasattr(self, 'find_spec'): @@ -85,7 +98,7 @@ _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.PathFinder, machinery.WindowsRegistryFinder) -class PathEntryFinder(Finder): +class PathEntryFinder(metaclass=abc.ABCMeta): """Abstract base class for path entry finders used by PathFinder.""" @@ -134,53 +147,6 @@ class PathEntryFinder(Finder): _register(PathEntryFinder, machinery.FileFinder) -class Loader(metaclass=abc.ABCMeta): - - """Abstract base class for import loaders.""" - - def create_module(self, spec): - """Return a module to initialize and into which to load. - - This method should raise ImportError if anything prevents it - from creating a new module. It may return None to indicate - that the spec should create the new module. - """ - # By default, defer to default semantics for the new module. - return None - - # We don't define exec_module() here since that would break - # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - return _bootstrap._load_module_shim(self, fullname) - - def module_repr(self, module): - """Return a module's repr. - - Used by the module type when the method does not raise - NotImplementedError. - - This method is deprecated. - - """ - # The exception will cause ModuleType.__repr__ to ignore this method. - raise NotImplementedError - - class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their @@ -344,49 +310,45 @@ _register(SourceLoader, machinery.SourceFileLoader) class ResourceReader(metaclass=abc.ABCMeta): - - """Abstract base class to provide resource-reading support. - - Loaders that support resource reading are expected to implement - the ``get_resource_reader(fullname)`` method and have it either return None - or an object compatible with this ABC. - """ + """Abstract base class for loaders to provide resource reading support.""" @abc.abstractmethod - def open_resource(self, resource): + def open_resource(self, resource: Text) -> BinaryIO: """Return an opened, file-like object for binary reading. - The 'resource' argument is expected to represent only a file name - and thus not contain any subdirectory components. - + The 'resource' argument is expected to represent only a file name. If the resource cannot be found, FileNotFoundError is raised. """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. raise FileNotFoundError @abc.abstractmethod - def resource_path(self, resource): + def resource_path(self, resource: Text) -> Text: """Return the file system path to the specified resource. - The 'resource' argument is expected to represent only a file name - and thus not contain any subdirectory components. - + The 'resource' argument is expected to represent only a file name. If the resource does not exist on the file system, raise FileNotFoundError. """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. raise FileNotFoundError @abc.abstractmethod - def is_resource(self, name): - """Return True if the named 'name' is consider a resource.""" + def is_resource(self, path: Text) -> bool: + """Return True if the named 'path' is a resource. + + Files are resources, directories are not. + """ raise FileNotFoundError @abc.abstractmethod - def contents(self): - """Return an iterable of strings over the contents of the package.""" - return [] - - -_register(ResourceReader, machinery.SourceFileLoader) + def contents(self) -> Iterable[str]: + """Return an iterable of entries in `package`.""" + raise FileNotFoundError @runtime_checkable @@ -402,26 +364,28 @@ class Traversable(Protocol): Yield Traversable objects in self """ - @abc.abstractmethod def read_bytes(self): """ Read contents of self as bytes """ + with self.open('rb') as strm: + return strm.read() - @abc.abstractmethod def read_text(self, encoding=None): """ - Read contents of self as bytes + Read contents of self as text """ + with self.open(encoding=encoding) as strm: + return strm.read() @abc.abstractmethod - def is_dir(self): + def is_dir(self) -> bool: """ Return True if self is a dir """ @abc.abstractmethod - def is_file(self): + def is_file(self) -> bool: """ Return True if self is a file """ @@ -432,11 +396,11 @@ class Traversable(Protocol): Return Traversable child in self """ - @abc.abstractmethod def __truediv__(self, child): """ Return Traversable child in self """ + return self.joinpath(child) @abc.abstractmethod def open(self, mode='r', *args, **kwargs): @@ -449,14 +413,18 @@ class Traversable(Protocol): """ @abc.abstractproperty - def name(self): - # type: () -> str + def name(self) -> str: """ The base name of this object without any parent references. """ class TraversableResources(ResourceReader): + """ + The required interface for providing traversable + resources. + """ + @abc.abstractmethod def files(self): """Return a Traversable object for the loaded package.""" @@ -468,7 +436,7 @@ class TraversableResources(ResourceReader): raise FileNotFoundError(resource) def is_resource(self, path): - return self.files().joinpath(path).isfile() + return self.files().joinpath(path).is_file() def contents(self): return (item.name for item in self.files().iterdir()) diff --git a/contrib/tools/python3/src/Lib/importlib/machinery.py b/contrib/tools/python3/src/Lib/importlib/machinery.py index 1b2b5c9b4f3..9a7757fb6e4 100644 --- a/contrib/tools/python3/src/Lib/importlib/machinery.py +++ b/contrib/tools/python3/src/Lib/importlib/machinery.py @@ -1,7 +1,5 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" -import _imp - from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter diff --git a/contrib/tools/python3/src/Lib/importlib/metadata.py b/contrib/tools/python3/src/Lib/importlib/metadata.py deleted file mode 100644 index 647fd3e4dbf..00000000000 --- a/contrib/tools/python3/src/Lib/importlib/metadata.py +++ /dev/null @@ -1,604 +0,0 @@ -import io -import os -import re -import abc -import csv -import sys -import email -import pathlib -import zipfile -import operator -import functools -import itertools -import posixpath -import collections - -from configparser import ConfigParser -from contextlib import suppress -from importlib import import_module -from importlib.abc import MetaPathFinder -from itertools import starmap - - -__all__ = [ - 'Distribution', - 'DistributionFinder', - 'PackageNotFoundError', - 'distribution', - 'distributions', - 'entry_points', - 'files', - 'metadata', - 'requires', - 'version', - ] - - -class PackageNotFoundError(ModuleNotFoundError): - """The package was not found.""" - - -class EntryPoint( - collections.namedtuple('EntryPointBase', 'name value group')): - """An entry point as defined by Python packaging conventions. - - See `the packaging docs on entry points - `_ - for more information. - - >>> ep = EntryPoint( - ... name=None, group=None, value='package.module:attr [extra1, extra2]') - >>> ep.module - 'package.module' - >>> ep.attr - 'attr' - >>> ep.extras - ['extra1', 'extra2'] - """ - - pattern = re.compile( - r'(?P[\w.]+)\s*' - r'(:\s*(?P[\w.]+)\s*)?' - r'((?P\[.*\])\s*)?$' - ) - """ - A regular expression describing the syntax for an entry point, - which might look like: - - - module - - package.module - - package.module:attribute - - package.module:object.attribute - - package.module:attr [extra1, extra2] - - Other combinations are possible as well. - - The expression is lenient about whitespace around the ':', - following the attr, and following any extras. - """ - - def load(self): - """Load the entry point from its definition. If only a module - is indicated by the value, return that module. Otherwise, - return the named object. - """ - match = self.pattern.match(self.value) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) - return functools.reduce(getattr, attrs, module) - - @property - def module(self): - match = self.pattern.match(self.value) - return match.group('module') - - @property - def attr(self): - match = self.pattern.match(self.value) - return match.group('attr') - - @property - def extras(self): - match = self.pattern.match(self.value) - return re.findall(r'\w+', match.group('extras') or '') - - @classmethod - def _from_config(cls, config): - return [ - cls(name, value, group) - for group in config.sections() - for name, value in config.items(group) - ] - - @classmethod - def _from_text(cls, text): - config = ConfigParser(delimiters='=') - # case sensitive: https://stackoverflow.com/q/1611799/812183 - config.optionxform = str - try: - config.read_string(text) - except AttributeError: # pragma: nocover - # Python 2 has no read_string - config.readfp(io.StringIO(text)) - return EntryPoint._from_config(config) - - def __iter__(self): - """ - Supply iter so one may construct dicts of EntryPoints easily. - """ - return iter((self.name, self)) - - def __reduce__(self): - return ( - self.__class__, - (self.name, self.value, self.group), - ) - - -class PackagePath(pathlib.PurePosixPath): - """A reference to a path in a package""" - - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() - - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() - - def locate(self): - """Return a path-like object for this path""" - return self.dist.locate_file(self) - - -class FileHash: - def __init__(self, spec): - self.mode, _, self.value = spec.partition('=') - - def __repr__(self): - return ''.format(self.mode, self.value) - - -class Distribution: - """A Python distribution package.""" - - @abc.abstractmethod - def read_text(self, filename): - """Attempt to load metadata file given by the name. - - :param filename: The name of the file in the distribution info. - :return: The text if found, otherwise None. - """ - - @abc.abstractmethod - def locate_file(self, path): - """ - Given a path to a file in this distribution, return a path - to it. - """ - - @classmethod - def from_name(cls, name): - """Return the Distribution for the given package name. - - :param name: The name of the distribution package to search for. - :return: The Distribution instance (or subclass thereof) for the named - package, if found. - :raises PackageNotFoundError: When the named package's distribution - metadata cannot be found. - """ - for resolver in cls._discover_resolvers(): - dists = resolver(DistributionFinder.Context(name=name)) - dist = next(iter(dists), None) - if dist is not None: - return dist - else: - raise PackageNotFoundError(name) - - @classmethod - def discover(cls, **kwargs): - """Return an iterable of Distribution objects for all packages. - - Pass a ``context`` or pass keyword arguments for constructing - a context. - - :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. - """ - context = kwargs.pop('context', None) - if context and kwargs: - raise ValueError("cannot accept context and kwargs") - context = context or DistributionFinder.Context(**kwargs) - return itertools.chain.from_iterable( - resolver(context) - for resolver in cls._discover_resolvers() - ) - - @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path - - :param path: a string or path-like object - :return: a concrete Distribution instance for the path - """ - return PathDistribution(pathlib.Path(path)) - - @staticmethod - def _discover_resolvers(): - """Search the meta_path for resolvers.""" - declared = ( - getattr(finder, 'find_distributions', None) - for finder in sys.meta_path - ) - return filter(None, declared) - - @classmethod - def _local(cls, root='.'): - from pep517 import build, meta - system = build.compat_system(root) - builder = functools.partial( - meta.build, - source_dir=root, - system=system, - ) - return PathDistribution(zipfile.Path(meta.build_as_zip(builder))) - - @property - def metadata(self): - """Return the parsed metadata for this Distribution. - - The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. - """ - text = ( - self.read_text('METADATA') - or self.read_text('PKG-INFO') - # This last clause is here to support old egg-info files. Its - # effect is to just end up using the PathDistribution's self._path - # (which points to the egg-info file) attribute unchanged. - or self.read_text('') - ) - return email.message_from_string(text) - - @property - def version(self): - """Return the 'Version' metadata for the distribution package.""" - return self.metadata['Version'] - - @property - def entry_points(self): - return EntryPoint._from_text(self.read_text('entry_points.txt')) - - @property - def files(self): - """Files in this distribution. - - :return: List of PackagePath for this distribution or None - - Result is `None` if the metadata file that enumerates files - (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is - missing. - Result may be empty if the metadata exists but is empty. - """ - file_lines = self._read_files_distinfo() or self._read_files_egginfo() - - def make_file(name, hash=None, size_str=None): - result = PackagePath(name) - result.hash = FileHash(hash) if hash else None - result.size = int(size_str) if size_str else None - result.dist = self - return result - - return file_lines and list(starmap(make_file, csv.reader(file_lines))) - - def _read_files_distinfo(self): - """ - Read the lines of RECORD - """ - text = self.read_text('RECORD') - return text and text.splitlines() - - def _read_files_egginfo(self): - """ - SOURCES.txt might contain literal commas, so wrap each line - in quotes. - """ - text = self.read_text('SOURCES.txt') - return text and map('"{}"'.format, text.splitlines()) - - @property - def requires(self): - """Generated requirements specified for this Distribution""" - reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() - return reqs and list(reqs) - - def _read_dist_info_reqs(self): - return self.metadata.get_all('Requires-Dist') - - def _read_egg_info_reqs(self): - source = self.read_text('requires.txt') - return None if source is None else self._deps_from_requires_text(source) - - @classmethod - def _deps_from_requires_text(cls, source): - section_pairs = cls._read_sections(source.splitlines()) - sections = { - section: list(map(operator.itemgetter('line'), results)) - for section, results in - itertools.groupby(section_pairs, operator.itemgetter('section')) - } - return cls._convert_egg_info_reqs_to_simple_reqs(sections) - - @staticmethod - def _read_sections(lines): - section = None - for line in filter(None, lines): - section_match = re.match(r'\[(.*)\]$', line) - if section_match: - section = section_match.group(1) - continue - yield locals() - - @staticmethod - def _convert_egg_info_reqs_to_simple_reqs(sections): - """ - Historically, setuptools would solicit and store 'extra' - requirements, including those with environment markers, - in separate sections. More modern tools expect each - dependency to be defined separately, with any relevant - extras and environment markers attached directly to that - requirement. This method converts the former to the - latter. See _test_deps_from_requires_text for an example. - """ - def make_condition(name): - return name and 'extra == "{name}"'.format(name=name) - - def quoted_marker(section): - section = section or '' - extra, sep, markers = section.partition(':') - if extra and markers: - markers = f'({markers})' - conditions = list(filter(None, [markers, make_condition(extra)])) - return '; ' + ' and '.join(conditions) if conditions else '' - - def url_req_space(req): - """ - PEP 508 requires a space between the url_spec and the quoted_marker. - Ref python/importlib_metadata#357. - """ - # '@' is uniquely indicative of a url_req. - return ' ' * ('@' in req) - - for section, deps in sections.items(): - for dep in deps: - space = url_req_space(dep) - yield dep + space + quoted_marker(section) - - -class DistributionFinder(MetaPathFinder): - """ - A MetaPathFinder capable of discovering installed distributions. - """ - - class Context: - """ - Keyword arguments presented by the caller to - ``distributions()`` or ``Distribution.discover()`` - to narrow the scope of a search for distributions - in all DistributionFinders. - - Each DistributionFinder may expect any parameters - and should attempt to honor the canonical - parameters defined below when appropriate. - """ - - name = None - """ - Specific name for which a distribution finder should match. - A name of ``None`` matches all distributions. - """ - - def __init__(self, **kwargs): - vars(self).update(kwargs) - - @property - def path(self): - """ - The path that a distribution finder should search. - - Typically refers to Python package paths and defaults - to ``sys.path``. - """ - return vars(self).get('path', sys.path) - - @abc.abstractmethod - def find_distributions(self, context=Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching the ``context``, - a DistributionFinder.Context instance. - """ - - -class FastPath: - """ - Micro-optimized class for searching a path for - children. - """ - - def __init__(self, root): - self.root = root - self.base = os.path.basename(self.root).lower() - - def joinpath(self, child): - return pathlib.Path(self.root, child) - - def children(self): - with suppress(Exception): - return os.listdir(self.root or '.') - with suppress(Exception): - return self.zip_children() - return [] - - def zip_children(self): - zip_path = zipfile.Path(self.root) - names = zip_path.root.namelist() - self.joinpath = zip_path.joinpath - - return dict.fromkeys( - child.split(posixpath.sep, 1)[0] - for child in names - ) - - def is_egg(self, search): - base = self.base - return ( - base == search.versionless_egg_name - or base.startswith(search.prefix) - and base.endswith('.egg')) - - def search(self, name): - for child in self.children(): - n_low = child.lower() - if (n_low in name.exact_matches - or n_low.startswith(name.prefix) - and n_low.endswith(name.suffixes) - # legacy case: - or self.is_egg(name) and n_low == 'egg-info'): - yield self.joinpath(child) - - -class Prepared: - """ - A prepared search for metadata on a possibly-named package. - """ - normalized = '' - prefix = '' - suffixes = '.dist-info', '.egg-info' - exact_matches = [''][:0] - versionless_egg_name = '' - - def __init__(self, name): - self.name = name - if name is None: - return - self.normalized = name.lower().replace('-', '_') - self.prefix = self.normalized + '-' - self.exact_matches = [ - self.normalized + suffix for suffix in self.suffixes] - self.versionless_egg_name = self.normalized + '.egg' - - -class MetadataPathFinder(DistributionFinder): - @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching ``context.name`` - (or all names if ``None`` indicated) along the paths in the list - of directories ``context.path``. - """ - found = cls._search_paths(context.name, context.path) - return map(PathDistribution, found) - - @classmethod - def _search_paths(cls, name, paths): - """Find metadata directories in paths heuristically.""" - return itertools.chain.from_iterable( - path.search(Prepared(name)) - for path in map(FastPath, paths) - ) - - -class PathDistribution(Distribution): - def __init__(self, path): - """Construct a distribution from a path to the metadata directory. - - :param path: A pathlib.Path or similar object supporting - .joinpath(), __div__, .parent, and .read_text(). - """ - self._path = path - - def read_text(self, filename): - with suppress(FileNotFoundError, IsADirectoryError, KeyError, - NotADirectoryError, PermissionError): - return self._path.joinpath(filename).read_text(encoding='utf-8') - read_text.__doc__ = Distribution.read_text.__doc__ - - def locate_file(self, path): - return self._path.parent / path - - -def distribution(distribution_name): - """Get the ``Distribution`` instance for the named package. - - :param distribution_name: The name of the distribution package as a string. - :return: A ``Distribution`` instance (or subclass thereof). - """ - return Distribution.from_name(distribution_name) - - -def distributions(**kwargs): - """Get all ``Distribution`` instances in the current environment. - - :return: An iterable of ``Distribution`` instances. - """ - return Distribution.discover(**kwargs) - - -def metadata(distribution_name): - """Get the metadata for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: An email.Message containing the parsed metadata. - """ - return Distribution.from_name(distribution_name).metadata - - -def version(distribution_name): - """Get the version string for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: The version string for the package as defined in the package's - "Version" metadata key. - """ - return distribution(distribution_name).version - - -def entry_points(): - """Return EntryPoint objects for all installed packages. - - :return: EntryPoint objects for all installed packages. - """ - eps = itertools.chain.from_iterable( - dist.entry_points for dist in distributions()) - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return { - group: tuple(eps) - for group, eps in grouped - } - - -def files(distribution_name): - """Return a list of files for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: List of files composing the distribution. - """ - return distribution(distribution_name).files - - -def requires(distribution_name): - """ - Return a list of requirements for the named package. - - :return: An iterator of requirements, suitable for - packaging.requirement.Requirement. - """ - return distribution(distribution_name).requires diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/__init__.py b/contrib/tools/python3/src/Lib/importlib/metadata/__init__.py new file mode 100644 index 00000000000..b3063cd9915 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/__init__.py @@ -0,0 +1,1045 @@ +import os +import re +import abc +import csv +import sys +import email +import pathlib +import zipfile +import operator +import textwrap +import warnings +import functools +import itertools +import posixpath +import collections + +from . import _adapters, _meta +from ._meta import PackageMetadata +from ._collections import FreezableDefaultDict, Pair +from ._functools import method_cache +from ._itertools import unique_everseen +from ._meta import PackageMetadata, SimplePath + +from contextlib import suppress +from importlib import import_module +from importlib.abc import MetaPathFinder +from itertools import starmap +from typing import List, Mapping, Optional, Union + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageMetadata', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'packages_distributions', + 'requires', + 'version', +] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + def __str__(self): + return f"No package metadata was found for {self.name}" + + @property + def name(self): + (name,) = self.args + return name + + +class Sectioned: + """ + A simple entry point config parser for performance + + >>> for item in Sectioned.read(Sectioned._sample): + ... print(item) + Pair(name='sec1', value='# comments ignored') + Pair(name='sec1', value='a = 1') + Pair(name='sec1', value='b = 2') + Pair(name='sec2', value='a = 2') + + >>> res = Sectioned.section_pairs(Sectioned._sample) + >>> item = next(res) + >>> item.name + 'sec1' + >>> item.value + Pair(name='a', value='1') + >>> item = next(res) + >>> item.value + Pair(name='b', value='2') + >>> item = next(res) + >>> item.name + 'sec2' + >>> item.value + Pair(name='a', value='2') + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + # comments ignored + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + @classmethod + def section_pairs(cls, text): + return ( + section._replace(value=Pair.parse(section.value)) + for section in cls.read(text, filter_=cls.valid) + if section.name is not None + ) + + @staticmethod + def read(text, filter_=None): + lines = filter(filter_, map(str.strip, text.splitlines())) + name = None + for value in lines: + section_match = value.startswith('[') and value.endswith(']') + if section_match: + name = value.strip('[]') + continue + yield Pair(name, value) + + @staticmethod + def valid(line): + return line and not line.startswith('#') + + +class EntryPoint( + collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + `_ + for more information. + + >>> ep = EntryPoint( + ... name=None, group=None, value='package.module:attr [extra1, extra2]') + >>> ep.module + 'package.module' + >>> ep.attr + 'attr' + >>> ep.extras + ['extra1', 'extra2'] + """ + + pattern = re.compile( + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+)\s*)?' + r'((?P\[.*\])\s*)?$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + dist: Optional['Distribution'] = None + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + + @property + def extras(self): + match = self.pattern.match(self.value) + return re.findall(r'\w+', match.group('extras') or '') + + def _for(self, dist): + self.dist = dist + return self + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints by name. + """ + msg = ( + "Construction of dict of EntryPoints is deprecated in " + "favor of EntryPoints." + ) + warnings.warn(msg, DeprecationWarning) + return iter((self.name, self)) + + def __reduce__(self): + return ( + self.__class__, + (self.name, self.value, self.group), + ) + + def matches(self, **params): + """ + EntryPoint matches the given parameters. + + >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]') + >>> ep.matches(group='foo') + True + >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]') + True + >>> ep.matches(group='foo', name='other') + False + >>> ep.matches() + True + >>> ep.matches(extras=['extra1', 'extra2']) + True + >>> ep.matches(module='bing') + True + >>> ep.matches(attr='bong') + True + """ + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + + +class DeprecatedList(list): + """ + Allow an otherwise immutable object to implement mutability + for compatibility. + + >>> recwarn = getfixture('recwarn') + >>> dl = DeprecatedList(range(3)) + >>> dl[0] = 1 + >>> dl.append(3) + >>> del dl[3] + >>> dl.reverse() + >>> dl.sort() + >>> dl.extend([4]) + >>> dl.pop(-1) + 4 + >>> dl.remove(1) + >>> dl += [5] + >>> dl + [6] + [1, 2, 5, 6] + >>> dl + (6,) + [1, 2, 5, 6] + >>> dl.insert(0, 0) + >>> dl + [0, 1, 2, 5] + >>> dl == [0, 1, 2, 5] + True + >>> dl == (0, 1, 2, 5) + True + >>> len(recwarn) + 1 + """ + + __slots__ = () + + _warn = functools.partial( + warnings.warn, + "EntryPoints list interface is deprecated. Cast to list if needed.", + DeprecationWarning, + stacklevel=2, + ) + + def __setitem__(self, *args, **kwargs): + self._warn() + return super().__setitem__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._warn() + return super().__delitem__(*args, **kwargs) + + def append(self, *args, **kwargs): + self._warn() + return super().append(*args, **kwargs) + + def reverse(self, *args, **kwargs): + self._warn() + return super().reverse(*args, **kwargs) + + def extend(self, *args, **kwargs): + self._warn() + return super().extend(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._warn() + return super().pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + self._warn() + return super().remove(*args, **kwargs) + + def __iadd__(self, *args, **kwargs): + self._warn() + return super().__iadd__(*args, **kwargs) + + def __add__(self, other): + if not isinstance(other, tuple): + self._warn() + other = tuple(other) + return self.__class__(tuple(self) + other) + + def insert(self, *args, **kwargs): + self._warn() + return super().insert(*args, **kwargs) + + def sort(self, *args, **kwargs): + self._warn() + return super().sort(*args, **kwargs) + + def __eq__(self, other): + if not isinstance(other, tuple): + self._warn() + other = tuple(other) + + return tuple(self).__eq__(other) + + +class EntryPoints(DeprecatedList): + """ + An immutable collection of selectable EntryPoint objects. + """ + + __slots__ = () + + def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ + if isinstance(name, int): + warnings.warn( + "Accessing entry points by index is deprecated. " + "Cast to tuple if needed.", + DeprecationWarning, + stacklevel=2, + ) + return super().__getitem__(name) + try: + return next(iter(self.select(name=name))) + except StopIteration: + raise KeyError(name) + + def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ + return EntryPoints(ep for ep in self if ep.matches(**params)) + + @property + def names(self): + """ + Return the set of all names of all entry points. + """ + return set(ep.name for ep in self) + + @property + def groups(self): + """ + Return the set of all groups of all entry points. + + For coverage while SelectableGroups is present. + >>> EntryPoints().groups + set() + """ + return set(ep.group for ep in self) + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @classmethod + def _from_text(cls, text): + return itertools.starmap(EntryPoint, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (item.value.name, item.value.value, item.name) + for item in Sectioned.section_pairs(text) + ) + + +class Deprecated: + """ + Compatibility add-in for mapping to indicate that + mapping behavior is deprecated. + + >>> recwarn = getfixture('recwarn') + >>> class DeprecatedDict(Deprecated, dict): pass + >>> dd = DeprecatedDict(foo='bar') + >>> dd.get('baz', None) + >>> dd['foo'] + 'bar' + >>> list(dd) + ['foo'] + >>> list(dd.keys()) + ['foo'] + >>> 'foo' in dd + True + >>> list(dd.values()) + ['bar'] + >>> len(recwarn) + 1 + """ + + _warn = functools.partial( + warnings.warn, + "SelectableGroups dict interface is deprecated. Use select.", + DeprecationWarning, + stacklevel=2, + ) + + def __getitem__(self, name): + self._warn() + return super().__getitem__(name) + + def get(self, name, default=None): + self._warn() + return super().get(name, default) + + def __iter__(self): + self._warn() + return super().__iter__() + + def __contains__(self, *args): + self._warn() + return super().__contains__(*args) + + def keys(self): + self._warn() + return super().keys() + + def values(self): + self._warn() + return super().values() + + +class SelectableGroups(Deprecated, dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def _all(self): + """ + Reconstruct a list of all entrypoints from the groups. + """ + groups = super(Deprecated, self).values() + return EntryPoints(itertools.chain.from_iterable(groups)) + + @property + def groups(self): + return self._all.groups + + @property + def names(self): + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return self._all.names + + def select(self, **params): + if not params: + return self + return self._all.select(**params) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return f'' + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(DistributionFinder.Context(name=name)) + dist = next(iter(dists), None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) for finder in sys.meta_path + ) + return filter(None, declared) + + @classmethod + def _local(cls, root='.'): + from pep517 import build, meta + + system = build.compat_system(root) + builder = functools.partial( + meta.build, + source_dir=root, + system=system, + ) + return PathDistribution(zipfile.Path(meta.build_as_zip(builder))) + + @property + def metadata(self) -> _meta.PackageMetadata: + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + return _adapters.Message(email.message_from_string(text)) + + @property + def name(self): + """Return the 'Name' metadata for the distribution package.""" + return self.metadata['Name'] + + @property + def _normalized_name(self): + """Return a normalized version of the name.""" + return Prepared.normalize(self.name) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is + missing. + Result may be empty if the metadata exists but is empty. + """ + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and list(starmap(make_file, csv.reader(file_lines))) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return None if source is None else self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + + def make_condition(name): + return name and f'extra == "{name}"' + + def quoted_marker(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = f'({markers})' + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + def url_req_space(req): + """ + PEP 508 requires a space between the url_spec and the quoted_marker. + Ref python/importlib_metadata#357. + """ + # '@' is uniquely indicative of a url_req. + return ' ' * ('@' in req) + + for section in sections: + space = url_req_space(section.value) + yield section.value + space + quoted_marker(section.name) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The sequence of directory path that a distribution finder + should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + """ + + @functools.lru_cache() # type: ignore + def __new__(cls, root): + return super().__new__(cls) + + def __init__(self, root): + self.root = root + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '.') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipfile.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) + + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with suppress(OSError): + return os.stat(self.root).st_mtime + self.lookup.cache_clear() + + @method_cache + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) + + for child in path.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos[normalized].append(path.joinpath(child)) + elif base_is_egg and low == "egg-info": + name = base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs[legacy_normalized].append(path.joinpath(child)) + + self.infos.freeze() + self.eggs.freeze() + + def search(self, prepared): + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + + normalized = None + legacy_normalized = None + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = self.normalize(name) + self.legacy_normalized = self.legacy_normalize(name) + + @staticmethod + def normalize(name): + """ + PEP 503 normalization plus dashes as underscores. + """ + return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') + + @staticmethod + def legacy_normalize(name): + """ + Normalize the package name as found in the convention in + older packaging tools versions and specs. + """ + return name.lower().replace('-', '_') + + def __bool__(self): + return bool(self.name) + + +class MetadataPathFinder(DistributionFinder): + @classmethod + def find_distributions(cls, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = cls._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + prepared = Prepared(name) + return itertools.chain.from_iterable( + path.search(prepared) for path in map(FastPath, paths) + ) + + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + + +class PathDistribution(Distribution): + def __init__(self, path: SimplePath): + """Construct a distribution. + + :param path: SimplePath indicating the metadata directory. + """ + self._path = path + + def read_text(self, filename): + with suppress( + FileNotFoundError, + IsADirectoryError, + KeyError, + NotADirectoryError, + PermissionError, + ): + return self._path.joinpath(filename).read_text(encoding='utf-8') + + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + @property + def _normalized_name(self): + """ + Performance optimization: where possible, resolve the + normalized name from the file system path. + """ + stem = os.path.basename(str(self._path)) + return self._name_from_stem(stem) or super()._normalized_name + + def _name_from_stem(self, stem): + name, ext = os.path.splitext(stem) + if ext not in ('.dist-info', '.egg-info'): + return + name, sep, rest = stem.partition('-') + return name + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name) -> _meta.PackageMetadata: + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: A PackageMetadata containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: + """Return EntryPoint objects for all installed packages. + + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + For compatibility, returns ``SelectableGroups`` object unless + selection parameters are supplied. In the future, this function + will return ``EntryPoints`` instead of ``SelectableGroups`` + even when no selection parameters are supplied. + + For maximum future compatibility, pass selection parameters + or invoke ``.select`` with parameters on the result. + + :return: EntryPoints or SelectableGroups for all installed packages. + """ + norm_name = operator.attrgetter('_normalized_name') + unique = functools.partial(unique_everseen, key=norm_name) + eps = itertools.chain.from_iterable( + dist.entry_points for dist in unique(distributions()) + ) + return SelectableGroups.load(eps).select(**params) + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> import collections.abc + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in (dist.read_text('top_level.txt') or '').split(): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_adapters.py b/contrib/tools/python3/src/Lib/importlib/metadata/_adapters.py new file mode 100644 index 00000000000..aa460d3eda5 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_adapters.py @@ -0,0 +1,68 @@ +import re +import textwrap +import email.message + +from ._text import FoldedCase + + +class Message(email.message.Message): + multiple_use_keys = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + 'Dynamic', + ], + ) + ) + """ + Keys that may be indicated multiple times per PEP 566. + """ + + def __new__(cls, orig: email.message.Message): + res = super().__new__(cls) + vars(res).update(vars(orig)) + return res + + def __init__(self, *args, **kwargs): + self._headers = self._repair_headers() + + # suppress spurious error from mypy + def __iter__(self): + return super().__iter__() + + def _repair_headers(self): + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + headers = [(key, redent(value)) for key, value in vars(self)['_headers']] + if self._payload: + headers.append(('Description', self.get_payload())) + return headers + + @property + def json(self): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + + def transform(key): + value = self.get_all(key) if key in self.multiple_use_keys else self[key] + if key == 'Keywords': + value = re.split(r'\s+', value) + tk = key.lower().replace('-', '_') + return tk, value + + return dict(map(transform, map(FoldedCase, self))) diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_collections.py b/contrib/tools/python3/src/Lib/importlib/metadata/_collections.py new file mode 100644 index 00000000000..cf0954e1a30 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_collections.py @@ -0,0 +1,30 @@ +import collections + + +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): + """ + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. + + >>> dd = FreezableDefaultDict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() + + +class Pair(collections.namedtuple('Pair', 'name value')): + @classmethod + def parse(cls, text): + return cls(*map(str.strip, text.split("=", 1))) diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_functools.py b/contrib/tools/python3/src/Lib/importlib/metadata/_functools.py new file mode 100644 index 00000000000..73f50d00bc0 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_itertools.py b/contrib/tools/python3/src/Lib/importlib/metadata/_itertools.py new file mode 100644 index 00000000000..dd45f2f0966 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_itertools.py @@ -0,0 +1,19 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_meta.py b/contrib/tools/python3/src/Lib/importlib/metadata/_meta.py new file mode 100644 index 00000000000..1a6edbf957d --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_meta.py @@ -0,0 +1,47 @@ +from typing import Any, Dict, Iterator, List, Protocol, TypeVar, Union + + +_T = TypeVar("_T") + + +class PackageMetadata(Protocol): + def __len__(self) -> int: + ... # pragma: no cover + + def __contains__(self, item: str) -> bool: + ... # pragma: no cover + + def __getitem__(self, key: str) -> str: + ... # pragma: no cover + + def __iter__(self) -> Iterator[str]: + ... # pragma: no cover + + def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: + """ + Return all values associated with a possibly multi-valued key. + """ + + @property + def json(self) -> Dict[str, Union[str, List[str]]]: + """ + A JSON-compatible form of the metadata. + """ + + +class SimplePath(Protocol): + """ + A minimal subset of pathlib.Path required by PathDistribution. + """ + + def joinpath(self) -> 'SimplePath': + ... # pragma: no cover + + def __div__(self) -> 'SimplePath': + ... # pragma: no cover + + def parent(self) -> 'SimplePath': + ... # pragma: no cover + + def read_text(self) -> str: + ... # pragma: no cover diff --git a/contrib/tools/python3/src/Lib/importlib/metadata/_text.py b/contrib/tools/python3/src/Lib/importlib/metadata/_text.py new file mode 100644 index 00000000000..766979d93c1 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/metadata/_text.py @@ -0,0 +1,99 @@ +import re + +from ._functools import method_cache + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super(FoldedCase, self).lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super(FoldedCase, self).lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) diff --git a/contrib/tools/python3/src/Lib/importlib/readers.py b/contrib/tools/python3/src/Lib/importlib/readers.py new file mode 100644 index 00000000000..41089c071d8 --- /dev/null +++ b/contrib/tools/python3/src/Lib/importlib/readers.py @@ -0,0 +1,123 @@ +import collections +import zipfile +import pathlib +from . import abc + + +def remove_duplicates(items): + return iter(collections.OrderedDict.fromkeys(items)) + + +class FileReader(abc.TraversableResources): + def __init__(self, loader): + self.path = pathlib.Path(loader.path).parent + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path + + +class ZipReader(abc.TraversableResources): + def __init__(self, loader, module): + _, _, name = module.rpartition('.') + self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.archive = loader.archive + + def open_resource(self, resource): + try: + return super().open_resource(resource) + except KeyError as exc: + raise FileNotFoundError(exc.args[0]) + + def is_resource(self, path): + # workaround for `zipfile.Path.is_file` returning true + # for non-existent paths. + target = self.files().joinpath(path) + return target.is_file() and target.exists() + + def files(self): + return zipfile.Path(self.archive, self.prefix) + + +class MultiplexedPath(abc.Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + + def __init__(self, *paths): + self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + if not self._paths: + message = 'MultiplexedPath must contain at least one path' + raise FileNotFoundError(message) + if not all(path.is_dir() for path in self._paths): + raise NotADirectoryError('MultiplexedPath only supports directories') + + def iterdir(self): + visited = [] + for path in self._paths: + for file in path.iterdir(): + if file.name in visited: + continue + visited.append(file.name) + yield file + + def read_bytes(self): + raise FileNotFoundError(f'{self} is not a file') + + def read_text(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + def is_dir(self): + return True + + def is_file(self): + return False + + def joinpath(self, child): + # first try to find child in current paths + for file in self.iterdir(): + if file.name == child: + return file + # if it does not exist, construct it with the first path + return self._paths[0] / child + + __truediv__ = joinpath + + def open(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + @property + def name(self): + return self._paths[0].name + + def __repr__(self): + paths = ', '.join(f"'{path}'" for path in self._paths) + return f'MultiplexedPath({paths})' + + +class NamespaceReader(abc.TraversableResources): + def __init__(self, namespace_path): + if 'NamespacePath' not in str(namespace_path): + raise ValueError('Invalid path') + self.path = MultiplexedPath(*list(namespace_path)) + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path diff --git a/contrib/tools/python3/src/Lib/importlib/resources.py b/contrib/tools/python3/src/Lib/importlib/resources.py index b803a01c91d..8a98663ff8e 100644 --- a/contrib/tools/python3/src/Lib/importlib/resources.py +++ b/contrib/tools/python3/src/Lib/importlib/resources.py @@ -1,22 +1,26 @@ import os +import io -from . import abc as resources_abc from . import _common -from ._common import as_file -from contextlib import contextmanager, suppress -from importlib import import_module +from ._common import as_file, files +from .abc import ResourceReader +from contextlib import suppress from importlib.abc import ResourceLoader +from importlib.machinery import ModuleSpec from io import BytesIO, TextIOWrapper from pathlib import Path from types import ModuleType -from typing import ContextManager, Iterable, Optional, Union +from typing import ContextManager, Iterable, Union from typing import cast from typing.io import BinaryIO, TextIO +from collections.abc import Sequence +from functools import singledispatch __all__ = [ 'Package', 'Resource', + 'ResourceReader', 'as_file', 'contents', 'files', @@ -26,99 +30,57 @@ __all__ = [ 'path', 'read_binary', 'read_text', - ] +] Package = Union[str, ModuleType] Resource = Union[str, os.PathLike] -def _resolve(name) -> ModuleType: - """If name is a string, resolve to a module.""" - if hasattr(name, '__spec__'): - return name - return import_module(name) - - -def _get_package(package) -> ModuleType: - """Take a package name or module object and return the module. - - If a name, the module is imported. If the resolved module - object is not a package, raise an exception. - """ - module = _resolve(package) - if module.__spec__.submodule_search_locations is None: - raise TypeError('{!r} is not a package'.format(package)) - return module - - -def _normalize_path(path) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - parent, file_name = os.path.split(path) - if parent: - raise ValueError('{!r} must be only a file name'.format(path)) - return file_name - - -def _get_resource_reader( - package: ModuleType) -> Optional[resources_abc.ResourceReader]: - # Return the package's loader if it's a ResourceReader. We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - if hasattr(spec.loader, 'get_resource_reader'): - return cast(resources_abc.ResourceReader, - spec.loader.get_resource_reader(spec.name)) - return None - - -def _check_location(package): - if package.__spec__.origin is None or not package.__spec__.has_location: - raise FileNotFoundError(f'Package has no location {package!r}') - - def open_binary(package: Package, resource: Resource) -> BinaryIO: """Return a file-like object opened for binary reading of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - reader = _get_resource_reader(package) + resource = _common.normalize_path(resource) + package = _common.get_package(package) + reader = _common.get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - absolute_package_path = os.path.abspath( - package.__spec__.origin or 'non-existent file') - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) - try: - return open(full_path, mode='rb') - except OSError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for - # get_data() will make it clear what is needed from the loader. - loader = cast(ResourceLoader, package.__spec__.loader) - data = None - if hasattr(package.__spec__.loader, 'get_data'): - with suppress(OSError): - data = loader.get_data(full_path) - if data is None: - package_name = package.__spec__.name - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - return BytesIO(data) - - -def open_text(package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict') -> TextIO: + spec = cast(ModuleSpec, package.__spec__) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + if spec.submodule_search_locations is not None: + paths = spec.submodule_search_locations + elif spec.origin is not None: + paths = [os.path.dirname(os.path.abspath(spec.origin))] + + for package_path in paths: + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, spec.loader) + data = None + if hasattr(spec.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is not None: + return BytesIO(data) + + raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}') + + +def open_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> TextIO: """Return a file-like object opened for text reading of the resource.""" return TextIOWrapper( - open_binary(package, resource), encoding=encoding, errors=errors) + open_binary(package, resource), encoding=encoding, errors=errors + ) def read_binary(package: Package, resource: Resource) -> bytes: @@ -127,10 +89,12 @@ def read_binary(package: Package, resource: Resource) -> bytes: return fp.read() -def read_text(package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict') -> str: +def read_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> str: """Return the decoded string of the resource. The decoding-related arguments have the same semantics as those of @@ -140,16 +104,10 @@ def read_text(package: Package, return fp.read() -def files(package: Package) -> resources_abc.Traversable: - """ - Get a Traversable resource from a package - """ - return _common.from_package(_get_package(package)) - - def path( - package: Package, resource: Resource, - ) -> 'ContextManager[Path]': + package: Package, + resource: Resource, +) -> 'ContextManager[Path]': """A context manager providing a file path object to the resource. If the resource does not already exist on its own on the file system, @@ -158,23 +116,30 @@ def path( raised if the file was deleted prior to the context manager exiting). """ - reader = _get_resource_reader(_get_package(package)) + reader = _common.get_resource_reader(_common.get_package(package)) return ( - _path_from_reader(reader, resource) - if reader else - _common.as_file(files(package).joinpath(_normalize_path(resource))) + _path_from_reader(reader, _common.normalize_path(resource)) + if reader + else _common.as_file( + _common.files(package).joinpath(_common.normalize_path(resource)) ) + ) -@contextmanager def _path_from_reader(reader, resource): - norm_resource = _normalize_path(resource) + return _path_from_resource_path(reader, resource) or _path_from_open_resource( + reader, resource + ) + + +def _path_from_resource_path(reader, resource): with suppress(FileNotFoundError): - yield Path(reader.resource_path(norm_resource)) - return - opener_reader = reader.open_resource(norm_resource) - with _common._tempfile(opener_reader.read, suffix=norm_resource) as res: - yield res + return Path(reader.resource_path(resource)) + + +def _path_from_open_resource(reader, resource): + saved = io.BytesIO(reader.open_resource(resource).read()) + return _common._tempfile(saved.read, suffix=resource) def is_resource(package: Package, name: str) -> bool: @@ -182,9 +147,9 @@ def is_resource(package: Package, name: str) -> bool: Directories are *not* resources. """ - package = _get_package(package) - _normalize_path(name) - reader = _get_resource_reader(package) + package = _common.get_package(package) + _common.normalize_path(name) + reader = _common.get_resource_reader(package) if reader is not None: return reader.is_resource(name) package_contents = set(contents(package)) @@ -200,16 +165,21 @@ def contents(package: Package) -> Iterable[str]: not considered resources. Use `is_resource()` on each entry returned here to check if it is a resource or not. """ - package = _get_package(package) - reader = _get_resource_reader(package) + package = _common.get_package(package) + reader = _common.get_resource_reader(package) if reader is not None: - return reader.contents() - # Is the package a namespace package? By definition, namespace packages - # cannot have resources. - namespace = ( - package.__spec__.origin is None or - package.__spec__.origin == 'namespace' - ) - if namespace or not package.__spec__.has_location: - return () - return list(item.name for item in _common.from_package(package).iterdir()) + return _ensure_sequence(reader.contents()) + transversable = _common.from_package(package) + if transversable.is_dir(): + return list(item.name for item in transversable.iterdir()) + return [] + + +@singledispatch +def _ensure_sequence(iterable): + return list(iterable) + + +@_ensure_sequence.register(Sequence) +def _(iterable): + return iterable diff --git a/contrib/tools/python3/src/Lib/importlib/util.py b/contrib/tools/python3/src/Lib/importlib/util.py index 269a6fa930a..8623c89840c 100644 --- a/contrib/tools/python3/src/Lib/importlib/util.py +++ b/contrib/tools/python3/src/Lib/importlib/util.py @@ -1,5 +1,5 @@ """Utility code for constructing importers, etc.""" -from . import abc +from ._abc import Loader from ._bootstrap import module_from_spec from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader @@ -149,7 +149,8 @@ def set_package(fxn): """ @functools.wraps(fxn) def set_package_wrapper(*args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) module = fxn(*args, **kwargs) if getattr(module, '__package__', None) is None: @@ -168,7 +169,8 @@ def set_loader(fxn): """ @functools.wraps(fxn) def set_loader_wrapper(self, *args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) module = fxn(self, *args, **kwargs) if getattr(module, '__loader__', None) is None: @@ -195,7 +197,8 @@ def module_for_loader(fxn): the second argument. """ - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) @functools.wraps(fxn) def module_for_loader_wrapper(self, fullname, *args, **kwargs): @@ -232,7 +235,6 @@ class _LazyModule(types.ModuleType): # Figure out exactly what attributes were mutated between the creation # of the module and now. attrs_then = self.__spec__.loader_state['__dict__'] - original_type = self.__spec__.loader_state['__class__'] attrs_now = self.__dict__ attrs_updated = {} for key, value in attrs_now.items(): @@ -263,7 +265,7 @@ class _LazyModule(types.ModuleType): delattr(self, attr) -class LazyLoader(abc.Loader): +class LazyLoader(Loader): """A loader that creates a module which defers loading until attribute access.""" diff --git a/contrib/tools/python3/src/Lib/inspect.py b/contrib/tools/python3/src/Lib/inspect.py index 6f91435541b..c5881cc808d 100644 --- a/contrib/tools/python3/src/Lib/inspect.py +++ b/contrib/tools/python3/src/Lib/inspect.py @@ -24,6 +24,8 @@ Here are some of the useful functions provided by this module: stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable + + get_annotations() - safely compute an object's annotations """ # This module is in the public domain. No warranties. @@ -60,6 +62,122 @@ for k, v in dis.COMPILER_FLAG_NAMES.items(): # See Include/object.h TPFLAGS_IS_ABSTRACT = 1 << 20 + +def get_annotations(obj, *, globals=None, locals=None, eval_str=False): + """Compute the annotations dict for an object. + + obj may be a callable, class, or module. + Passing in an object of any other type raises TypeError. + + Returns a dict. get_annotations() returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if isinstance(obj, type): + # class + obj_dict = getattr(obj, '__dict__', None) + if obj_dict and hasattr(obj_dict, 'get'): + ann = obj_dict.get('__annotations__', None) + if isinstance(ann, types.GetSetDescriptorType): + ann = None + else: + ann = None + + obj_globals = None + module_name = getattr(obj, '__module__', None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, '__dict__', None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__dict__') + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__globals__', None) + obj_locals = None + unwrap = obj + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if unwrap is not None: + while True: + if hasattr(unwrap, '__wrapped__'): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + return_value = {key: + value if not isinstance(value, str) else eval(value, globals, locals) + for key, value in ann.items() } + return return_value + + # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module. @@ -407,7 +525,7 @@ def classify_class_attrs(cls): # attribute with the same name as a DynamicClassAttribute exists. for base in mro: for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): + if isinstance(v, types.DynamicClassAttribute) and v.fget is not None: names.append(k) result = [] processed = set() @@ -663,6 +781,8 @@ def getfile(object): module = sys.modules.get(object.__module__) if getattr(module, '__file__', None): return module.__file__ + if object.__module__ == '__main__': + raise OSError('source code not available') raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): object = object.__func__ @@ -706,10 +826,13 @@ def getsourcefile(object): if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + module = getmodule(object, filename) + if getattr(module, '__loader__', None) is not None: + return filename + elif getattr(getattr(module, "__spec__", None), "loader", None) is not None: return filename # or it is in the linecache - if filename in linecache.cache: + elif filename in linecache.cache: return filename def getabsfile(object, _filename=None): @@ -1162,7 +1285,8 @@ def getfullargspec(func): sig = _signature_from_callable(func, follow_wrapper_chains=False, skip_bound_arg=False, - sigcls=Signature) + sigcls=Signature, + eval_str=False) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1897,7 +2021,7 @@ def _signature_is_functionlike(obj): isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and (kwdefaults is None or isinstance(kwdefaults, dict)) and - isinstance(annotations, dict)) + (isinstance(annotations, (dict)) or annotations is None) ) def _signature_get_bound_param(spec): @@ -2149,7 +2273,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): return _signature_fromstr(cls, func, s, skip_bound_arg) -def _signature_from_function(cls, func, skip_bound_arg=True): +def _signature_from_function(cls, func, skip_bound_arg=True, + globals=None, locals=None, eval_str=False): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2175,7 +2300,7 @@ def _signature_from_function(cls, func, skip_bound_arg=True): positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = func.__annotations__ + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2246,6 +2371,9 @@ def _signature_from_function(cls, func, skip_bound_arg=True): def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, + globals=None, + locals=None, + eval_str=False, sigcls): """Private helper function to get signature for arbitrary @@ -2255,7 +2383,10 @@ def _signature_from_callable(obj, *, _get_signature_of = functools.partial(_signature_from_callable, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + globals=globals, + locals=locals, + sigcls=sigcls, + eval_str=eval_str) if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) @@ -2323,7 +2454,8 @@ def _signature_from_callable(obj, *, # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, - skip_bound_arg=skip_bound_arg) + skip_bound_arg=skip_bound_arg, + globals=globals, locals=locals, eval_str=eval_str) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, @@ -2857,10 +2989,12 @@ class Signature: return _signature_from_builtin(cls, func) @classmethod - def from_callable(cls, obj, *, follow_wrapped=True): + def from_callable(cls, obj, *, + follow_wrapped=True, globals=None, locals=None, eval_str=False): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, - follow_wrapper_chains=follow_wrapped) + follow_wrapper_chains=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str) @property def parameters(self): @@ -3108,9 +3242,10 @@ class Signature: return rendered -def signature(obj, *, follow_wrapped=True): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): """Get a signature object for the passed callable.""" - return Signature.from_callable(obj, follow_wrapped=follow_wrapped) + return Signature.from_callable(obj, follow_wrapped=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str) def _main(): diff --git a/contrib/tools/python3/src/Lib/io.py b/contrib/tools/python3/src/Lib/io.py index fbce6efc010..2a6140c3dd5 100644 --- a/contrib/tools/python3/src/Lib/io.py +++ b/contrib/tools/python3/src/Lib/io.py @@ -54,9 +54,24 @@ import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, - IncrementalNewlineDecoder, TextIOWrapper) + IncrementalNewlineDecoder, text_encoding, TextIOWrapper) + + +def __getattr__(name): + if name == "OpenWrapper": + # bpo-43680: Until Python 3.9, _pyio.open was not a static method and + # builtins.open was set to OpenWrapper to not become a bound method + # when set to a class variable. _io.open is a built-in function whereas + # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now + # a static method, and builtins.open() is now io.open(). + import warnings + warnings.warn('OpenWrapper is deprecated, use open instead', + DeprecationWarning, stacklevel=2) + global OpenWrapper + OpenWrapper = open + return OpenWrapper + raise AttributeError(name) -OpenWrapper = _io.open # for compatibility with _pyio # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" diff --git a/contrib/tools/python3/src/Lib/ipaddress.py b/contrib/tools/python3/src/Lib/ipaddress.py index 6cb92ed5520..4a6496a5da3 100644 --- a/contrib/tools/python3/src/Lib/ipaddress.py +++ b/contrib/tools/python3/src/Lib/ipaddress.py @@ -16,6 +16,7 @@ import functools IPV4LENGTH = 32 IPV6LENGTH = 128 + class AddressValueError(ValueError): """A Value Error related to the address.""" @@ -1214,7 +1215,7 @@ class _BaseV4: """ if not octet_str: raise ValueError("Empty octet not permitted") - # Whitelist the characters, since int() allows a lot of bizarre stuff. + # Reject non-ASCII digits. if not (octet_str.isascii() and octet_str.isdigit()): msg = "Only decimal digits permitted in %r" raise ValueError(msg % octet_str) @@ -1724,7 +1725,7 @@ class _BaseV6: [0..FFFF]. """ - # Whitelist the characters, since int() allows a lot of bizarre stuff. + # Reject non-ASCII digits. if not cls._HEX_DIGITS.issuperset(hextet_str): raise ValueError("Only hex digits permitted in %r" % hextet_str) # We do the length check second, since the invalid character error @@ -2002,9 +2003,13 @@ class IPv6Address(_BaseV6, _BaseAddress): Returns: A boolean, True if the address is reserved per - iana-ipv6-special-registry. + iana-ipv6-special-registry, or is ipv4_mapped and is + reserved in the iana-ipv4-special-registry. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_private return any(self in net for net in self._constants._private_networks) @property diff --git a/contrib/tools/python3/src/Lib/keyword.py b/contrib/tools/python3/src/Lib/keyword.py index 59fcfb0f0d1..cc2b46b7229 100644 --- a/contrib/tools/python3/src/Lib/keyword.py +++ b/contrib/tools/python3/src/Lib/keyword.py @@ -6,7 +6,7 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: PYTHONPATH=Tools/peg_generator python3 -m pegen.keywordgen \ - Grammar/Grammar \ + Grammar/python.gram \ Grammar/Tokens \ Lib/keyword.py @@ -19,7 +19,6 @@ kwlist = [ 'False', 'None', 'True', - '__peg_parser__', 'and', 'as', 'assert', @@ -55,7 +54,9 @@ kwlist = [ ] softkwlist = [ - + '_', + 'case', + 'match' ] iskeyword = frozenset(kwlist).__contains__ diff --git a/contrib/tools/python3/src/Lib/lib2to3/pgen2/pgen.py b/contrib/tools/python3/src/Lib/lib2to3/pgen2/pgen.py index b0cbd16c4da..7abd5cef1c3 100644 --- a/contrib/tools/python3/src/Lib/lib2to3/pgen2/pgen.py +++ b/contrib/tools/python3/src/Lib/lib2to3/pgen2/pgen.py @@ -12,7 +12,7 @@ class ParserGenerator(object): def __init__(self, filename, stream=None): close_stream = None if stream is None: - stream = open(filename) + stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename self.stream = stream diff --git a/contrib/tools/python3/src/Lib/linecache.py b/contrib/tools/python3/src/Lib/linecache.py index 6adce80e8f8..c28c33f7fd8 100644 --- a/contrib/tools/python3/src/Lib/linecache.py +++ b/contrib/tools/python3/src/Lib/linecache.py @@ -177,9 +177,14 @@ def lazycache(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return False # Try for a __loader__, if available - if module_globals and '__loader__' in module_globals: - name = module_globals.get('__name__') - loader = module_globals['__loader__'] + if module_globals and '__name__' in module_globals: + name = module_globals['__name__'] + if (loader := module_globals.get('__loader__')) is None: + if spec := module_globals.get('__spec__'): + try: + loader = spec.loader + except AttributeError: + pass get_source = getattr(loader, 'get_source', None) if name and get_source: diff --git a/contrib/tools/python3/src/Lib/locale.py b/contrib/tools/python3/src/Lib/locale.py index 1a4e9f694f3..6d4f5192992 100644 --- a/contrib/tools/python3/src/Lib/locale.py +++ b/contrib/tools/python3/src/Lib/locale.py @@ -185,8 +185,14 @@ def _format(percent, value, grouping=False, monetary=False, *additional): formatted = percent % ((value,) + additional) else: formatted = percent % value + if percent[-1] in 'eEfFgGdiu': + formatted = _localize(formatted, grouping, monetary) + return formatted + +# Transform formatted as locale number according to the locale settings +def _localize(formatted, grouping=False, monetary=False): # floats and decimal ints need special action! - if percent[-1] in 'eEfFgG': + if '.' in formatted: seps = 0 parts = formatted.split('.') if grouping: @@ -196,7 +202,7 @@ def _format(percent, value, grouping=False, monetary=False, *additional): formatted = decimal_point.join(parts) if seps: formatted = _strip_padding(formatted, seps) - elif percent[-1] in 'diu': + else: seps = 0 if grouping: formatted, seps = _group(formatted, monetary=monetary) @@ -267,7 +273,7 @@ def currency(val, symbol=True, grouping=False, international=False): raise ValueError("Currency formatting is not possible using " "the 'C' locale.") - s = _format('%%.%if' % digits, abs(val), grouping, monetary=True) + s = _localize(f'{abs(val):.{digits}f}', grouping, monetary=True) # '<' and '>' are markers if the sign must be inserted between symbol and value s = '<' + s + '>' @@ -323,6 +329,10 @@ def delocalize(string): string = string.replace(dd, '.') return string +def localize(string, grouping=False, monetary=False): + """Parses a string as locale number according to the locale settings.""" + return _localize(string, grouping, monetary) + def atof(string, func=float): "Parses a string as a float according to the locale settings." return func(delocalize(string)) @@ -619,53 +629,49 @@ def resetlocale(category=LC_ALL): """ _setlocale(category, _build_localename(getdefaultlocale())) -if sys.platform.startswith("win"): - # On Win32, this will return the ANSI code page - def getpreferredencoding(do_setlocale = True): - """Return the charset that the user is likely using.""" + +try: + from _locale import _get_locale_encoding +except ImportError: + def _get_locale_encoding(): + if hasattr(sys, 'getandroidapilevel'): + # On Android langinfo.h and CODESET are missing, and UTF-8 is + # always used in mbstowcs() and wcstombs(). + return 'UTF-8' if sys.flags.utf8_mode: return 'UTF-8' - import _bootlocale - return _bootlocale.getpreferredencoding(False) + encoding = getdefaultlocale()[1] + if encoding is None: + # LANG not set, default conservatively to ASCII + encoding = 'ascii' + return encoding + +try: + CODESET +except NameError: + def getpreferredencoding(do_setlocale=True): + """Return the charset that the user is likely using.""" + return _get_locale_encoding() else: # On Unix, if CODESET is available, use that. - try: - CODESET - except NameError: - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - def getpreferredencoding(do_setlocale = True): - return 'UTF-8' - else: - # Fall back to parsing environment variables :-( - def getpreferredencoding(do_setlocale = True): - """Return the charset that the user is likely using, - by looking at environment variables.""" - if sys.flags.utf8_mode: - return 'UTF-8' - res = getdefaultlocale()[1] - if res is None: - # LANG not set, default conservatively to ASCII - res = 'ascii' - return res - else: - def getpreferredencoding(do_setlocale = True): - """Return the charset that the user is likely using, - according to the system configuration.""" - if sys.flags.utf8_mode: - return 'UTF-8' - import _bootlocale - if do_setlocale: - oldloc = setlocale(LC_CTYPE) - try: - setlocale(LC_CTYPE, "") - except Error: - pass - result = _bootlocale.getpreferredencoding(False) - if do_setlocale: - setlocale(LC_CTYPE, oldloc) - return result + def getpreferredencoding(do_setlocale=True): + """Return the charset that the user is likely using, + according to the system configuration.""" + if sys.flags.utf8_mode: + return 'UTF-8' + + if not do_setlocale: + return _get_locale_encoding() + + old_loc = setlocale(LC_CTYPE) + try: + try: + setlocale(LC_CTYPE, "") + except Error: + pass + return _get_locale_encoding() + finally: + setlocale(LC_CTYPE, old_loc) ### Database diff --git a/contrib/tools/python3/src/Lib/logging/__init__.py b/contrib/tools/python3/src/Lib/logging/__init__.py index 1ab35a8c21a..19bd2bc20b2 100644 --- a/contrib/tools/python3/src/Lib/logging/__init__.py +++ b/contrib/tools/python3/src/Lib/logging/__init__.py @@ -198,7 +198,8 @@ def _checkLevel(level): raise ValueError("Unknown level: %r" % level) rv = _nameToLevel[level] else: - raise TypeError("Level not an integer or a valid string: %r" % level) + raise TypeError("Level not an integer or a valid string: %r" + % (level,)) return rv #--------------------------------------------------------------------------- @@ -415,8 +416,9 @@ class PercentStyle(object): asctime_search = '%(asctime)' validation_pattern = re.compile(r'%\(\w+\)[#0+ -]*(\*|\d+)?(\.(\*|\d+))?[diouxefgcrsa%]', re.I) - def __init__(self, fmt): + def __init__(self, fmt, *, defaults=None): self._fmt = fmt or self.default_format + self._defaults = defaults def usesTime(self): return self._fmt.find(self.asctime_search) >= 0 @@ -427,7 +429,11 @@ class PercentStyle(object): raise ValueError("Invalid format '%s' for '%s' style" % (self._fmt, self.default_format[0])) def _format(self, record): - return self._fmt % record.__dict__ + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._fmt % values def format(self, record): try: @@ -445,7 +451,11 @@ class StrFormatStyle(PercentStyle): field_spec = re.compile(r'^(\d+|\w+)(\.\w+|\[[^]]+\])*$') def _format(self, record): - return self._fmt.format(**record.__dict__) + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._fmt.format(**values) def validate(self): """Validate the input format, ensure it is the correct string formatting style""" @@ -471,8 +481,8 @@ class StringTemplateStyle(PercentStyle): asctime_format = '${asctime}' asctime_search = '${asctime}' - def __init__(self, fmt): - self._fmt = fmt or self.default_format + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._tpl = Template(self._fmt) def usesTime(self): @@ -494,7 +504,11 @@ class StringTemplateStyle(PercentStyle): raise ValueError('invalid format: no fields') def _format(self, record): - return self._tpl.substitute(**record.__dict__) + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._tpl.substitute(**values) BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" @@ -550,7 +564,8 @@ class Formatter(object): converter = time.localtime - def __init__(self, fmt=None, datefmt=None, style='%', validate=True): + def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, + defaults=None): """ Initialize the formatter with specified format strings. @@ -569,7 +584,7 @@ class Formatter(object): if style not in _STYLES: raise ValueError('Style must be one of: %s' % ','.join( _STYLES.keys())) - self._style = _STYLES[style][0](fmt) + self._style = _STYLES[style][0](fmt, defaults=defaults) if validate: self._style.validate() @@ -863,6 +878,7 @@ class Handler(Filterer): self._name = None self.level = _checkLevel(level) self.formatter = None + self._closed = False # Add the handler to the global _handlerList (for cleanup on shutdown) _addHandlerRef(self) self.createLock() @@ -981,6 +997,7 @@ class Handler(Filterer): #get the module data lock, as we're updating a shared structure. _acquireLock() try: #unlikely to raise an exception, but you never know... + self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] finally: @@ -1135,8 +1152,14 @@ class FileHandler(StreamHandler): self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + if "b" not in mode: + self.encoding = io.text_encoding(encoding) self.errors = errors self.delay = delay + # bpo-26789: FileHandler keeps a reference to the builtin open() + # function to be able to open or reopen the file during Python + # finalization. + self._builtin_open = open if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. @@ -1163,6 +1186,8 @@ class FileHandler(StreamHandler): finally: # Issue #19523: call unconditionally to # prevent a handler leak when delay is set + # Also see Issue #42378: we also rely on + # self._closed being set to True there StreamHandler.close(self) finally: self.release() @@ -1172,8 +1197,9 @@ class FileHandler(StreamHandler): Open the current base file with the (original) mode and encoding. Return the resulting stream. """ - return open(self.baseFilename, self.mode, encoding=self.encoding, - errors=self.errors) + open_func = self._builtin_open + return open_func(self.baseFilename, self.mode, + encoding=self.encoding, errors=self.errors) def emit(self, record): """ @@ -1181,10 +1207,15 @@ class FileHandler(StreamHandler): If the stream was not opened because 'delay' was specified in the constructor, open it before calling the superclass's emit. + + If stream is not open, current mode is 'w' and `_closed=True`, record + will not be emitted (see Issue #42378). """ if self.stream is None: - self.stream = self._open() - StreamHandler.emit(self, record) + if self.mode != 'w' or not self._closed: + self.stream = self._open() + if self.stream: + StreamHandler.emit(self, record) def __repr__(self): level = getLevelName(self.level) @@ -1492,7 +1523,11 @@ class Logger(Filterer): if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) - fatal = critical + def fatal(self, msg, *args, **kwargs): + """ + Don't use this method, use critical() instead. + """ + self.critical(msg, *args, **kwargs) def log(self, level, msg, *args, **kwargs): """ @@ -1763,7 +1798,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra): + def __init__(self, logger, extra=None): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1998,8 +2033,10 @@ def basicConfig(**kwargs): filename = kwargs.pop("filename", None) mode = kwargs.pop("filemode", 'a') if filename: - if 'b'in mode: + if 'b' in mode: errors = None + else: + encoding = io.text_encoding(encoding) h = FileHandler(filename, mode, encoding=encoding, errors=errors) else: @@ -2051,7 +2088,11 @@ def critical(msg, *args, **kwargs): basicConfig() root.critical(msg, *args, **kwargs) -fatal = critical +def fatal(msg, *args, **kwargs): + """ + Don't use this function, use critical() instead. + """ + critical(msg, *args, **kwargs) def error(msg, *args, **kwargs): """ diff --git a/contrib/tools/python3/src/Lib/logging/config.py b/contrib/tools/python3/src/Lib/logging/config.py index fd3aded7608..3bc63b78621 100644 --- a/contrib/tools/python3/src/Lib/logging/config.py +++ b/contrib/tools/python3/src/Lib/logging/config.py @@ -48,7 +48,7 @@ RESET_ERROR = errno.ECONNRESET # _listener holds the server object doing the listening _listener = None -def fileConfig(fname, defaults=None, disable_existing_loggers=True): +def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None): """ Read the logging configuration from a ConfigParser-format file. @@ -66,7 +66,8 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True): if hasattr(fname, 'readline'): cp.read_file(fname) else: - cp.read(fname) + encoding = io.text_encoding(encoding) + cp.read(fname, encoding=encoding) formatters = _create_formatters(cp) diff --git a/contrib/tools/python3/src/Lib/logging/handlers.py b/contrib/tools/python3/src/Lib/logging/handlers.py index 572e370110a..61a39958c0a 100644 --- a/contrib/tools/python3/src/Lib/logging/handlers.py +++ b/contrib/tools/python3/src/Lib/logging/handlers.py @@ -23,7 +23,7 @@ Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ -import logging, socket, os, pickle, struct, time, re +import io, logging, socket, os, pickle, struct, time, re from stat import ST_DEV, ST_INO, ST_MTIME import queue import threading @@ -150,6 +150,8 @@ class RotatingFileHandler(BaseRotatingHandler): # on each run. if maxBytes > 0: mode = 'a' + if "b" not in mode: + encoding = io.text_encoding(encoding) BaseRotatingHandler.__init__(self, filename, mode, encoding=encoding, delay=delay, errors=errors) self.maxBytes = maxBytes @@ -208,6 +210,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler): def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None, errors=None): + encoding = io.text_encoding(encoding) BaseRotatingHandler.__init__(self, filename, 'a', encoding=encoding, delay=delay, errors=errors) self.when = when.upper() @@ -467,6 +470,8 @@ class WatchedFileHandler(logging.FileHandler): """ def __init__(self, filename, mode='a', encoding=None, delay=False, errors=None): + if "b" not in mode: + encoding = io.text_encoding(encoding) logging.FileHandler.__init__(self, filename, mode=mode, encoding=encoding, delay=delay, errors=errors) @@ -1167,7 +1172,7 @@ class NTEventLogHandler(logging.Handler): class HTTPHandler(logging.Handler): """ - A class which sends records to a Web server, using either GET or + A class which sends records to a web server, using either GET or POST semantics. """ def __init__(self, host, url, method="GET", secure=False, credentials=None, @@ -1216,7 +1221,7 @@ class HTTPHandler(logging.Handler): """ Emit a record. - Send the record to the Web server as a percent-encoded dictionary + Send the record to the web server as a percent-encoded dictionary """ try: import urllib.parse diff --git a/contrib/tools/python3/src/Lib/lzma.py b/contrib/tools/python3/src/Lib/lzma.py index 0aa30fe87f8..800f52198fb 100644 --- a/contrib/tools/python3/src/Lib/lzma.py +++ b/contrib/tools/python3/src/Lib/lzma.py @@ -310,6 +310,7 @@ def open(filename, mode="rb", *, preset=preset, filters=filters) if "t" in mode: + encoding = io.text_encoding(encoding) return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file diff --git a/contrib/tools/python3/src/Lib/mimetypes.py b/contrib/tools/python3/src/Lib/mimetypes.py index 58ace01b41a..b72ce083a69 100644 --- a/contrib/tools/python3/src/Lib/mimetypes.py +++ b/contrib/tools/python3/src/Lib/mimetypes.py @@ -27,6 +27,12 @@ import os import sys import posixpath import urllib.parse + +try: + from _winapi import _mimetypes_read_windows_registry +except ImportError: + _mimetypes_read_windows_registry = None + try: import winreg as _winreg except ImportError: @@ -235,10 +241,21 @@ class MimeTypes: types. """ - # Windows only - if not _winreg: + if not _mimetypes_read_windows_registry and not _winreg: return + add_type = self.add_type + if strict: + add_type = lambda type, ext: self.add_type(type, ext, True) + + # Accelerated function if it is available + if _mimetypes_read_windows_registry: + _mimetypes_read_windows_registry(add_type) + elif _winreg: + self._read_windows_registry(add_type) + + @classmethod + def _read_windows_registry(cls, add_type): def enum_types(mimedb): i = 0 while True: @@ -263,7 +280,7 @@ class MimeTypes: subkey, 'Content Type') if datatype != _winreg.REG_SZ: continue - self.add_type(mimetype, subkeyname, strict) + add_type(mimetype, subkeyname) except OSError: continue @@ -347,8 +364,8 @@ def init(files=None): if files is None or _db is None: db = MimeTypes() - if _winreg: - db.read_windows_registry() + # Quick return if not supported + db.read_windows_registry() if files is None: files = knownfiles @@ -446,6 +463,7 @@ def _default_mime_types(): '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', + '.h5' : 'application/x-hdf5', '.latex' : 'application/x-latex', '.mif' : 'application/x-mif', '.cdf' : 'application/x-netcdf', @@ -478,10 +496,19 @@ def _default_mime_types(): '.wsdl' : 'application/xml', '.xpdl' : 'application/xml', '.zip' : 'application/zip', + '.3gp' : 'audio/3gpp', + '.3gpp' : 'audio/3gpp', + '.3g2' : 'audio/3gpp2', + '.3gpp2' : 'audio/3gpp2', + '.aac' : 'audio/aac', + '.adts' : 'audio/aac', + '.loas' : 'audio/aac', + '.ass' : 'audio/aac', '.au' : 'audio/basic', '.snd' : 'audio/basic', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', + '.opus' : 'audio/opus', '.aif' : 'audio/x-aiff', '.aifc' : 'audio/x-aiff', '.aiff' : 'audio/x-aiff', @@ -493,6 +520,8 @@ def _default_mime_types(): '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', + '.heic' : 'image/heic', + '.heif' : 'image/heif', '.png' : 'image/png', '.svg' : 'image/svg+xml', '.tiff' : 'image/tiff', diff --git a/contrib/tools/python3/src/Lib/multiprocessing/managers.py b/contrib/tools/python3/src/Lib/multiprocessing/managers.py index dfa566c6fc3..b6b4cdd9ac1 100644 --- a/contrib/tools/python3/src/Lib/multiprocessing/managers.py +++ b/contrib/tools/python3/src/Lib/multiprocessing/managers.py @@ -193,11 +193,8 @@ class Server(object): t.daemon = True t.start() - def handle_request(self, c): - ''' - Handle a new connection - ''' - funcname = result = request = None + def _handle_request(self, c): + request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) @@ -214,6 +211,7 @@ class Server(object): msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) + try: c.send(msg) except Exception as e: @@ -225,7 +223,17 @@ class Server(object): util.info(' ... request was %r', request) util.info(' ... exception was %r', e) - c.close() + def handle_request(self, conn): + ''' + Handle a new connection + ''' + try: + self._handle_request(conn) + except SystemExit: + # Server.serve_client() calls sys.exit(0) on EOF + pass + finally: + conn.close() def serve_client(self, conn): ''' diff --git a/contrib/tools/python3/src/Lib/multiprocessing/resource_tracker.py b/contrib/tools/python3/src/Lib/multiprocessing/resource_tracker.py index c9bfa9b82b6..cc42dbdda05 100644 --- a/contrib/tools/python3/src/Lib/multiprocessing/resource_tracker.py +++ b/contrib/tools/python3/src/Lib/multiprocessing/resource_tracker.py @@ -37,8 +37,16 @@ if os.name == 'posix': import _multiprocessing import _posixshmem + # Use sem_unlink() to clean up named semaphores. + # + # sem_unlink() may be missing if the Python build process detected the + # absence of POSIX named semaphores. In that case, no named semaphores were + # ever opened, so no cleanup would be necessary. + if hasattr(_multiprocessing, 'sem_unlink'): + _CLEANUP_FUNCS.update({ + 'semaphore': _multiprocessing.sem_unlink, + }) _CLEANUP_FUNCS.update({ - 'semaphore': _multiprocessing.sem_unlink, 'shared_memory': _posixshmem.shm_unlink, }) diff --git a/contrib/tools/python3/src/Lib/multiprocessing/util.py b/contrib/tools/python3/src/Lib/multiprocessing/util.py index e94466be8ed..230ffd11e64 100644 --- a/contrib/tools/python3/src/Lib/multiprocessing/util.py +++ b/contrib/tools/python3/src/Lib/multiprocessing/util.py @@ -422,7 +422,7 @@ def _close_stdin(): try: fd = os.open(os.devnull, os.O_RDONLY) try: - sys.stdin = open(fd, closefd=False) + sys.stdin = open(fd, encoding="utf-8", closefd=False) except: os.close(fd) raise diff --git a/contrib/tools/python3/src/Lib/netrc.py b/contrib/tools/python3/src/Lib/netrc.py index f0ae48cfed9..734d94c8a62 100644 --- a/contrib/tools/python3/src/Lib/netrc.py +++ b/contrib/tools/python3/src/Lib/netrc.py @@ -26,8 +26,12 @@ class netrc: file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} - with open(file) as fp: - self._parse(file, fp, default_netrc) + try: + with open(file, encoding="utf-8") as fp: + self._parse(file, fp, default_netrc) + except UnicodeDecodeError: + with open(file, encoding="locale") as fp: + self._parse(file, fp, default_netrc) def _parse(self, file, fp, default_netrc): lexer = shlex.shlex(fp) diff --git a/contrib/tools/python3/src/Lib/ntpath.py b/contrib/tools/python3/src/Lib/ntpath.py index 6f771773a7d..527c7ae1938 100644 --- a/contrib/tools/python3/src/Lib/ntpath.py +++ b/contrib/tools/python3/src/Lib/ntpath.py @@ -312,12 +312,25 @@ def expanduser(path): drive = '' userhome = join(drive, os.environ['HOMEPATH']) + if i != 1: #~user + target_user = path[1:i] + if isinstance(target_user, bytes): + target_user = os.fsdecode(target_user) + current_user = os.environ.get('USERNAME') + + if target_user != current_user: + # Try to guess user home directory. By default all user + # profile directories are located in the same place and are + # named by corresponding usernames. If userhome isn't a + # normal profile directory, this guess is likely wrong, + # so we bail out. + if current_user != basename(userhome): + return path + userhome = join(dirname(userhome), target_user) + if isinstance(path, bytes): userhome = os.fsencode(userhome) - if i != 1: #~user - userhome = join(dirname(userhome), path[1:i]) - return userhome + path[i:] @@ -622,7 +635,7 @@ else: tail = join(name, tail) if tail else name return tail - def realpath(path): + def realpath(path, *, strict=False): path = normpath(path) if isinstance(path, bytes): prefix = b'\\\\?\\' @@ -647,6 +660,8 @@ else: path = _getfinalpathname(path) initial_winerror = 0 except OSError as ex: + if strict: + raise initial_winerror = ex.winerror path = _getfinalpathname_nonstrict(path) # The path returned by _getfinalpathname will always start with \\?\ - diff --git a/contrib/tools/python3/src/Lib/opcode.py b/contrib/tools/python3/src/Lib/opcode.py index ac1aa535f66..37e88e92df7 100644 --- a/contrib/tools/python3/src/Lib/opcode.py +++ b/contrib/tools/python3/src/Lib/opcode.py @@ -67,7 +67,6 @@ def_op('UNARY_NEGATIVE', 11) def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) - def_op('BINARY_MATRIX_MULTIPLY', 16) def_op('INPLACE_MATRIX_MULTIPLY', 17) @@ -82,8 +81,12 @@ def_op('BINARY_FLOOR_DIVIDE', 26) def_op('BINARY_TRUE_DIVIDE', 27) def_op('INPLACE_FLOOR_DIVIDE', 28) def_op('INPLACE_TRUE_DIVIDE', 29) +def_op('GET_LEN', 30) +def_op('MATCH_MAPPING', 31) +def_op('MATCH_SEQUENCE', 32) +def_op('MATCH_KEYS', 33) +def_op('COPY_DICT_WITHOUT_KEYS', 34) -def_op('RERAISE', 48) def_op('WITH_EXCEPT_START', 49) def_op('GET_AITER', 50) def_op('GET_ANEXT', 51) @@ -105,7 +108,6 @@ def_op('BINARY_OR', 66) def_op('INPLACE_POWER', 67) def_op('GET_ITER', 68) def_op('GET_YIELD_FROM_ITER', 69) - def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) def_op('YIELD_FROM', 72) @@ -137,6 +139,7 @@ name_op('STORE_ATTR', 95) # Index in name list name_op('DELETE_ATTR', 96) # "" name_op('STORE_GLOBAL', 97) # "" name_op('DELETE_GLOBAL', 98) # "" +def_op('ROT_N', 99) def_op('LOAD_CONST', 100) # Index in const list hasconst.append(100) name_op('LOAD_NAME', 101) # Index in name list @@ -149,18 +152,16 @@ def_op('COMPARE_OP', 107) # Comparison operator hascompare.append(107) name_op('IMPORT_NAME', 108) # Index in name list name_op('IMPORT_FROM', 109) # Index in name list - jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" jabs_op('JUMP_ABSOLUTE', 113) # "" jabs_op('POP_JUMP_IF_FALSE', 114) # "" jabs_op('POP_JUMP_IF_TRUE', 115) # "" - name_op('LOAD_GLOBAL', 116) # Index in name list - def_op('IS_OP', 117) def_op('CONTAINS_OP', 118) +def_op('RERAISE', 119) jabs_op('JUMP_IF_NOT_EXC_MATCH', 121) jrel_op('SETUP_FINALLY', 122) # Distance to target address @@ -172,10 +173,12 @@ haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) +def_op('GEN_START', 129) # Kind of generator/coroutine def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args def_op('MAKE_FUNCTION', 132) # Flags def_op('BUILD_SLICE', 133) # Number of items + def_op('LOAD_CLOSURE', 135) hasfree.append(135) def_op('LOAD_DEREF', 136) @@ -187,28 +190,24 @@ hasfree.append(138) def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs def_op('CALL_FUNCTION_EX', 142) # Flags - jrel_op('SETUP_WITH', 143) - +def_op('EXTENDED_ARG', 144) +EXTENDED_ARG = 144 def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) - def_op('LOAD_CLASSDEREF', 148) hasfree.append(148) -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 +def_op('MATCH_CLASS', 152) jrel_op('SETUP_ASYNC_WITH', 154) - def_op('FORMAT_VALUE', 155) def_op('BUILD_CONST_KEY_MAP', 156) def_op('BUILD_STRING', 157) name_op('LOAD_METHOD', 160) def_op('CALL_METHOD', 161) - def_op('LIST_EXTEND', 162) def_op('SET_UPDATE', 163) def_op('DICT_MERGE', 164) diff --git a/contrib/tools/python3/src/Lib/os.py b/contrib/tools/python3/src/Lib/os.py index b794159f86c..d26cfc99939 100644 --- a/contrib/tools/python3/src/Lib/os.py +++ b/contrib/tools/python3/src/Lib/os.py @@ -36,7 +36,7 @@ _names = sys.builtin_module_names __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep", "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR", "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen", - "popen", "extsep"] + "extsep"] def _exists(name): return name in globals() @@ -969,58 +969,64 @@ otherwise return -SIG, where SIG is the signal that killed it. """ __all__.extend(["spawnlp", "spawnlpe"]) - -# Supply os.popen() -def popen(cmd, mode="r", buffering=-1): - if not isinstance(cmd, str): - raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) - if mode not in ("r", "w"): - raise ValueError("invalid mode %r" % mode) - if buffering == 0 or buffering is None: - raise ValueError("popen() does not support unbuffered streams") - import subprocess, io - if mode == "r": - proc = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - bufsize=buffering) - return _wrap_close(io.TextIOWrapper(proc.stdout), proc) - else: - proc = subprocess.Popen(cmd, - shell=True, - stdin=subprocess.PIPE, - bufsize=buffering) - return _wrap_close(io.TextIOWrapper(proc.stdin), proc) - -# Helper for popen() -- a proxy for a file whose close waits for the process -class _wrap_close: - def __init__(self, stream, proc): - self._stream = stream - self._proc = proc - def close(self): - self._stream.close() - returncode = self._proc.wait() - if returncode == 0: - return None - if name == 'nt': - return returncode +# VxWorks has no user space shell provided. As a result, running +# command in a shell can't be supported. +if sys.platform != 'vxworks': + # Supply os.popen() + def popen(cmd, mode="r", buffering=-1): + if not isinstance(cmd, str): + raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) + if mode not in ("r", "w"): + raise ValueError("invalid mode %r" % mode) + if buffering == 0 or buffering is None: + raise ValueError("popen() does not support unbuffered streams") + import subprocess, io + if mode == "r": + proc = subprocess.Popen(cmd, + shell=True, text=True, + stdout=subprocess.PIPE, + bufsize=buffering) + return _wrap_close(proc.stdout, proc) else: - return returncode << 8 # Shift left to match old behavior - def __enter__(self): - return self - def __exit__(self, *args): - self.close() - def __getattr__(self, name): - return getattr(self._stream, name) - def __iter__(self): - return iter(self._stream) + proc = subprocess.Popen(cmd, + shell=True, text=True, + stdin=subprocess.PIPE, + bufsize=buffering) + return _wrap_close(proc.stdin, proc) + + # Helper for popen() -- a proxy for a file whose close waits for the process + class _wrap_close: + def __init__(self, stream, proc): + self._stream = stream + self._proc = proc + def close(self): + self._stream.close() + returncode = self._proc.wait() + if returncode == 0: + return None + if name == 'nt': + return returncode + else: + return returncode << 8 # Shift left to match old behavior + def __enter__(self): + return self + def __exit__(self, *args): + self.close() + def __getattr__(self, name): + return getattr(self._stream, name) + def __iter__(self): + return iter(self._stream) + + __all__.append("popen") # Supply os.fdopen() -def fdopen(fd, *args, **kwargs): +def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs): if not isinstance(fd, int): raise TypeError("invalid fd type (%s, expected integer)" % type(fd)) import io - return io.open(fd, *args, **kwargs) + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(fd, mode, buffering, encoding, *args, **kwargs) # For testing purposes, make sure the function is available when the C diff --git a/contrib/tools/python3/src/Lib/pathlib.py b/contrib/tools/python3/src/Lib/pathlib.py index 7aeda14a141..621fba0e75c 100644 --- a/contrib/tools/python3/src/Lib/pathlib.py +++ b/contrib/tools/python3/src/Lib/pathlib.py @@ -6,6 +6,7 @@ import os import posixpath import re import sys +import warnings from _collections_abc import Sequence from errno import EINVAL, ENOENT, ENOTDIR, EBADF, ELOOP from operator import attrgetter @@ -13,18 +14,6 @@ from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from urllib.parse import quote_from_bytes as urlquote_from_bytes -supports_symlinks = True -if os.name == 'nt': - import nt - if sys.getwindowsversion()[:2] >= (6, 0): - from nt import _getfinalpathname - else: - supports_symlinks = False - _getfinalpathname = None -else: - nt = None - - __all__ = [ "PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath", @@ -34,14 +23,17 @@ __all__ = [ # Internals # +_WINERROR_NOT_READY = 21 # drive exists but is not accessible +_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 +_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself + # EBADF - guard against macOS `stat` throwing EBADF _IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF, ELOOP) _IGNORED_WINERRORS = ( - 21, # ERROR_NOT_READY - drive exists but is not accessible - 123, # ERROR_INVALID_NAME - fix for bpo-35306 - 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself -) + _WINERROR_NOT_READY, + _WINERROR_INVALID_NAME, + _WINERROR_CANT_RESOLVE_FILENAME) def _ignore_error(exception): return (getattr(exception, 'errno', None) in _IGNORED_ERROS or @@ -200,30 +192,6 @@ class _WindowsFlavour(_Flavour): def compile_pattern(self, pattern): return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch - def resolve(self, path, strict=False): - s = str(path) - if not s: - return os.getcwd() - previous_s = None - if _getfinalpathname is not None: - if strict: - return self._ext_to_normal(_getfinalpathname(s)) - else: - tail_parts = [] # End of the path after the first one not found - while True: - try: - s = self._ext_to_normal(_getfinalpathname(s)) - except FileNotFoundError: - previous_s = s - s, tail = os.path.split(s) - tail_parts.append(tail) - if previous_s == s: - return path - else: - return os.path.join(s, *reversed(tail_parts)) - # Means fallback on absolute - return None - def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix): prefix = '' if s.startswith(ext_prefix): @@ -234,10 +202,6 @@ class _WindowsFlavour(_Flavour): s = '\\' + s[3:] return prefix, s - def _ext_to_normal(self, s): - # Turn back an extended path into a normal DOS-like path - return self._split_extended_path(s)[1] - def is_reserved(self, parts): # NOTE: the rules for reserved names seem somewhat complicated # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not @@ -263,34 +227,6 @@ class _WindowsFlavour(_Flavour): # It's a path on a network drive => 'file://host/share/a/b' return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8')) - def gethomedir(self, username): - if 'USERPROFILE' in os.environ: - userhome = os.environ['USERPROFILE'] - elif 'HOMEPATH' in os.environ: - try: - drv = os.environ['HOMEDRIVE'] - except KeyError: - drv = '' - userhome = drv + os.environ['HOMEPATH'] - else: - raise RuntimeError("Can't determine home directory") - - if username: - # Try to guess user home directory. By default all users - # directories are located in the same place and are named by - # corresponding usernames. If current user home directory points - # to nonstandard place, this guess is likely wrong. - if os.environ['USERNAME'] != username: - drv, root, parts = self.parse_parts((userhome,)) - if parts[-1] != os.environ['USERNAME']: - raise RuntimeError("Can't determine home directory " - "for %r" % username) - parts[-1] = username - if drv or root: - userhome = drv + root + self.join(parts[1:]) - else: - userhome = self.join(parts) - return userhome class _PosixFlavour(_Flavour): sep = '/' @@ -324,54 +260,6 @@ class _PosixFlavour(_Flavour): def compile_pattern(self, pattern): return re.compile(fnmatch.translate(pattern)).fullmatch - def resolve(self, path, strict=False): - sep = self.sep - accessor = path._accessor - seen = {} - def _resolve(path, rest): - if rest.startswith(sep): - path = '' - - for name in rest.split(sep): - if not name or name == '.': - # current dir - continue - if name == '..': - # parent dir - path, _, _ = path.rpartition(sep) - continue - if path.endswith(sep): - newpath = path + name - else: - newpath = path + sep + name - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - raise RuntimeError("Symlink loop from %r" % newpath) - # Resolve the symbolic link - try: - target = accessor.readlink(newpath) - except OSError as e: - if e.errno != EINVAL and strict: - raise - # Not a symlink, or non-strict mode. We just leave the path - # untouched. - path = newpath - else: - seen[newpath] = None # not resolved symlink - path = _resolve(path, target) - seen[newpath] = path # resolved symlink - - return path - # NOTE: according to POSIX, getcwd() cannot contain path components - # which are symlinks. - base = '' if path.is_absolute() else os.getcwd() - return _resolve(base, str(path)) or sep - def is_reserved(self, parts): return False @@ -381,21 +269,6 @@ class _PosixFlavour(_Flavour): bpath = bytes(path) return 'file://' + urlquote_from_bytes(bpath) - def gethomedir(self, username): - if not username: - try: - return os.environ['HOME'] - except KeyError: - import pwd - return pwd.getpwuid(os.getuid()).pw_dir - else: - import pwd - try: - return pwd.getpwnam(username).pw_dir - except KeyError: - raise RuntimeError("Can't determine home directory " - "for %r" % username) - _windows_flavour = _WindowsFlavour() _posix_flavour = _PosixFlavour() @@ -410,9 +283,7 @@ class _NormalAccessor(_Accessor): stat = os.stat - lstat = os.lstat - - open = os.open + open = io.open listdir = os.listdir @@ -420,21 +291,14 @@ class _NormalAccessor(_Accessor): chmod = os.chmod - if hasattr(os, "lchmod"): - lchmod = os.lchmod - else: - def lchmod(self, pathobj, mode): - raise NotImplementedError("lchmod() not available on this system") - mkdir = os.mkdir unlink = os.unlink if hasattr(os, "link"): - link_to = os.link + link = os.link else: - @staticmethod - def link_to(self, target): + def link(self, src, dst): raise NotImplementedError("os.link() not available on this system") rmdir = os.rmdir @@ -443,23 +307,35 @@ class _NormalAccessor(_Accessor): replace = os.replace - if nt: - if supports_symlinks: - symlink = os.symlink - else: - def symlink(a, b, target_is_directory): - raise NotImplementedError("symlink() not available on this system") + if hasattr(os, "symlink"): + symlink = os.symlink else: - # Under POSIX, os.symlink() takes two args - @staticmethod - def symlink(a, b, target_is_directory): - return os.symlink(a, b) + def symlink(self, src, dst, target_is_directory=False): + raise NotImplementedError("os.symlink() not available on this system") - utime = os.utime + def touch(self, path, mode=0o666, exist_ok=True): + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(path, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(path, flags, mode) + os.close(fd) - # Helper for resolve() - def readlink(self, path): - return os.readlink(path) + if hasattr(os, "readlink"): + readlink = os.readlink + else: + def readlink(self, path): + raise NotImplementedError("os.readlink() not available on this system") def owner(self, path): try: @@ -475,6 +351,12 @@ class _NormalAccessor(_Accessor): except ImportError: raise NotImplementedError("Path.group() is unsupported on this system") + getcwd = os.getcwd + + expanduser = staticmethod(os.path.expanduser) + + realpath = staticmethod(os.path.realpath) + _normal_accessor = _NormalAccessor() @@ -641,7 +523,10 @@ class _PathParents(Sequence): return len(self._parts) def __getitem__(self, idx): - if idx < 0 or idx >= len(self): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): raise IndexError(idx) return self._pathcls._from_parsed_parts(self._drv, self._root, self._parts[:-idx - 1]) @@ -700,7 +585,7 @@ class PurePath(object): return cls._flavour.parse_parts(parts) @classmethod - def _from_parts(cls, args, init=True): + def _from_parts(cls, args): # We need to call _parse_args on the instance, so as to get the # right flavour. self = object.__new__(cls) @@ -708,18 +593,14 @@ class PurePath(object): self._drv = drv self._root = root self._parts = parts - if init: - self._init() return self @classmethod - def _from_parsed_parts(cls, drv, root, parts, init=True): + def _from_parsed_parts(cls, drv, root, parts): self = object.__new__(cls) self._drv = drv self._root = root self._parts = parts - if init: - self._init() return self @classmethod @@ -729,10 +610,6 @@ class PurePath(object): else: return cls._flavour.join(parts) - def _init(self): - # Overridden in concrete Path - pass - def _make_child(self, args): drv, root, parts = self._parse_args(args) drv, root, parts = self._flavour.join_parsed_parts( @@ -1072,29 +949,18 @@ class Path(PurePath): object. You can also instantiate a PosixPath or WindowsPath directly, but cannot instantiate a WindowsPath on a POSIX system or vice versa. """ - __slots__ = ( - '_accessor', - ) + _accessor = _normal_accessor + __slots__ = () def __new__(cls, *args, **kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath - self = cls._from_parts(args, init=False) + self = cls._from_parts(args) if not self._flavour.is_supported: raise NotImplementedError("cannot instantiate %r on your system" % (cls.__name__,)) - self._init() return self - def _init(self, - # Private non-constructor arguments - template=None, - ): - if template is not None: - self._accessor = template._accessor - else: - self._accessor = _normal_accessor - def _make_child_relpath(self, part): # This is an optimization used for dir walking. `part` must be # a single part relative to this path. @@ -1115,17 +981,6 @@ class Path(PurePath): # removed in the future. pass - def _opener(self, name, flags, mode=0o666): - # A stub for the opener argument to built-in open() - return self._accessor.open(self, flags, mode) - - def _raw_open(self, flags, mode=0o777): - """ - Open the file pointed by this path and return a file descriptor, - as os.open() does. - """ - return self._accessor.open(self, flags, mode) - # Public API @classmethod @@ -1133,14 +988,14 @@ class Path(PurePath): """Return a new path pointing to the current working directory (as returned by os.getcwd()). """ - return cls(os.getcwd()) + return cls(cls._accessor.getcwd()) @classmethod def home(cls): """Return a new path pointing to the user's home directory (as returned by os.path.expanduser('~')). """ - return cls(cls()._flavour.gethomedir(None)) + return cls("~").expanduser() def samefile(self, other_path): """Return whether other_path is the same or not as this file @@ -1202,9 +1057,7 @@ class Path(PurePath): return self # FIXME this must defer to the specific flavour (and, under Windows, # use nt._getfullpathname()) - obj = self._from_parts([os.getcwd()] + self._parts, init=False) - obj._init(template=self) - return obj + return self._from_parts([self._accessor.getcwd()] + self._parts) def resolve(self, strict=False): """ @@ -1212,24 +1065,34 @@ class Path(PurePath): normalizing it (for example turning slashes into backslashes under Windows). """ - s = self._flavour.resolve(self, strict=strict) - if s is None: - # No symlink resolution => for consistency, raise an error if - # the path doesn't exist or is forbidden - self.stat() - s = str(self.absolute()) - # Now we have no symlinks in the path, it's safe to normalize it. - normed = self._flavour.pathmod.normpath(s) - obj = self._from_parts((normed,), init=False) - obj._init(template=self) - return obj - def stat(self): + def check_eloop(e): + winerror = getattr(e, 'winerror', 0) + if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: + raise RuntimeError("Symlink loop from %r" % e.filename) + + try: + s = self._accessor.realpath(self, strict=strict) + except OSError as e: + check_eloop(e) + raise + p = self._from_parts((s,)) + + # In non-strict mode, realpath() doesn't raise on symlink loops. + # Ensure we get an exception by calling stat() + if not strict: + try: + p.stat() + except OSError as e: + check_eloop(e) + return p + + def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - return self._accessor.stat(self) + return self._accessor.stat(self, follow_symlinks=follow_symlinks) def owner(self): """ @@ -1249,8 +1112,10 @@ class Path(PurePath): Open the file pointed by this path and return a file object, as the built-in open() function does. """ - return io.open(self, mode, buffering, encoding, errors, newline, - opener=self._opener) + if "b" not in mode: + encoding = io.text_encoding(encoding) + return self._accessor.open(self, mode, buffering, encoding, errors, + newline) def read_bytes(self): """ @@ -1263,6 +1128,7 @@ class Path(PurePath): """ Open the file in text mode, read it, and close the file. """ + encoding = io.text_encoding(encoding) with self.open(mode='r', encoding=encoding, errors=errors) as f: return f.read() @@ -1275,14 +1141,15 @@ class Path(PurePath): with self.open(mode='wb') as f: return f.write(view) - def write_text(self, data, encoding=None, errors=None): + def write_text(self, data, encoding=None, errors=None, newline=None): """ Open the file in text mode, write to it, and close the file. """ if not isinstance(data, str): raise TypeError('data must be str, not %s' % data.__class__.__name__) - with self.open(mode='w', encoding=encoding, errors=errors) as f: + encoding = io.text_encoding(encoding) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) def readlink(self): @@ -1290,30 +1157,13 @@ class Path(PurePath): Return the path to which the symbolic link points. """ path = self._accessor.readlink(self) - obj = self._from_parts((path,), init=False) - obj._init(template=self) - return obj + return self._from_parts((path,)) def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. """ - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - self._accessor.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = self._raw_open(flags, mode) - os.close(fd) + self._accessor.touch(self, mode, exist_ok) def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ @@ -1332,18 +1182,18 @@ class Path(PurePath): if not exist_ok or not self.is_dir(): raise - def chmod(self, mode): + def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). """ - self._accessor.chmod(self, mode) + self._accessor.chmod(self, mode, follow_symlinks=follow_symlinks) def lchmod(self, mode): """ Like chmod(), except if the path points to a symlink, the symlink's permissions are changed, rather than its target's. """ - self._accessor.lchmod(self, mode) + self.chmod(mode, follow_symlinks=False) def unlink(self, missing_ok=False): """ @@ -1367,7 +1217,7 @@ class Path(PurePath): Like stat(), except if the path points to a symlink, the symlink's status information is returned, rather than its target's. """ - return self._accessor.lstat(self) + return self.stat(follow_symlinks=False) def rename(self, target): """ @@ -1402,6 +1252,14 @@ class Path(PurePath): """ self._accessor.symlink(target, self, target_is_directory) + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + self._accessor.link(target, self) + def link_to(self, target): """ Make the target path a hard link pointing to this path. @@ -1411,8 +1269,14 @@ class Path(PurePath): of arguments (target, link) is the reverse of Path.symlink_to, but matches that of os.link. + Deprecated since Python 3.10 and scheduled for removal in Python 3.12. + Use `hardlink_to()` instead. """ - self._accessor.link_to(self, target) + warnings.warn("pathlib.Path.link_to() is deprecated and is scheduled " + "for removal in Python 3.12. " + "Use pathlib.Path.hardlink_to() instead.", + DeprecationWarning, stacklevel=2) + self._accessor.link(self, target) # Convenience functions for querying the stat results @@ -1569,7 +1433,9 @@ class Path(PurePath): """ if (not (self._drv or self._root) and self._parts and self._parts[0][:1] == '~'): - homedir = self._flavour.gethomedir(self._parts[0][1:]) + homedir = self._accessor.expanduser(self._parts[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") return self._from_parts([homedir] + self._parts[1:]) return self diff --git a/contrib/tools/python3/src/Lib/pickle.py b/contrib/tools/python3/src/Lib/pickle.py index 3d2c75a8538..e7f30f22610 100644 --- a/contrib/tools/python3/src/Lib/pickle.py +++ b/contrib/tools/python3/src/Lib/pickle.py @@ -818,6 +818,7 @@ class _Pickler: self._write_large_bytes(BYTEARRAY8 + pack("{_DOTTED_WORDS})(?P:(?P{_DOTTED_WORDS})?)?$', re.U) -del _DOTTED_WORDS +_NAME_PATTERN = None def resolve_name(name): """ @@ -675,6 +673,15 @@ def resolve_name(name): AttributeError - if a failure occurred when traversing the object hierarchy within the imported package to get to the desired object. """ + global _NAME_PATTERN + if _NAME_PATTERN is None: + # Lazy import to speedup Python startup time + import re + dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' + _NAME_PATTERN = re.compile(f'^(?P{dotted_words})' + f'(?P:(?P{dotted_words})?)?$', + re.UNICODE) + m = _NAME_PATTERN.match(name) if not m: raise ValueError(f'invalid format: {name!r}') diff --git a/contrib/tools/python3/src/Lib/platform.py b/contrib/tools/python3/src/Lib/platform.py index d6412e169b4..e32f9c11cdb 100755 --- a/contrib/tools/python3/src/Lib/platform.py +++ b/contrib/tools/python3/src/Lib/platform.py @@ -174,7 +174,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): The file is read and scanned in chunks of chunksize bytes. """ - if executable is None: + if not executable: try: ver = os.confstr('CS_GNU_LIBC_VERSION') # parse 'glibc 2.28' as ('glibc', '2.28') @@ -526,16 +526,6 @@ def system_alias(system, release, version): # XXX Whatever the new SunOS marketing name is... system = 'Solaris' - elif system == 'IRIX64': - # IRIX reports IRIX64 on platforms with 64-bit support; yet it - # is really a version and not a different platform, since 32-bit - # apps are also supported.. - system = 'IRIX' - if version: - version = version + ' (64bit)' - else: - version = '64bit' - elif system in ('win32', 'win16'): # In case one of the other tricks system = 'Windows' @@ -700,9 +690,6 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Bits if '32-bit' in fileout: bits = '32bit' - elif 'N32' in fileout: - # On Irix only - bits = 'n32bit' elif '64-bit' in fileout: bits = '64bit' @@ -1258,6 +1245,63 @@ def platform(aliased=0, terse=0): _platform_cache[(aliased, terse)] = platform return platform +### freedesktop.org os-release standard +# https://www.freedesktop.org/software/systemd/man/os-release.html + +# NAME=value with optional quotes (' or "). The regular expression is less +# strict than shell lexer, but that's ok. +_os_release_line = re.compile( + "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" +) +# unescape five special characters mentioned in the standard +_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") +# /etc takes precedence over /usr/lib +_os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") +_os_release_cache = None + + +def _parse_os_release(lines): + # These fields are mandatory fields with well-known defaults + # in practice all Linux distributions override NAME, ID, and PRETTY_NAME. + info = { + "NAME": "Linux", + "ID": "linux", + "PRETTY_NAME": "Linux", + } + + for line in lines: + mo = _os_release_line.match(line) + if mo is not None: + info[mo.group('name')] = _os_release_unescape.sub( + r"\1", mo.group('value') + ) + + return info + + +def freedesktop_os_release(): + """Return operation system identification from freedesktop.org os-release + """ + global _os_release_cache + + if _os_release_cache is None: + errno = None + for candidate in _os_release_candidates: + try: + with open(candidate, encoding="utf-8") as f: + _os_release_cache = _parse_os_release(f) + break + except OSError as e: + errno = e.errno + else: + raise OSError( + errno, + f"Unable to read files {', '.join(_os_release_candidates)}" + ) + + return _os_release_cache.copy() + + ### Command line interface if __name__ == '__main__': diff --git a/contrib/tools/python3/src/Lib/posixpath.py b/contrib/tools/python3/src/Lib/posixpath.py index af2814bdb05..195374613a7 100644 --- a/contrib/tools/python3/src/Lib/posixpath.py +++ b/contrib/tools/python3/src/Lib/posixpath.py @@ -262,6 +262,9 @@ def expanduser(path): # password database, return the path unchanged return path userhome = pwent.pw_dir + # if no user home, return the path unchanged on VxWorks + if userhome is None and sys.platform == "vxworks": + return path if isinstance(path, bytes): userhome = os.fsencode(userhome) root = b'/' @@ -385,16 +388,16 @@ def abspath(path): # Return a canonical path (i.e. the absolute location of a file on the # filesystem). -def realpath(filename): +def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, {}) + path, ok = _joinrealpath(filename[:0], filename, strict, {}) return abspath(path) # Join two paths, normalizing and eliminating any symbolic links # encountered in the second path. -def _joinrealpath(path, rest, seen): +def _joinrealpath(path, rest, strict, seen): if isinstance(path, bytes): sep = b'/' curdir = b'.' @@ -423,7 +426,15 @@ def _joinrealpath(path, rest, seen): path = pardir continue newpath = join(path, name) - if not islink(newpath): + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: path = newpath continue # Resolve the symbolic link @@ -434,10 +445,14 @@ def _joinrealpath(path, rest, seen): # use cached value continue # The symlink is not resolved, so we must have a symlink loop. - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return join(newpath, rest), False seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), seen) + path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) if not ok: return join(path, rest), False seen[newpath] = path # resolved symlink diff --git a/contrib/tools/python3/src/Lib/pprint.py b/contrib/tools/python3/src/Lib/pprint.py index 7c1118a484b..d91421f0a6b 100644 --- a/contrib/tools/python3/src/Lib/pprint.py +++ b/contrib/tools/python3/src/Lib/pprint.py @@ -35,6 +35,7 @@ saferepr() """ import collections as _collections +import dataclasses as _dataclasses import re import sys as _sys import types as _types @@ -45,18 +46,20 @@ __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", def pprint(object, stream=None, indent=1, width=80, depth=None, *, - compact=False, sort_dicts=True): + compact=False, sort_dicts=True, underscore_numbers=False): """Pretty-print a Python object to a stream [default is sys.stdout].""" printer = PrettyPrinter( stream=stream, indent=indent, width=width, depth=depth, - compact=compact, sort_dicts=sort_dicts) + compact=compact, sort_dicts=sort_dicts, + underscore_numbers=underscore_numbers) printer.pprint(object) def pformat(object, indent=1, width=80, depth=None, *, - compact=False, sort_dicts=True): + compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" return PrettyPrinter(indent=indent, width=width, depth=depth, - compact=compact, sort_dicts=sort_dicts).pformat(object) + compact=compact, sort_dicts=sort_dicts, + underscore_numbers=underscore_numbers).pformat(object) def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" @@ -64,15 +67,15 @@ def pp(object, *args, sort_dicts=False, **kwargs): def saferepr(object): """Version of repr() which can handle recursive data structures.""" - return _safe_repr(object, {}, None, 0, True)[0] + return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] def isreadable(object): """Determine if saferepr(object) is readable by eval().""" - return _safe_repr(object, {}, None, 0, True)[1] + return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] def isrecursive(object): """Determine if object requires a recursive representation.""" - return _safe_repr(object, {}, None, 0, True)[2] + return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -102,7 +105,7 @@ def _safe_tuple(t): class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, - compact=False, sort_dicts=True): + compact=False, sort_dicts=True, underscore_numbers=False): """Handle pretty printing operations onto a stream using a set of configured parameters. @@ -143,6 +146,7 @@ class PrettyPrinter: self._stream = _sys.stdout self._compact = bool(compact) self._sort_dicts = sort_dicts + self._underscore_numbers = underscore_numbers def pprint(self, object): self._format(object, self._stream, 0, 0, {}, 0) @@ -176,14 +180,26 @@ class PrettyPrinter: p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif isinstance(object, dict): + elif (_dataclasses.is_dataclass(object) and + not isinstance(object, type) and + object.__dataclass_params__.repr and + # Check dataclass has generated repr method. + hasattr(object.__repr__, "__wrapped__") and + "__create_fn__" in object.__repr__.__wrapped__.__qualname__): context[objid] = 1 - self._pprint_dict(object, stream, indent, allowance, - context, level + 1) + self._pprint_dataclass(object, stream, indent, allowance, context, level + 1) del context[objid] return stream.write(rep) + def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + cls_name = object.__class__.__name__ + indent += len(cls_name) + 1 + items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + stream.write(cls_name + '(') + self._format_namespace_items(items, stream, indent, allowance, context, level) + stream.write(')') + _dispatch = {} def _pprint_dict(self, object, stream, indent, allowance, context, level): @@ -350,21 +366,9 @@ class PrettyPrinter: else: cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - delimnl = ',\n' + ' ' * indent items = object.__dict__.items() - last_index = len(items) - 1 - stream.write(cls_name + '(') - for i, (key, ent) in enumerate(items): - stream.write(key) - stream.write('=') - - last = i == last_index - self._format(ent, stream, indent + len(key) + 1, - allowance if last else 1, - context, level) - if not last: - stream.write(delimnl) + self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') _dispatch[_types.SimpleNamespace.__repr__] = _pprint_simplenamespace @@ -386,6 +390,25 @@ class PrettyPrinter: if not last: write(delimnl) + def _format_namespace_items(self, items, stream, indent, allowance, context, level): + write = stream.write + delimnl = ',\n' + ' ' * indent + last_index = len(items) - 1 + for i, (key, ent) in enumerate(items): + last = i == last_index + write(key) + write('=') + if id(ent) in context: + # Special-case representation of recursion to match standard + # recursive dataclass repr. + write("...") + else: + self._format(ent, stream, indent + len(key) + 1, + allowance if last else 1, + context, level) + if not last: + write(delimnl) + def _format_items(self, items, stream, indent, allowance, context, level): write = stream.write indent += self._indent_per_level @@ -441,7 +464,7 @@ class PrettyPrinter: and flags indicating whether the representation is 'readable' and whether the object represents a recursive construct. """ - return _safe_repr(object, context, maxlevels, level, self._sort_dicts) + return self._safe_repr(object, context, maxlevels, level) def _pprint_default_dict(self, object, stream, indent, allowance, context, level): if not len(object): @@ -524,79 +547,88 @@ class PrettyPrinter: _dispatch[_collections.UserString.__repr__] = _pprint_user_string -# Return triple (repr_string, isreadable, isrecursive). + def _safe_repr(self, object, context, maxlevels, level): + # Return triple (repr_string, isreadable, isrecursive). + typ = type(object) + if typ in _builtin_scalars: + return repr(object), True, False -def _safe_repr(object, context, maxlevels, level, sort_dicts): - typ = type(object) - if typ in _builtin_scalars: - return repr(object), True, False + r = getattr(typ, "__repr__", None) - r = getattr(typ, "__repr__", None) - if issubclass(typ, dict) and r is dict.__repr__: - if not object: - return "{}", True, False - objid = id(object) - if maxlevels and level >= maxlevels: - return "{...}", False, objid in context - if objid in context: - return _recursion(object), False, True - context[objid] = 1 - readable = True - recursive = False - components = [] - append = components.append - level += 1 - if sort_dicts: - items = sorted(object.items(), key=_safe_tuple) - else: - items = object.items() - for k, v in items: - krepr, kreadable, krecur = _safe_repr(k, context, maxlevels, level, sort_dicts) - vrepr, vreadable, vrecur = _safe_repr(v, context, maxlevels, level, sort_dicts) - append("%s: %s" % (krepr, vrepr)) - readable = readable and kreadable and vreadable - if krecur or vrecur: - recursive = True - del context[objid] - return "{%s}" % ", ".join(components), readable, recursive - - if (issubclass(typ, list) and r is list.__repr__) or \ - (issubclass(typ, tuple) and r is tuple.__repr__): - if issubclass(typ, list): - if not object: - return "[]", True, False - format = "[%s]" - elif len(object) == 1: - format = "(%s,)" - else: + if issubclass(typ, int) and r is int.__repr__: + if self._underscore_numbers: + return f"{object:_d}", True, False + else: + return repr(object), True, False + + if issubclass(typ, dict) and r is dict.__repr__: if not object: - return "()", True, False - format = "(%s)" - objid = id(object) - if maxlevels and level >= maxlevels: - return format % "...", False, objid in context - if objid in context: - return _recursion(object), False, True - context[objid] = 1 - readable = True - recursive = False - components = [] - append = components.append - level += 1 - for o in object: - orepr, oreadable, orecur = _safe_repr(o, context, maxlevels, level, sort_dicts) - append(orepr) - if not oreadable: - readable = False - if orecur: - recursive = True - del context[objid] - return format % ", ".join(components), readable, recursive - - rep = repr(object) - return rep, (rep and not rep.startswith('<')), False - -_builtin_scalars = frozenset({str, bytes, bytearray, int, float, complex, + return "{}", True, False + objid = id(object) + if maxlevels and level >= maxlevels: + return "{...}", False, objid in context + if objid in context: + return _recursion(object), False, True + context[objid] = 1 + readable = True + recursive = False + components = [] + append = components.append + level += 1 + if self._sort_dicts: + items = sorted(object.items(), key=_safe_tuple) + else: + items = object.items() + for k, v in items: + krepr, kreadable, krecur = self.format( + k, context, maxlevels, level) + vrepr, vreadable, vrecur = self.format( + v, context, maxlevels, level) + append("%s: %s" % (krepr, vrepr)) + readable = readable and kreadable and vreadable + if krecur or vrecur: + recursive = True + del context[objid] + return "{%s}" % ", ".join(components), readable, recursive + + if (issubclass(typ, list) and r is list.__repr__) or \ + (issubclass(typ, tuple) and r is tuple.__repr__): + if issubclass(typ, list): + if not object: + return "[]", True, False + format = "[%s]" + elif len(object) == 1: + format = "(%s,)" + else: + if not object: + return "()", True, False + format = "(%s)" + objid = id(object) + if maxlevels and level >= maxlevels: + return format % "...", False, objid in context + if objid in context: + return _recursion(object), False, True + context[objid] = 1 + readable = True + recursive = False + components = [] + append = components.append + level += 1 + for o in object: + orepr, oreadable, orecur = self.format( + o, context, maxlevels, level) + append(orepr) + if not oreadable: + readable = False + if orecur: + recursive = True + del context[objid] + return format % ", ".join(components), readable, recursive + + rep = repr(object) + return rep, (rep and not rep.startswith('<')), False + +_builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) def _recursion(object): @@ -610,7 +642,7 @@ def _perfcheck(object=None): object = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000 p = PrettyPrinter() t1 = time.perf_counter() - _safe_repr(object, {}, None, 0, True) + p._safe_repr(object, {}, None, 0, True) t2 = time.perf_counter() p.pformat(object) t3 = time.perf_counter() diff --git a/contrib/tools/python3/src/Lib/pty.py b/contrib/tools/python3/src/Lib/pty.py index a32432041fa..8d8ce40df54 100644 --- a/contrib/tools/python3/src/Lib/pty.py +++ b/contrib/tools/python3/src/Lib/pty.py @@ -1,7 +1,7 @@ """Pseudo terminal utilities.""" # Bugs: No signal handling. Doesn't set slave termios and window size. -# Only tested on Linux. +# Only tested on Linux, FreeBSD, and macOS. # See: W. Richard Stevens. 1992. Advanced Programming in the # UNIX Environment. Chapter 19. # Author: Steen Lumholt -- with additions by Guido. @@ -11,7 +11,11 @@ import os import sys import tty -__all__ = ["openpty","fork","spawn"] +# names imported directly for test mocking purposes +from os import close, waitpid +from tty import setraw, tcgetattr, tcsetattr + +__all__ = ["openpty", "fork", "spawn"] STDIN_FILENO = 0 STDOUT_FILENO = 1 @@ -105,8 +109,8 @@ def fork(): os.dup2(slave_fd, STDIN_FILENO) os.dup2(slave_fd, STDOUT_FILENO) os.dup2(slave_fd, STDERR_FILENO) - if (slave_fd > STDERR_FILENO): - os.close (slave_fd) + if slave_fd > STDERR_FILENO: + os.close(slave_fd) # Explicitly open the tty to make it become a controlling tty. tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) @@ -133,14 +137,22 @@ def _copy(master_fd, master_read=_read, stdin_read=_read): pty master -> standard output (master_read) standard input -> pty master (stdin_read)""" fds = [master_fd, STDIN_FILENO] - while True: - rfds, wfds, xfds = select(fds, [], []) + while fds: + rfds, _wfds, _xfds = select(fds, [], []) + if master_fd in rfds: - data = master_read(master_fd) + # Some OSes signal EOF by returning an empty byte string, + # some throw OSErrors. + try: + data = master_read(master_fd) + except OSError: + data = b"" if not data: # Reached EOF. - fds.remove(master_fd) + return # Assume the child process has exited and is + # unreachable, so we clean up. else: os.write(STDOUT_FILENO, data) + if STDIN_FILENO in rfds: data = stdin_read(STDIN_FILENO) if not data: @@ -153,20 +165,23 @@ def spawn(argv, master_read=_read, stdin_read=_read): if type(argv) == type(''): argv = (argv,) sys.audit('pty.spawn', argv) + pid, master_fd = fork() if pid == CHILD: os.execlp(argv[0], *argv) + try: - mode = tty.tcgetattr(STDIN_FILENO) - tty.setraw(STDIN_FILENO) - restore = 1 + mode = tcgetattr(STDIN_FILENO) + setraw(STDIN_FILENO) + restore = True except tty.error: # This is the same as termios.error - restore = 0 + restore = False + try: _copy(master_fd, master_read, stdin_read) - except OSError: + finally: if restore: - tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode) + tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode) - os.close(master_fd) - return os.waitpid(pid, 0)[1] + close(master_fd) + return waitpid(pid, 0)[1] diff --git a/contrib/tools/python3/src/Lib/py_compile.py b/contrib/tools/python3/src/Lib/py_compile.py index a81f4937310..388614e51b1 100644 --- a/contrib/tools/python3/src/Lib/py_compile.py +++ b/contrib/tools/python3/src/Lib/py_compile.py @@ -173,43 +173,40 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, return cfile -def main(args=None): - """Compile several source files. - - The files named in 'args' (or on the command line, if 'args' is - not specified) are compiled and the resulting bytecode is cached - in the normal manner. This function does not search a directory - structure to locate source files; it only compiles files named - explicitly. If '-' is the only parameter in args, the list of - files is taken from standard input. - - """ - if args is None: - args = sys.argv[1:] - rv = 0 - if args == ['-']: - while True: - filename = sys.stdin.readline() - if not filename: - break - filename = filename.rstrip('\n') - try: - compile(filename, doraise=True) - except PyCompileError as error: - rv = 1 - sys.stderr.write("%s\n" % error.msg) - except OSError as error: - rv = 1 - sys.stderr.write("%s\n" % error) +def main(): + import argparse + + description = 'A simple command-line interface for py_compile module.' + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + '-q', '--quiet', + action='store_true', + help='Suppress error output', + ) + parser.add_argument( + 'filenames', + nargs='+', + help='Files to compile', + ) + args = parser.parse_args() + if args.filenames == ['-']: + filenames = [filename.rstrip('\n') for filename in sys.stdin.readlines()] else: - for filename in args: - try: - compile(filename, doraise=True) - except PyCompileError as error: - # return value to indicate at least one failure - rv = 1 - sys.stderr.write("%s\n" % error.msg) - return rv + filenames = args.filenames + for filename in filenames: + try: + compile(filename, doraise=True) + except PyCompileError as error: + if args.quiet: + parser.exit(1) + else: + parser.exit(1, error.msg) + except OSError as error: + if args.quiet: + parser.exit(1) + else: + parser.exit(1, str(error)) + if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/contrib/tools/python3/src/Lib/pyclbr.py b/contrib/tools/python3/src/Lib/pyclbr.py index 99a17343fb6..37f86995d6c 100644 --- a/contrib/tools/python3/src/Lib/pyclbr.py +++ b/contrib/tools/python3/src/Lib/pyclbr.py @@ -21,11 +21,14 @@ has the following attributes: name -- name of the object; file -- file in which the object is defined; lineno -- line in the file where the object's definition starts; + end_lineno -- line in the file where the object's definition ends; parent -- parent of this object, if any; children -- nested objects contained in this object. The 'children' attribute is a dictionary mapping names to objects. -Instances of Function describe functions with the attributes from _Object. +Instances of Function describe functions with the attributes from _Object, +plus the following: + is_async -- if a function is defined with an 'async' prefix Instances of Class describe classes with the attributes from _Object, plus the following: @@ -38,11 +41,9 @@ are recognized and imported modules are scanned as well, this shouldn't happen often. """ -import io +import ast import sys import importlib.util -import tokenize -from token import NAME, DEDENT, OP __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] @@ -51,48 +52,50 @@ _modules = {} # Initialize cache of modules we've seen. class _Object: "Information about Python class or function." - def __init__(self, module, name, file, lineno, parent): + def __init__(self, module, name, file, lineno, end_lineno, parent): self.module = module self.name = name self.file = file self.lineno = lineno + self.end_lineno = end_lineno self.parent = parent self.children = {} - - def _addchild(self, name, obj): - self.children[name] = obj + if parent is not None: + parent.children[name] = self +# Odd Function and Class signatures are for back-compatibility. class Function(_Object): "Information about a Python function, including methods." - def __init__(self, module, name, file, lineno, parent=None): - _Object.__init__(self, module, name, file, lineno, parent) + def __init__(self, module, name, file, lineno, + parent=None, is_async=False, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.is_async = is_async + if isinstance(parent, Class): + parent.methods[name] = lineno class Class(_Object): "Information about a Python class." - def __init__(self, module, name, super, file, lineno, parent=None): - _Object.__init__(self, module, name, file, lineno, parent) - self.super = [] if super is None else super + def __init__(self, module, name, super_, file, lineno, + parent=None, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.super = super_ or [] self.methods = {} - def _addmethod(self, name, lineno): - self.methods[name] = lineno - -def _nest_function(ob, func_name, lineno): +# These 2 functions are used in these tests +# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py +def _nest_function(ob, func_name, lineno, end_lineno, is_async=False): "Return a Function after nesting within ob." - newfunc = Function(ob.module, func_name, ob.file, lineno, ob) - ob._addchild(func_name, newfunc) - if isinstance(ob, Class): - ob._addmethod(func_name, lineno) - return newfunc + return Function(ob.module, func_name, ob.file, lineno, + parent=ob, is_async=is_async, end_lineno=end_lineno) -def _nest_class(ob, class_name, lineno, super=None): +def _nest_class(ob, class_name, lineno, end_lineno, super=None): "Return a Class after nesting within ob." - newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) - ob._addchild(class_name, newclass) - return newclass + return Class(ob.module, class_name, super, ob.file, lineno, + parent=ob, end_lineno=end_lineno) + def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -115,6 +118,7 @@ def readmodule_ex(module, path=None): """ return _readmodule(module, path or []) + def _readmodule(module, path, inpackage=None): """Do the hard work for readmodule[_ex]. @@ -179,187 +183,93 @@ def _readmodule(module, path, inpackage=None): return _create_tree(fullmodule, path, fname, source, tree, inpackage) -def _create_tree(fullmodule, path, fname, source, tree, inpackage): - """Return the tree for a particular module. - - fullmodule (full module name), inpackage+module, becomes o.module. - path is passed to recursive calls of _readmodule. - fname becomes o.file. - source is tokenized. Imports cause recursive calls to _readmodule. - tree is {} or {'__path__': }. - inpackage, None or string, is passed to recursive calls of _readmodule. - - The effect of recursive calls is mutation of global _modules. - """ - f = io.StringIO(source) +class _ModuleBrowser(ast.NodeVisitor): + def __init__(self, module, path, file, tree, inpackage): + self.path = path + self.tree = tree + self.file = file + self.module = module + self.inpackage = inpackage + self.stack = [] + + def visit_ClassDef(self, node): + bases = [] + for base in node.bases: + name = ast.unparse(base) + if name in self.tree: + # We know this super class. + bases.append(self.tree[name]) + elif len(names := name.split(".")) > 1: + # Super class form is module.class: + # look in module for class. + *_, module, class_ = names + if module in _modules: + bases.append(_modules[module].get(class_, name)) + else: + bases.append(name) + + parent = self.stack[-1] if self.stack else None + class_ = Class(self.module, node.name, bases, self.file, node.lineno, + parent=parent, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = class_ + self.stack.append(class_) + self.generic_visit(node) + self.stack.pop() + + def visit_FunctionDef(self, node, *, is_async=False): + parent = self.stack[-1] if self.stack else None + function = Function(self.module, node.name, self.file, node.lineno, + parent, is_async, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = function + self.stack.append(function) + self.generic_visit(node) + self.stack.pop() + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node, is_async=True) + + def visit_Import(self, node): + if node.col_offset != 0: + return + + for module in node.names: + try: + try: + _readmodule(module.name, self.path, self.inpackage) + except ImportError: + _readmodule(module.name, []) + except (ImportError, SyntaxError): + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + + def visit_ImportFrom(self, node): + if node.col_offset != 0: + return + try: + module = "." * node.level + if node.module: + module += node.module + module = _readmodule(module, self.path, self.inpackage) + except (ImportError, SyntaxError): + return + + for name in node.names: + if name.name in module: + self.tree[name.asname or name.name] = module[name.name] + elif name.name == "*": + for import_name, import_value in module.items(): + if import_name.startswith("_"): + continue + self.tree[import_name] = import_value - stack = [] # Initialize stack of (class, indent) pairs. - g = tokenize.generate_tokens(f.readline) - try: - for tokentype, token, start, _end, _line in g: - if tokentype == DEDENT: - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - elif token == 'def': - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - tokentype, func_name, start = next(g)[0:3] - if tokentype != NAME: - continue # Skip def with syntax error. - cur_func = None - if stack: - cur_obj = stack[-1][0] - cur_func = _nest_function(cur_obj, func_name, lineno) - else: - # It is just a function. - cur_func = Function(fullmodule, func_name, fname, lineno) - tree[func_name] = cur_func - stack.append((cur_func, thisindent)) - elif token == 'class': - lineno, thisindent = start - # Close previous nested classes and defs. - while stack and stack[-1][1] >= thisindent: - del stack[-1] - tokentype, class_name, start = next(g)[0:3] - if tokentype != NAME: - continue # Skip class with syntax error. - # Parse what follows the class name. - tokentype, token, start = next(g)[0:3] - inherit = None - if token == '(': - names = [] # Initialize list of superclasses. - level = 1 - super = [] # Tokens making up current superclass. - while True: - tokentype, token, start = next(g)[0:3] - if token in (')', ',') and level == 1: - n = "".join(super) - if n in tree: - # We know this super class. - n = tree[n] - else: - c = n.split('.') - if len(c) > 1: - # Super class form is module.class: - # look in module for class. - m = c[-2] - c = c[-1] - if m in _modules: - d = _modules[m] - if c in d: - n = d[c] - names.append(n) - super = [] - if token == '(': - level += 1 - elif token == ')': - level -= 1 - if level == 0: - break - elif token == ',' and level == 1: - pass - # Only use NAME and OP (== dot) tokens for type name. - elif tokentype in (NAME, OP) and level == 1: - super.append(token) - # Expressions in the base list are not supported. - inherit = names - if stack: - cur_obj = stack[-1][0] - cur_class = _nest_class( - cur_obj, class_name, lineno, inherit) - else: - cur_class = Class(fullmodule, class_name, inherit, - fname, lineno) - tree[class_name] = cur_class - stack.append((cur_class, thisindent)) - elif token == 'import' and start[1] == 0: - modules = _getnamelist(g) - for mod, _mod2 in modules: - try: - # Recursively read the imported module. - if inpackage is None: - _readmodule(mod, path) - else: - try: - _readmodule(mod, path, inpackage) - except ImportError: - _readmodule(mod, []) - except: - # If we can't find or parse the imported module, - # too bad -- don't die here. - pass - elif token == 'from' and start[1] == 0: - mod, token = _getname(g) - if not mod or token != "import": - continue - names = _getnamelist(g) - try: - # Recursively read the imported module. - d = _readmodule(mod, path, inpackage) - except: - # If we can't find or parse the imported module, - # too bad -- don't die here. - continue - # Add any classes that were defined in the imported module - # to our name space if they were mentioned in the list. - for n, n2 in names: - if n in d: - tree[n2 or n] = d[n] - elif n == '*': - # Don't add names that start with _. - for n in d: - if n[0] != '_': - tree[n] = d[n] - except StopIteration: - pass - - f.close() - return tree - - -def _getnamelist(g): - """Return list of (dotted-name, as-name or None) tuples for token source g. - - An as-name is the name that follows 'as' in an as clause. - """ - names = [] - while True: - name, token = _getname(g) - if not name: - break - if token == 'as': - name2, token = _getname(g) - else: - name2 = None - names.append((name, name2)) - while token != "," and "\n" not in token: - token = next(g)[1] - if token != ",": - break - return names - - -def _getname(g): - "Return (dotted-name or None, next-token) tuple for token source g." - parts = [] - tokentype, token = next(g)[0:2] - if tokentype != NAME and token != '*': - return (None, token) - parts.append(token) - while True: - tokentype, token = next(g)[0:2] - if token != '.': - break - tokentype, token = next(g)[0:2] - if tokentype != NAME: - break - parts.append(token) - return (".".join(parts), token) +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage) + mbrowser.visit(ast.parse(source)) + return mbrowser.tree def _main(): diff --git a/contrib/tools/python3/src/Lib/pydoc.py b/contrib/tools/python3/src/Lib/pydoc.py index ead67853076..e00ba4191c4 100755 --- a/contrib/tools/python3/src/Lib/pydoc.py +++ b/contrib/tools/python3/src/Lib/pydoc.py @@ -23,7 +23,7 @@ Run "pydoc -p " to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and -open a Web browser to interactively browse documentation. Combine with +open a web browser to interactively browse documentation. Combine with the -n and -p options to control the hostname and port used. Run "pydoc -w " to write out the HTML documentation for a module @@ -508,7 +508,7 @@ class Doc: not file.startswith(os.path.join(basedir, 'site-packages')))) and object.__name__ not in ('xml.etree', 'test.pydoc_mod')): if docloc.startswith(("http://", "https://")): - docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) + docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) else: docloc = os.path.join(docloc, object.__name__.lower() + ".html") else: @@ -1598,9 +1598,10 @@ def plain(text): def pipepager(text, cmd): """Page through text by feeding it to another program.""" import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace') try: - with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: + with proc.stdin as pipe: try: pipe.write(text) except KeyboardInterrupt: @@ -1822,7 +1823,6 @@ class Helper: 'False': '', 'None': '', 'True': '', - '__peg_parser__': '', 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), @@ -2070,7 +2070,7 @@ has the same effect as typing a particular string at the help> prompt. Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out -the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. +the tutorial on the internet at https://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and @@ -2284,13 +2284,13 @@ def apropos(key): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------- enhanced Web browser interface +# --------------------------------------- enhanced web browser interface def _start_server(urlhandler, hostname, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be - browsed dynamically and interactively with a Web browser. Example use: + browsed dynamically and interactively with a web browser. Example use: >>> import time >>> import pydoc @@ -2676,7 +2676,7 @@ def _url_handler(url, content_type="text/html"): def browse(port=0, *, open_browser=True, hostname='localhost'): - """Start the enhanced pydoc Web server and open a Web browser. + """Start the enhanced pydoc web server and open a web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. @@ -2828,7 +2828,7 @@ def cli(): number 0 can be used to get an arbitrary unused port. {cmd} -b - Start an HTTP server on an arbitrary unused port and open a Web browser + Start an HTTP server on an arbitrary unused port and open a web browser to interactively browse documentation. This option can be used in combination with -n and/or -p. diff --git a/contrib/tools/python3/src/Lib/pydoc_data/topics.py b/contrib/tools/python3/src/Lib/pydoc_data/topics.py index 2546eb933bc..76db0426df3 100644 --- a/contrib/tools/python3/src/Lib/pydoc_data/topics.py +++ b/contrib/tools/python3/src/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Wed Mar 23 22:08:02 2022 +# Autogenerated by Sphinx on Wed Mar 23 20:11:40 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -433,11 +433,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'Execution of Python coroutines can be suspended and resumed at ' 'many\n' - 'points (see *coroutine*). Inside the body of a coroutine ' - 'function,\n' - '"await" and "async" identifiers become reserved keywords; "await"\n' - 'expressions, "async for" and "async with" can only be used in\n' - 'coroutine function bodies.\n' + 'points (see *coroutine*). "await" expressions, "async for" and ' + '"async\n' + 'with" can only be used in the body of a coroutine function.\n' '\n' 'Functions defined with "async def" syntax are always coroutine\n' 'functions, even if they do not contain "await" or "async" ' @@ -453,6 +451,10 @@ topics = {'assert': 'The "assert" statement\n' ' do_stuff()\n' ' await some_coroutine()\n' '\n' + 'Changed in version 3.7: "await" and "async" are now keywords;\n' + 'previously they were only treated as such inside the body of a\n' + 'coroutine function.\n' + '\n' '\n' 'The "async for" statement\n' '=========================\n' @@ -549,13 +551,65 @@ topics = {'assert': 'The "assert" statement\n' 'exception.\n' ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the ' + '[2] In pattern matching, a sequence is defined as one of the\n' + ' following:\n' + '\n' + ' * a class that inherits from "collections.abc.Sequence"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Sequence"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_SEQUENCE"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The following standard library classes are sequences:\n' + '\n' + ' * "array.array"\n' + '\n' + ' * "collections.deque"\n' + '\n' + ' * "list"\n' + '\n' + ' * "memoryview"\n' + '\n' + ' * "range"\n' + '\n' + ' * "tuple"\n' + '\n' + ' Note:\n' + '\n' + ' Subject values of type "str", "bytes", and "bytearray" do ' + 'not\n' + ' match sequence patterns.\n' + '\n' + '[3] In pattern matching, a mapping is defined as one of the ' + 'following:\n' + '\n' + ' * a class that inherits from "collections.abc.Mapping"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Mapping"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_MAPPING"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The standard library classes "dict" and ' + '"types.MappingProxyType"\n' + ' are mappings.\n' + '\n' + '[4] A string literal appearing as the first statement in the ' 'function\n' ' body is transformed into the function’s "__doc__" attribute ' 'and\n' ' therefore the function’s *docstring*.\n' '\n' - '[3] A string literal appearing as the first statement in the class\n' + '[5] A string literal appearing as the first statement in the class\n' ' body is transformed into the namespace’s "__doc__" item and\n' ' therefore the class’s *docstring*.\n', 'atom-identifiers': 'Identifiers (Names)\n' @@ -882,32 +936,6 @@ topics = {'assert': 'The "assert" statement\n' '*instance* of the\n' ' owner class.\n' '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is ' - 'created. The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' Note:\n' - '\n' - ' "__set_name__()" is only called implicitly as part ' - 'of the "type"\n' - ' constructor, so it will need to be called ' - 'explicitly with the\n' - ' appropriate parameters when a descriptor is added ' - 'to a class\n' - ' after initial creation:\n' - '\n' - ' class A:\n' - ' pass\n' - ' descr = custom_descriptor()\n' - ' A.attr = descr\n' - " descr.__set_name__(A, 'attr')\n" - '\n' - ' See Creating the class object for more details.\n' - '\n' - ' New in version 3.6.\n' - '\n' 'The attribute "__objclass__" is interpreted by the ' '"inspect" module as\n' 'specifying the class where this object was defined ' @@ -1736,7 +1764,7 @@ topics = {'assert': 'The "assert" statement\n' 'original global namespace. (Usually, the suite contains mostly\n' 'function definitions.) When the class’s suite finishes execution, ' 'its\n' - 'execution frame is discarded but its local namespace is saved. [3] ' + 'execution frame is discarded but its local namespace is saved. [5] ' 'A\n' 'class object is then created using the inheritance list for the ' 'base\n' @@ -2219,6 +2247,7 @@ topics = {'assert': 'The "assert" statement\n' ' | for_stmt\n' ' | try_stmt\n' ' | with_stmt\n' + ' | match_stmt\n' ' | funcdef\n' ' | classdef\n' ' | async_with_stmt\n' @@ -2356,33 +2385,6 @@ topics = {'assert': 'The "assert" statement\n' ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, ' '2]".\n' '\n' - 'Note:\n' - '\n' - ' There is a subtlety when the sequence is being modified by the ' - 'loop\n' - ' (this can only occur for mutable sequences, e.g. lists). An\n' - ' internal counter is used to keep track of which item is used ' - 'next,\n' - ' and this is incremented on each iteration. When this counter ' - 'has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item ' - 'from the\n' - ' sequence, the next item will be skipped (since it gets the ' - 'index of\n' - ' the current item which has already been treated). Likewise, ' - 'if the\n' - ' suite inserts an item in the sequence before the current item, ' - 'the\n' - ' current item will be treated again the next time through the ' - 'loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n' - '\n' '\n' 'The "try" statement\n' '===================\n' @@ -2487,11 +2489,32 @@ topics = {'assert': 'The "assert" statement\n' '(see\n' 'section The standard type hierarchy) identifying the point in ' 'the\n' - 'program where the exception occurred. "sys.exc_info()" values ' - 'are\n' - 'restored to their previous values (before the call) when ' - 'returning\n' - 'from a function that handled an exception.\n' + 'program where the exception occurred. The details about the ' + 'exception\n' + 'accessed via "sys.exc_info()" are restored to their previous ' + 'values\n' + 'when leaving an exception handler:\n' + '\n' + ' >>> print(sys.exc_info())\n' + ' (None, None, None)\n' + ' >>> try:\n' + ' ... raise TypeError\n' + ' ... except:\n' + ' ... print(sys.exc_info())\n' + ' ... try:\n' + ' ... raise ValueError\n' + ' ... except:\n' + ' ... print(sys.exc_info())\n' + ' ... print(sys.exc_info())\n' + ' ...\n' + " (, TypeError(), )\n' + " (, ValueError(), )\n' + " (, TypeError(), )\n' + ' >>> print(sys.exc_info())\n' + ' (None, None, None)\n' '\n' 'The optional "else" clause is executed if the control flow ' 'leaves the\n' @@ -2576,8 +2599,10 @@ topics = {'assert': 'The "assert" statement\n' 'usage\n' 'patterns to be encapsulated for convenient reuse.\n' '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' + ' with_stmt ::= "with" ( "(" with_stmt_contents ","? ' + '")" | with_stmt_contents ) ":" suite\n' + ' with_stmt_contents ::= with_item ("," with_item)*\n' + ' with_item ::= expression ["as" target]\n' '\n' 'The execution of the "with" statement with one “item” proceeds ' 'as\n' @@ -2669,9 +2694,23 @@ topics = {'assert': 'The "assert" statement\n' ' with B() as b:\n' ' SUITE\n' '\n' + 'You can also write multi-item context managers in multiple lines ' + 'if\n' + 'the items are surrounded by parentheses. For example:\n' + '\n' + ' with (\n' + ' A() as a,\n' + ' B() as b,\n' + ' ):\n' + ' SUITE\n' + '\n' 'Changed in version 3.1: Support for multiple context ' 'expressions.\n' '\n' + 'Changed in version 3.10: Support for using grouping parentheses ' + 'to\n' + 'break the statement in multiple lines.\n' + '\n' 'See also:\n' '\n' ' **PEP 343** - The “with” statement\n' @@ -2680,6 +2719,746 @@ topics = {'assert': 'The "assert" statement\n' ' statement.\n' '\n' '\n' + 'The "match" statement\n' + '=====================\n' + '\n' + 'New in version 3.10.\n' + '\n' + 'The match statement is used for pattern matching. Syntax:\n' + '\n' + ' match_stmt ::= \'match\' subject_expr ":" NEWLINE INDENT ' + 'case_block+ DEDENT\n' + ' subject_expr ::= star_named_expression "," ' + 'star_named_expressions?\n' + ' | named_expression\n' + ' case_block ::= \'case\' patterns [guard] ":" block\n' + '\n' + 'Note:\n' + '\n' + ' This section uses single quotes to denote soft keywords.\n' + '\n' + 'Pattern matching takes a pattern as input (following "case") and ' + 'a\n' + 'subject value (following "match"). The pattern (which may ' + 'contain\n' + 'subpatterns) is matched against the subject value. The outcomes ' + 'are:\n' + '\n' + '* A match success or failure (also termed a pattern success or\n' + ' failure).\n' + '\n' + '* Possible binding of matched values to a name. The ' + 'prerequisites for\n' + ' this are further discussed below.\n' + '\n' + 'The "match" and "case" keywords are soft keywords.\n' + '\n' + 'See also:\n' + '\n' + ' * **PEP 634** – Structural Pattern Matching: Specification\n' + '\n' + ' * **PEP 636** – Structural Pattern Matching: Tutorial\n' + '\n' + '\n' + 'Overview\n' + '--------\n' + '\n' + 'Here’s an overview of the logical flow of a match statement:\n' + '\n' + '1. The subject expression "subject_expr" is evaluated and a ' + 'resulting\n' + ' subject value obtained. If the subject expression contains a ' + 'comma,\n' + ' a tuple is constructed using the standard rules.\n' + '\n' + '2. Each pattern in a "case_block" is attempted to match with ' + 'the\n' + ' subject value. The specific rules for success or failure are\n' + ' described below. The match attempt can also bind some or all ' + 'of the\n' + ' standalone names within the pattern. The precise pattern ' + 'binding\n' + ' rules vary per pattern type and are specified below. **Name\n' + ' bindings made during a successful pattern match outlive the\n' + ' executed block and can be used after the match statement**.\n' + '\n' + ' Note:\n' + '\n' + ' During failed pattern matches, some subpatterns may ' + 'succeed.\n' + ' Do not rely on bindings being made for a failed match.\n' + ' Conversely, do not rely on variables remaining unchanged ' + 'after\n' + ' a failed match. The exact behavior is dependent on\n' + ' implementation and may vary. This is an intentional ' + 'decision\n' + ' made to allow different implementations to add ' + 'optimizations.\n' + '\n' + '3. If the pattern succeeds, the corresponding guard (if present) ' + 'is\n' + ' evaluated. In this case all name bindings are guaranteed to ' + 'have\n' + ' happened.\n' + '\n' + ' * If the guard evaluates as true or is missing, the "block" ' + 'inside\n' + ' "case_block" is executed.\n' + '\n' + ' * Otherwise, the next "case_block" is attempted as described ' + 'above.\n' + '\n' + ' * If there are no further case blocks, the match statement ' + 'is\n' + ' completed.\n' + '\n' + 'Note:\n' + '\n' + ' Users should generally never rely on a pattern being ' + 'evaluated.\n' + ' Depending on implementation, the interpreter may cache values ' + 'or use\n' + ' other optimizations which skip repeated evaluations.\n' + '\n' + 'A sample match statement:\n' + '\n' + ' >>> flag = False\n' + ' >>> match (100, 200):\n' + ' ... case (100, 300): # Mismatch: 200 != 300\n' + " ... print('Case 1')\n" + ' ... case (100, 200) if flag: # Successful match, but ' + 'guard fails\n' + " ... print('Case 2')\n" + ' ... case (100, y): # Matches and binds y to 200\n' + " ... print(f'Case 3, y: {y}')\n" + ' ... case _: # Pattern not attempted\n' + " ... print('Case 4, I match anything!')\n" + ' ...\n' + ' Case 3, y: 200\n' + '\n' + 'In this case, "if flag" is a guard. Read more about that in the ' + 'next\n' + 'section.\n' + '\n' + '\n' + 'Guards\n' + '------\n' + '\n' + ' guard ::= "if" named_expression\n' + '\n' + 'A "guard" (which is part of the "case") must succeed for code ' + 'inside\n' + 'the "case" block to execute. It takes the form: "if" followed ' + 'by an\n' + 'expression.\n' + '\n' + 'The logical flow of a "case" block with a "guard" follows:\n' + '\n' + '1. Check that the pattern in the "case" block succeeded. If ' + 'the\n' + ' pattern failed, the "guard" is not evaluated and the next ' + '"case"\n' + ' block is checked.\n' + '\n' + '2. If the pattern succeeded, evaluate the "guard".\n' + '\n' + ' * If the "guard" condition evaluates as true, the case block ' + 'is\n' + ' selected.\n' + '\n' + ' * If the "guard" condition evaluates as false, the case block ' + 'is\n' + ' not selected.\n' + '\n' + ' * If the "guard" raises an exception during evaluation, the\n' + ' exception bubbles up.\n' + '\n' + 'Guards are allowed to have side effects as they are ' + 'expressions.\n' + 'Guard evaluation must proceed from the first to the last case ' + 'block,\n' + 'one at a time, skipping case blocks whose pattern(s) don’t all\n' + 'succeed. (I.e., guard evaluation must happen in order.) Guard\n' + 'evaluation must stop once a case block is selected.\n' + '\n' + '\n' + 'Irrefutable Case Blocks\n' + '-----------------------\n' + '\n' + 'An irrefutable case block is a match-all case block. A match\n' + 'statement may have at most one irrefutable case block, and it ' + 'must be\n' + 'last.\n' + '\n' + 'A case block is considered irrefutable if it has no guard and ' + 'its\n' + 'pattern is irrefutable. A pattern is considered irrefutable if ' + 'we can\n' + 'prove from its syntax alone that it will always succeed. Only ' + 'the\n' + 'following patterns are irrefutable:\n' + '\n' + '* AS Patterns whose left-hand side is irrefutable\n' + '\n' + '* OR Patterns containing at least one irrefutable pattern\n' + '\n' + '* Capture Patterns\n' + '\n' + '* Wildcard Patterns\n' + '\n' + '* parenthesized irrefutable patterns\n' + '\n' + '\n' + 'Patterns\n' + '--------\n' + '\n' + 'Note:\n' + '\n' + ' This section uses grammar notations beyond standard EBNF:\n' + '\n' + ' * the notation "SEP.RULE+" is shorthand for "RULE (SEP ' + 'RULE)*"\n' + '\n' + ' * the notation "!RULE" is shorthand for a negative lookahead\n' + ' assertion\n' + '\n' + 'The top-level syntax for "patterns" is:\n' + '\n' + ' patterns ::= open_sequence_pattern | pattern\n' + ' pattern ::= as_pattern | or_pattern\n' + ' closed_pattern ::= | literal_pattern\n' + ' | capture_pattern\n' + ' | wildcard_pattern\n' + ' | value_pattern\n' + ' | group_pattern\n' + ' | sequence_pattern\n' + ' | mapping_pattern\n' + ' | class_pattern\n' + '\n' + 'The descriptions below will include a description “in simple ' + 'terms” of\n' + 'what a pattern does for illustration purposes (credits to ' + 'Raymond\n' + 'Hettinger for a document that inspired most of the ' + 'descriptions). Note\n' + 'that these descriptions are purely for illustration purposes and ' + '**may\n' + 'not** reflect the underlying implementation. Furthermore, they ' + 'do not\n' + 'cover all valid forms.\n' + '\n' + '\n' + 'OR Patterns\n' + '~~~~~~~~~~~\n' + '\n' + 'An OR pattern is two or more patterns separated by vertical bars ' + '"|".\n' + 'Syntax:\n' + '\n' + ' or_pattern ::= "|".closed_pattern+\n' + '\n' + 'Only the final subpattern may be irrefutable, and each ' + 'subpattern must\n' + 'bind the same set of names to avoid ambiguity.\n' + '\n' + 'An OR pattern matches each of its subpatterns in turn to the ' + 'subject\n' + 'value, until one succeeds. The OR pattern is then considered\n' + 'successful. Otherwise, if none of the subpatterns succeed, the ' + 'OR\n' + 'pattern fails.\n' + '\n' + 'In simple terms, "P1 | P2 | ..." will try to match "P1", if it ' + 'fails\n' + 'it will try to match "P2", succeeding immediately if any ' + 'succeeds,\n' + 'failing otherwise.\n' + '\n' + '\n' + 'AS Patterns\n' + '~~~~~~~~~~~\n' + '\n' + 'An AS pattern matches an OR pattern on the left of the "as" ' + 'keyword\n' + 'against a subject. Syntax:\n' + '\n' + ' as_pattern ::= or_pattern "as" capture_pattern\n' + '\n' + 'If the OR pattern fails, the AS pattern fails. Otherwise, the ' + 'AS\n' + 'pattern binds the subject to the name on the right of the as ' + 'keyword\n' + 'and succeeds. "capture_pattern" cannot be a a "_".\n' + '\n' + 'In simple terms "P as NAME" will match with "P", and on success ' + 'it\n' + 'will set "NAME = ".\n' + '\n' + '\n' + 'Literal Patterns\n' + '~~~~~~~~~~~~~~~~\n' + '\n' + 'A literal pattern corresponds to most literals in Python. ' + 'Syntax:\n' + '\n' + ' literal_pattern ::= signed_number\n' + ' | signed_number "+" NUMBER\n' + ' | signed_number "-" NUMBER\n' + ' | strings\n' + ' | "None"\n' + ' | "True"\n' + ' | "False"\n' + ' | signed_number: NUMBER | "-" NUMBER\n' + '\n' + 'The rule "strings" and the token "NUMBER" are defined in the ' + 'standard\n' + 'Python grammar. Triple-quoted strings are supported. Raw ' + 'strings and\n' + 'byte strings are supported. Formatted string literals are not\n' + 'supported.\n' + '\n' + 'The forms "signed_number \'+\' NUMBER" and "signed_number \'-\' ' + 'NUMBER"\n' + 'are for expressing complex numbers; they require a real number ' + 'on the\n' + 'left and an imaginary number on the right. E.g. "3 + 4j".\n' + '\n' + 'In simple terms, "LITERAL" will succeed only if " ==\n' + 'LITERAL". For the singletons "None", "True" and "False", the ' + '"is"\n' + 'operator is used.\n' + '\n' + '\n' + 'Capture Patterns\n' + '~~~~~~~~~~~~~~~~\n' + '\n' + 'A capture pattern binds the subject value to a name. Syntax:\n' + '\n' + " capture_pattern ::= !'_' NAME\n" + '\n' + 'A single underscore "_" is not a capture pattern (this is what ' + '"!\'_\'"\n' + 'expresses). It is instead treated as a "wildcard_pattern".\n' + '\n' + 'In a given pattern, a given name can only be bound once. E.g. ' + '"case\n' + 'x, x: ..." is invalid while "case [x] | x: ..." is allowed.\n' + '\n' + 'Capture patterns always succeed. The binding follows scoping ' + 'rules\n' + 'established by the assignment expression operator in **PEP ' + '572**; the\n' + 'name becomes a local variable in the closest containing function ' + 'scope\n' + 'unless there’s an applicable "global" or "nonlocal" statement.\n' + '\n' + 'In simple terms "NAME" will always succeed and it will set "NAME ' + '=\n' + '".\n' + '\n' + '\n' + 'Wildcard Patterns\n' + '~~~~~~~~~~~~~~~~~\n' + '\n' + 'A wildcard pattern always succeeds (matches anything) and binds ' + 'no\n' + 'name. Syntax:\n' + '\n' + " wildcard_pattern ::= '_'\n" + '\n' + '"_" is a soft keyword within any pattern, but only within ' + 'patterns.\n' + 'It is an identifier, as usual, even within "match" subject\n' + 'expressions, "guard"s, and "case" blocks.\n' + '\n' + 'In simple terms, "_" will always succeed.\n' + '\n' + '\n' + 'Value Patterns\n' + '~~~~~~~~~~~~~~\n' + '\n' + 'A value pattern represents a named value in Python. Syntax:\n' + '\n' + ' value_pattern ::= attr\n' + ' attr ::= name_or_attr "." NAME\n' + ' name_or_attr ::= attr | NAME\n' + '\n' + 'The dotted name in the pattern is looked up using standard ' + 'Python name\n' + 'resolution rules. The pattern succeeds if the value found ' + 'compares\n' + 'equal to the subject value (using the "==" equality operator).\n' + '\n' + 'In simple terms "NAME1.NAME2" will succeed only if " ' + '==\n' + 'NAME1.NAME2"\n' + '\n' + 'Note:\n' + '\n' + ' If the same value occurs multiple times in the same match ' + 'statement,\n' + ' the interpreter may cache the first value found and reuse it ' + 'rather\n' + ' than repeat the same lookup. This cache is strictly tied to a ' + 'given\n' + ' execution of a given match statement.\n' + '\n' + '\n' + 'Group Patterns\n' + '~~~~~~~~~~~~~~\n' + '\n' + 'A group pattern allows users to add parentheses around patterns ' + 'to\n' + 'emphasize the intended grouping. Otherwise, it has no ' + 'additional\n' + 'syntax. Syntax:\n' + '\n' + ' group_pattern ::= "(" pattern ")"\n' + '\n' + 'In simple terms "(P)" has the same effect as "P".\n' + '\n' + '\n' + 'Sequence Patterns\n' + '~~~~~~~~~~~~~~~~~\n' + '\n' + 'A sequence pattern contains several subpatterns to be matched ' + 'against\n' + 'sequence elements. The syntax is similar to the unpacking of a ' + 'list or\n' + 'tuple.\n' + '\n' + ' sequence_pattern ::= "[" [maybe_sequence_pattern] "]"\n' + ' | "(" [open_sequence_pattern] ")"\n' + ' open_sequence_pattern ::= maybe_star_pattern "," ' + '[maybe_sequence_pattern]\n' + ' maybe_sequence_pattern ::= ",".maybe_star_pattern+ ","?\n' + ' maybe_star_pattern ::= star_pattern | pattern\n' + ' star_pattern ::= "*" (capture_pattern | ' + 'wildcard_pattern)\n' + '\n' + 'There is no difference if parentheses or square brackets are ' + 'used for\n' + 'sequence patterns (i.e. "(...)" vs "[...]" ).\n' + '\n' + 'Note:\n' + '\n' + ' A single pattern enclosed in parentheses without a trailing ' + 'comma\n' + ' (e.g. "(3 | 4)") is a group pattern. While a single pattern ' + 'enclosed\n' + ' in square brackets (e.g. "[3 | 4]") is still a sequence ' + 'pattern.\n' + '\n' + 'At most one star subpattern may be in a sequence pattern. The ' + 'star\n' + 'subpattern may occur in any position. If no star subpattern is\n' + 'present, the sequence pattern is a fixed-length sequence ' + 'pattern;\n' + 'otherwise it is a variable-length sequence pattern.\n' + '\n' + 'The following is the logical flow for matching a sequence ' + 'pattern\n' + 'against a subject value:\n' + '\n' + '1. If the subject value is not a sequence [2], the sequence ' + 'pattern\n' + ' fails.\n' + '\n' + '2. If the subject value is an instance of "str", "bytes" or\n' + ' "bytearray" the sequence pattern fails.\n' + '\n' + '3. The subsequent steps depend on whether the sequence pattern ' + 'is\n' + ' fixed or variable-length.\n' + '\n' + ' If the sequence pattern is fixed-length:\n' + '\n' + ' 1. If the length of the subject sequence is not equal to the ' + 'number\n' + ' of subpatterns, the sequence pattern fails\n' + '\n' + ' 2. Subpatterns in the sequence pattern are matched to their\n' + ' corresponding items in the subject sequence from left to ' + 'right.\n' + ' Matching stops as soon as a subpattern fails. If all\n' + ' subpatterns succeed in matching their corresponding item, ' + 'the\n' + ' sequence pattern succeeds.\n' + '\n' + ' Otherwise, if the sequence pattern is variable-length:\n' + '\n' + ' 1. If the length of the subject sequence is less than the ' + 'number of\n' + ' non-star subpatterns, the sequence pattern fails.\n' + '\n' + ' 2. The leading non-star subpatterns are matched to their\n' + ' corresponding items as for fixed-length sequences.\n' + '\n' + ' 3. If the previous step succeeds, the star subpattern matches ' + 'a\n' + ' list formed of the remaining subject items, excluding the\n' + ' remaining items corresponding to non-star subpatterns ' + 'following\n' + ' the star subpattern.\n' + '\n' + ' 4. Remaining non-star subpatterns are matched to their\n' + ' corresponding subject items, as for a fixed-length ' + 'sequence.\n' + '\n' + ' Note:\n' + '\n' + ' The length of the subject sequence is obtained via "len()" ' + '(i.e.\n' + ' via the "__len__()" protocol). This length may be cached ' + 'by the\n' + ' interpreter in a similar manner as value patterns.\n' + '\n' + 'In simple terms "[P1, P2, P3," … ", P]" matches only if all ' + 'the\n' + 'following happens:\n' + '\n' + '* check "" is a sequence\n' + '\n' + '* "len(subject) == "\n' + '\n' + '* "P1" matches "[0]" (note that this match can also ' + 'bind\n' + ' names)\n' + '\n' + '* "P2" matches "[1]" (note that this match can also ' + 'bind\n' + ' names)\n' + '\n' + '* … and so on for the corresponding pattern/element.\n' + '\n' + '\n' + 'Mapping Patterns\n' + '~~~~~~~~~~~~~~~~\n' + '\n' + 'A mapping pattern contains one or more key-value patterns. The ' + 'syntax\n' + 'is similar to the construction of a dictionary. Syntax:\n' + '\n' + ' mapping_pattern ::= "{" [items_pattern] "}"\n' + ' items_pattern ::= ",".key_value_pattern+ ","?\n' + ' key_value_pattern ::= (literal_pattern | value_pattern) ":" ' + 'pattern\n' + ' | double_star_pattern\n' + ' double_star_pattern ::= "**" capture_pattern\n' + '\n' + 'At most one double star pattern may be in a mapping pattern. ' + 'The\n' + 'double star pattern must be the last subpattern in the mapping\n' + 'pattern.\n' + '\n' + 'Duplicate keys in mapping patterns are disallowed. Duplicate ' + 'literal\n' + 'keys will raise a "SyntaxError". Two keys that otherwise have ' + 'the same\n' + 'value will raise a "ValueError" at runtime.\n' + '\n' + 'The following is the logical flow for matching a mapping ' + 'pattern\n' + 'against a subject value:\n' + '\n' + '1. If the subject value is not a mapping [3],the mapping ' + 'pattern\n' + ' fails.\n' + '\n' + '2. If every key given in the mapping pattern is present in the ' + 'subject\n' + ' mapping, and the pattern for each key matches the ' + 'corresponding\n' + ' item of the subject mapping, the mapping pattern succeeds.\n' + '\n' + '3. If duplicate keys are detected in the mapping pattern, the ' + 'pattern\n' + ' is considered invalid. A "SyntaxError" is raised for ' + 'duplicate\n' + ' literal values; or a "ValueError" for named keys of the same ' + 'value.\n' + '\n' + 'Note:\n' + '\n' + ' Key-value pairs are matched using the two-argument form of ' + 'the\n' + ' mapping subject’s "get()" method. Matched key-value pairs ' + 'must\n' + ' already be present in the mapping, and not created on-the-fly ' + 'via\n' + ' "__missing__()" or "__getitem__()".\n' + '\n' + 'In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all ' + 'the\n' + 'following happens:\n' + '\n' + '* check "" is a mapping\n' + '\n' + '* "KEY1 in "\n' + '\n' + '* "P1" matches "[KEY1]"\n' + '\n' + '* … and so on for the corresponding KEY/pattern pair.\n' + '\n' + '\n' + 'Class Patterns\n' + '~~~~~~~~~~~~~~\n' + '\n' + 'A class pattern represents a class and its positional and ' + 'keyword\n' + 'arguments (if any). Syntax:\n' + '\n' + ' class_pattern ::= name_or_attr "(" [pattern_arguments ' + '","?] ")"\n' + ' pattern_arguments ::= positional_patterns ["," ' + 'keyword_patterns]\n' + ' | keyword_patterns\n' + ' positional_patterns ::= ",".pattern+\n' + ' keyword_patterns ::= ",".keyword_pattern+\n' + ' keyword_pattern ::= NAME "=" pattern\n' + '\n' + 'The same keyword should not be repeated in class patterns.\n' + '\n' + 'The following is the logical flow for matching a class pattern ' + 'against\n' + 'a subject value:\n' + '\n' + '1. If "name_or_attr" is not an instance of the builtin "type" , ' + 'raise\n' + ' "TypeError".\n' + '\n' + '2. If the subject value is not an instance of "name_or_attr" ' + '(tested\n' + ' via "isinstance()"), the class pattern fails.\n' + '\n' + '3. If no pattern arguments are present, the pattern succeeds.\n' + ' Otherwise, the subsequent steps depend on whether keyword or\n' + ' positional argument patterns are present.\n' + '\n' + ' For a number of built-in types (specified below), a single\n' + ' positional subpattern is accepted which will match the ' + 'entire\n' + ' subject; for these types keyword patterns also work as for ' + 'other\n' + ' types.\n' + '\n' + ' If only keyword patterns are present, they are processed as\n' + ' follows, one by one:\n' + '\n' + ' I. The keyword is looked up as an attribute on the subject.\n' + '\n' + ' * If this raises an exception other than "AttributeError", ' + 'the\n' + ' exception bubbles up.\n' + '\n' + ' * If this raises "AttributeError", the class pattern has ' + 'failed.\n' + '\n' + ' * Else, the subpattern associated with the keyword pattern ' + 'is\n' + ' matched against the subject’s attribute value. If this ' + 'fails,\n' + ' the class pattern fails; if this succeeds, the match ' + 'proceeds\n' + ' to the next keyword.\n' + '\n' + ' II. If all keyword patterns succeed, the class pattern ' + 'succeeds.\n' + '\n' + ' If any positional patterns are present, they are converted ' + 'to\n' + ' keyword patterns using the "__match_args__" attribute on the ' + 'class\n' + ' "name_or_attr" before matching:\n' + '\n' + ' I. The equivalent of "getattr(cls, "__match_args__", ())" is\n' + ' called.\n' + '\n' + ' * If this raises an exception, the exception bubbles up.\n' + '\n' + ' * If the returned value is not a tuple, the conversion ' + 'fails and\n' + ' "TypeError" is raised.\n' + '\n' + ' * If there are more positional patterns than\n' + ' "len(cls.__match_args__)", "TypeError" is raised.\n' + '\n' + ' * Otherwise, positional pattern "i" is converted to a ' + 'keyword\n' + ' pattern using "__match_args__[i]" as the keyword.\n' + ' "__match_args__[i]" must be a string; if not "TypeError" ' + 'is\n' + ' raised.\n' + '\n' + ' * If there are duplicate keywords, "TypeError" is raised.\n' + '\n' + ' See also:\n' + '\n' + ' Customizing positional arguments in class pattern ' + 'matching\n' + '\n' + ' II. Once all positional patterns have been converted to ' + 'keyword\n' + ' patterns,\n' + ' the match proceeds as if there were only keyword ' + 'patterns.\n' + '\n' + ' For the following built-in types the handling of positional\n' + ' subpatterns is different:\n' + '\n' + ' * "bool"\n' + '\n' + ' * "bytearray"\n' + '\n' + ' * "bytes"\n' + '\n' + ' * "dict"\n' + '\n' + ' * "float"\n' + '\n' + ' * "frozenset"\n' + '\n' + ' * "int"\n' + '\n' + ' * "list"\n' + '\n' + ' * "set"\n' + '\n' + ' * "str"\n' + '\n' + ' * "tuple"\n' + '\n' + ' These classes accept a single positional argument, and the ' + 'pattern\n' + ' there is matched against the whole object rather than an ' + 'attribute.\n' + ' For example "int(0|1)" matches the value "0", but not the ' + 'values\n' + ' "0.0" or "False".\n' + '\n' + 'In simple terms "CLS(P1, attr=P2)" matches only if the ' + 'following\n' + 'happens:\n' + '\n' + '* "isinstance(, CLS)"\n' + '\n' + '* convert "P1" to a keyword pattern using "CLS.__match_args__"\n' + '\n' + '* For each keyword argument "attr=P2":\n' + ' * "hasattr(, "attr")"\n' + '\n' + ' * "P2" matches ".attr"\n' + '\n' + '* … and so on for the corresponding keyword argument/pattern ' + 'pair.\n' + '\n' + 'See also:\n' + '\n' + ' * **PEP 634** – Structural Pattern Matching: Specification\n' + '\n' + ' * **PEP 636** – Structural Pattern Matching: Tutorial\n' + '\n' + '\n' 'Function definitions\n' '====================\n' '\n' @@ -2717,7 +3496,7 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The function definition does not execute the function body; this ' 'gets\n' - 'executed only when the function is called. [2]\n' + 'executed only when the function is called. [4]\n' '\n' 'A function definition may be wrapped by one or more *decorator*\n' 'expressions. Decorator expressions are evaluated when the ' @@ -2770,17 +3549,17 @@ topics = {'assert': 'The "assert" statement\n' '“pre-\n' 'computed” value is used for each call. This is especially ' 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' + 'to understand when a default parameter value is a mutable ' + 'object, such\n' + 'as a list or a dictionary: if the function modifies the object ' + '(e.g.\n' + 'by appending an item to a list), the default parameter value is ' + 'in\n' + 'effect modified. This is generally not what was intended. A ' + 'way\n' + 'around this is to use "None" as the default, and explicitly test ' + 'for\n' + 'it in the body of the function, e.g.:\n' '\n' ' def whats_on_the_telly(penguin=None):\n' ' if penguin is None:\n' @@ -2922,7 +3701,7 @@ topics = {'assert': 'The "assert" statement\n' 'function definitions.) When the class’s suite finishes ' 'execution, its\n' 'execution frame is discarded but its local namespace is saved. ' - '[3] A\n' + '[5] A\n' 'class object is then created using the inheritance list for the ' 'base\n' 'classes and the saved local namespace for the attribute ' @@ -3007,12 +3786,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'Execution of Python coroutines can be suspended and resumed at ' 'many\n' - 'points (see *coroutine*). Inside the body of a coroutine ' - 'function,\n' - '"await" and "async" identifiers become reserved keywords; ' - '"await"\n' - 'expressions, "async for" and "async with" can only be used in\n' - 'coroutine function bodies.\n' + 'points (see *coroutine*). "await" expressions, "async for" and ' + '"async\n' + 'with" can only be used in the body of a coroutine function.\n' '\n' 'Functions defined with "async def" syntax are always coroutine\n' 'functions, even if they do not contain "await" or "async" ' @@ -3028,6 +3804,10 @@ topics = {'assert': 'The "assert" statement\n' ' do_stuff()\n' ' await some_coroutine()\n' '\n' + 'Changed in version 3.7: "await" and "async" are now keywords;\n' + 'previously they were only treated as such inside the body of a\n' + 'coroutine function.\n' + '\n' '\n' 'The "async for" statement\n' '-------------------------\n' @@ -3125,13 +3905,65 @@ topics = {'assert': 'The "assert" statement\n' 'exception.\n' ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the ' + '[2] In pattern matching, a sequence is defined as one of the\n' + ' following:\n' + '\n' + ' * a class that inherits from "collections.abc.Sequence"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Sequence"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_SEQUENCE"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The following standard library classes are sequences:\n' + '\n' + ' * "array.array"\n' + '\n' + ' * "collections.deque"\n' + '\n' + ' * "list"\n' + '\n' + ' * "memoryview"\n' + '\n' + ' * "range"\n' + '\n' + ' * "tuple"\n' + '\n' + ' Note:\n' + '\n' + ' Subject values of type "str", "bytes", and "bytearray" do ' + 'not\n' + ' match sequence patterns.\n' + '\n' + '[3] In pattern matching, a mapping is defined as one of the ' + 'following:\n' + '\n' + ' * a class that inherits from "collections.abc.Mapping"\n' + '\n' + ' * a Python class that has been registered as\n' + ' "collections.abc.Mapping"\n' + '\n' + ' * a builtin class that has its (CPython) ' + '"Py_TPFLAGS_MAPPING"\n' + ' bit set\n' + '\n' + ' * a class that inherits from any of the above\n' + '\n' + ' The standard library classes "dict" and ' + '"types.MappingProxyType"\n' + ' are mappings.\n' + '\n' + '[4] A string literal appearing as the first statement in the ' 'function\n' ' body is transformed into the function’s "__doc__" attribute ' 'and\n' ' therefore the function’s *docstring*.\n' '\n' - '[3] A string literal appearing as the first statement in the ' + '[5] A string literal appearing as the first statement in the ' 'class\n' ' body is transformed into the namespace’s "__doc__" item and\n' ' therefore the class’s *docstring*.\n', @@ -4637,20 +5469,32 @@ topics = {'assert': 'The "assert" statement\n' 'binding\n' 'operations.\n' '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, ' - 'or after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. ' - 'This form\n' - 'may only be used at the module level.\n' + 'The following constructs bind names:\n' + '\n' + '* formal parameters to functions,\n' + '\n' + '* class definitions,\n' + '\n' + '* function definitions,\n' + '\n' + '* assignment expressions,\n' + '\n' + '* targets that are identifiers if occurring in an assignment:\n' + '\n' + ' * "for" loop header,\n' + '\n' + ' * after "as" in a "with" statement, "except" clause or in the ' + 'as-\n' + ' pattern in structural pattern matching,\n' + '\n' + ' * in a capture pattern in structural pattern matching\n' + '\n' + '* "import" statements.\n' + '\n' + 'The "import" statement of the form "from ... import *" binds ' + 'all names\n' + 'defined in the imported module, except those beginning with an\n' + 'underscore. This form may only be used at the module level.\n' '\n' 'A target occurring in a "del" statement is also considered ' 'bound for\n' @@ -5031,30 +5875,7 @@ topics = {'assert': 'The "assert" statement\n' 'all by the loop. Hint: the built-in function "range()" returns an\n' 'iterator of integers suitable to emulate the effect of Pascal’s "for ' 'i\n' - ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n' - '\n' - 'Note:\n' - '\n' - ' There is a subtlety when the sequence is being modified by the ' - 'loop\n' - ' (this can only occur for mutable sequences, e.g. lists). An\n' - ' internal counter is used to keep track of which item is used next,\n' - ' and this is incremented on each iteration. When this counter has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item from ' - 'the\n' - ' sequence, the next item will be skipped (since it gets the index ' - 'of\n' - ' the current item which has already been treated). Likewise, if ' - 'the\n' - ' suite inserts an item in the sequence before the current item, the\n' - ' current item will be treated again the next time through the loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n', + ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n', 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' @@ -5300,9 +6121,9 @@ topics = {'assert': 'The "assert" statement\n' ' | | in the form ‘+000000120’. This alignment ' 'option is only |\n' ' | | valid for numeric types. It becomes the ' - 'default when ‘0’ |\n' - ' | | immediately precedes the field ' - 'width. |\n' + 'default for |\n' + ' | | numbers when ‘0’ immediately precedes the ' + 'field width. |\n' ' ' '+-----------+------------------------------------------------------------+\n' ' | "\'^\'" | Forces the field to be centered within ' @@ -5410,6 +6231,10 @@ topics = {'assert': 'The "assert" statement\n' 'with an\n' '*alignment* type of "\'=\'".\n' '\n' + 'Changed in version 3.10: Preceding the *width* field by ' + '"\'0\'" no\n' + 'longer affects the default alignment for strings.\n' + '\n' 'The *precision* is a decimal integer indicating how many ' 'digits should\n' 'be displayed after the decimal point for presentation types ' @@ -5868,7 +6693,7 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The function definition does not execute the function body; this ' 'gets\n' - 'executed only when the function is called. [2]\n' + 'executed only when the function is called. [4]\n' '\n' 'A function definition may be wrapped by one or more *decorator*\n' 'expressions. Decorator expressions are evaluated when the ' @@ -5921,17 +6746,17 @@ topics = {'assert': 'The "assert" statement\n' '“pre-\n' 'computed” value is used for each call. This is especially ' 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' + 'to understand when a default parameter value is a mutable ' + 'object, such\n' + 'as a list or a dictionary: if the function modifies the object ' + '(e.g.\n' + 'by appending an item to a list), the default parameter value is ' + 'in\n' + 'effect modified. This is generally not what was intended. A ' + 'way\n' + 'around this is to use "None" as the default, and explicitly test ' + 'for\n' + 'it in the body of the function, e.g.:\n' '\n' ' def whats_on_the_telly(penguin=None):\n' ' if penguin is None:\n' @@ -6054,8 +6879,10 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'Names listed in a "global" statement must not be defined as ' 'formal\n' - 'parameters or in a "for" loop control target, "class" definition,\n' - 'function definition, "import" statement, or variable annotation.\n' + 'parameters, or as targets in "with" statements or "except" ' + 'clauses, or\n' + 'in a "for" target list, "class" definition, function definition,\n' + '"import" statement, or variable annotation.\n' '\n' '**CPython implementation detail:** The current implementation does ' 'not\n' @@ -6087,22 +6914,31 @@ topics = {'assert': 'The "assert" statement\n' 'trailing underscore characters:\n' '\n' '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' - 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' + ' Not imported by "from module import *".\n' + '\n' + '"_"\n' + ' In a "case" pattern within a "match" statement, "_" is a ' + 'soft\n' + ' keyword that denotes a wildcard.\n' + '\n' + ' Separately, the interactive interpreter makes the result of ' + 'the\n' + ' last evaluation available in the variable "_". (It is ' + 'stored in the\n' + ' "builtins" module, alongside built-in functions like ' + '"print".)\n' + '\n' + ' Elsewhere, "_" is a regular identifier. It is often used to ' + 'name\n' + ' “special” items, but it is not special to Python itself.\n' '\n' ' Note:\n' '\n' ' The name "_" is often used in conjunction with\n' ' internationalization; refer to the documentation for the\n' ' "gettext" module for more information on this ' - 'convention.\n' + 'convention.It is\n' + ' also commonly used for unused variables.\n' '\n' '"__*__"\n' ' System-defined names, informally known as “dunder” names. ' @@ -6225,6 +7061,28 @@ topics = {'assert': 'The "assert" statement\n' ' async elif if or yield\n' '\n' '\n' + 'Soft Keywords\n' + '=============\n' + '\n' + 'New in version 3.10.\n' + '\n' + 'Some identifiers are only reserved under specific contexts. ' + 'These are\n' + 'known as *soft keywords*. The identifiers "match", "case" ' + 'and "_" can\n' + 'syntactically act as keywords in contexts related to the ' + 'pattern\n' + 'matching statement, but this distinction is done at the ' + 'parser level,\n' + 'not when tokenizing.\n' + '\n' + 'As soft keywords, their use with pattern matching is possible ' + 'while\n' + 'still preserving compatibility with existing code that uses ' + '"match",\n' + '"case" and "_" as identifier names.\n' + '\n' + '\n' 'Reserved classes of identifiers\n' '===============================\n' '\n' @@ -6235,15 +7093,23 @@ topics = {'assert': 'The "assert" statement\n' 'trailing underscore characters:\n' '\n' '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' + ' Not imported by "from module import *".\n' + '\n' + '"_"\n' + ' In a "case" pattern within a "match" statement, "_" is a ' + 'soft\n' + ' keyword that denotes a wildcard.\n' + '\n' + ' Separately, the interactive interpreter makes the result ' 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' + ' last evaluation available in the variable "_". (It is ' + 'stored in the\n' + ' "builtins" module, alongside built-in functions like ' + '"print".)\n' + '\n' + ' Elsewhere, "_" is a regular identifier. It is often used ' + 'to name\n' + ' “special” items, but it is not special to Python itself.\n' '\n' ' Note:\n' '\n' @@ -6251,7 +7117,8 @@ topics = {'assert': 'The "assert" statement\n' ' internationalization; refer to the documentation for ' 'the\n' ' "gettext" module for more information on this ' - 'convention.\n' + 'convention.It is\n' + ' also commonly used for unused variables.\n' '\n' '"__*__"\n' ' System-defined names, informally known as “dunder” names. ' @@ -6712,20 +7579,32 @@ topics = {'assert': 'The "assert" statement\n' '*Names* refer to objects. Names are introduced by name binding\n' 'operations.\n' '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, or ' - 'after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. This ' - 'form\n' - 'may only be used at the module level.\n' + 'The following constructs bind names:\n' + '\n' + '* formal parameters to functions,\n' + '\n' + '* class definitions,\n' + '\n' + '* function definitions,\n' + '\n' + '* assignment expressions,\n' + '\n' + '* targets that are identifiers if occurring in an assignment:\n' + '\n' + ' * "for" loop header,\n' + '\n' + ' * after "as" in a "with" statement, "except" clause or in the ' + 'as-\n' + ' pattern in structural pattern matching,\n' + '\n' + ' * in a capture pattern in structural pattern matching\n' + '\n' + '* "import" statements.\n' + '\n' + 'The "import" statement of the form "from ... import *" binds all ' + 'names\n' + 'defined in the imported module, except those beginning with an\n' + 'underscore. This form may only be used at the module level.\n' '\n' 'A target occurring in a "del" statement is also considered bound ' 'for\n' @@ -7092,16 +7971,6 @@ topics = {'assert': 'The "assert" statement\n' 'the data\n' ' model.\n' '\n' - ' Note:\n' - '\n' - ' Due to a bug in the dispatching mechanism for "**=", a ' - 'class that\n' - ' defines "__ipow__()" but returns "NotImplemented" ' - 'would fail to\n' - ' fall back to "x.__pow__(y)" and "y.__rpow__(x)". This ' - 'bug is\n' - ' fixed in Python 3.10.\n' - '\n' 'object.__neg__(self)\n' 'object.__pos__(self)\n' 'object.__abs__(self)\n' @@ -7851,19 +8720,13 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'object.__iter__(self)\n' '\n' - ' This method is called when an iterator is required for ' - 'a container.\n' - ' This method should return a new iterator object that ' - 'can iterate\n' - ' over all the objects in the container. For mappings, ' - 'it should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; ' - 'they are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' + ' This method is called when an *iterator* is required ' + 'for a\n' + ' container. This method should return a new iterator ' + 'object that can\n' + ' iterate over all the objects in the container. For ' + 'mappings, it\n' + ' should iterate over the keys of the container.\n' '\n' 'object.__reversed__(self)\n' '\n' @@ -8820,32 +9683,6 @@ topics = {'assert': 'The "assert" statement\n' 'of the\n' ' owner class.\n' '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is created. ' - 'The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' Note:\n' - '\n' - ' "__set_name__()" is only called implicitly as part of ' - 'the "type"\n' - ' constructor, so it will need to be called explicitly ' - 'with the\n' - ' appropriate parameters when a descriptor is added to a ' - 'class\n' - ' after initial creation:\n' - '\n' - ' class A:\n' - ' pass\n' - ' descr = custom_descriptor()\n' - ' A.attr = descr\n' - " descr.__set_name__(A, 'attr')\n" - '\n' - ' See Creating the class object for more details.\n' - '\n' - ' New in version 3.6.\n' - '\n' 'The attribute "__objclass__" is interpreted by the "inspect" ' 'module as\n' 'specifying the class where this object was defined (setting ' @@ -9141,6 +9978,38 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' New in version 3.6.\n' '\n' + 'When a class is created, "type.__new__()" scans the class ' + 'variables\n' + 'and makes callbacks to those with a "__set_name__()" hook.\n' + '\n' + 'object.__set_name__(self, owner, name)\n' + '\n' + ' Automatically called at the time the owning class *owner* ' + 'is\n' + ' created. The object has been assigned to *name* in that ' + 'class:\n' + '\n' + ' class A:\n' + ' x = C() # Automatically calls: x.__set_name__(A, ' + "'x')\n" + '\n' + ' If the class variable is assigned after the class is ' + 'created,\n' + ' "__set_name__()" will not be called automatically. If ' + 'needed,\n' + ' "__set_name__()" can be called directly:\n' + '\n' + ' class A:\n' + ' pass\n' + '\n' + ' c = C()\n' + ' A.x = c # The hook is not called\n' + " c.__set_name__(A, 'x') # Manually invoke the hook\n" + '\n' + ' See Creating the class object for more details.\n' + '\n' + ' New in version 3.6.\n' + '\n' '\n' 'Metaclasses\n' '-----------\n' @@ -9336,22 +10205,21 @@ topics = {'assert': 'The "assert" statement\n' 'When using the default metaclass "type", or any metaclass ' 'that\n' 'ultimately calls "type.__new__", the following additional\n' - 'customisation steps are invoked after creating the class ' + 'customization steps are invoked after creating the class ' 'object:\n' '\n' - '* first, "type.__new__" collects all of the descriptors in ' - 'the class\n' - ' namespace that define a "__set_name__()" method;\n' + '1. The "type.__new__" method collects all of the attributes ' + 'in the\n' + ' class namespace that define a "__set_name__()" method;\n' '\n' - '* second, all of these "__set_name__" methods are called ' - 'with the\n' - ' class being defined and the assigned name of that ' - 'particular\n' - ' descriptor;\n' + '2. Those "__set_name__" methods are called with the class ' + 'being\n' + ' defined and the assigned name of that particular ' + 'attribute;\n' '\n' - '* finally, the "__init_subclass__()" hook is called on the ' - 'immediate\n' - ' parent of the new class in its method resolution order.\n' + '3. The "__init_subclass__()" hook is called on the immediate ' + 'parent of\n' + ' the new class in its method resolution order.\n' '\n' 'After the class object is created, it is passed to the ' 'class\n' @@ -9814,19 +10682,13 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'object.__iter__(self)\n' '\n' - ' This method is called when an iterator is required for a ' - 'container.\n' - ' This method should return a new iterator object that can ' - 'iterate\n' - ' over all the objects in the container. For mappings, it ' - 'should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; they ' - 'are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' + ' This method is called when an *iterator* is required for ' + 'a\n' + ' container. This method should return a new iterator ' + 'object that can\n' + ' iterate over all the objects in the container. For ' + 'mappings, it\n' + ' should iterate over the keys of the container.\n' '\n' 'object.__reversed__(self)\n' '\n' @@ -10019,16 +10881,6 @@ topics = {'assert': 'The "assert" statement\n' 'the data\n' ' model.\n' '\n' - ' Note:\n' - '\n' - ' Due to a bug in the dispatching mechanism for "**=", a ' - 'class that\n' - ' defines "__ipow__()" but returns "NotImplemented" would ' - 'fail to\n' - ' fall back to "x.__pow__(y)" and "y.__rpow__(x)". This ' - 'bug is\n' - ' fixed in Python 3.10.\n' - '\n' 'object.__neg__(self)\n' 'object.__pos__(self)\n' 'object.__abs__(self)\n' @@ -10147,6 +10999,51 @@ topics = {'assert': 'The "assert" statement\n' ' statement.\n' '\n' '\n' + 'Customizing positional arguments in class pattern matching\n' + '==========================================================\n' + '\n' + 'When using a class name in a pattern, positional arguments ' + 'in the\n' + 'pattern are not allowed by default, i.e. "case MyClass(x, ' + 'y)" is\n' + 'typically invalid without special support in "MyClass". To ' + 'be able to\n' + 'use that kind of patterns, the class needs to define a\n' + '*__match_args__* attribute.\n' + '\n' + 'object.__match_args__\n' + '\n' + ' This class variable can be assigned a tuple of strings. ' + 'When this\n' + ' class is used in a class pattern with positional ' + 'arguments, each\n' + ' positional argument will be converted into a keyword ' + 'argument,\n' + ' using the corresponding value in *__match_args__* as the ' + 'keyword.\n' + ' The absence of this attribute is equivalent to setting it ' + 'to "()".\n' + '\n' + 'For example, if "MyClass.__match_args__" is "("left", ' + '"center",\n' + '"right")" that means that "case MyClass(x, y)" is equivalent ' + 'to "case\n' + 'MyClass(left=x, center=y)". Note that the number of ' + 'arguments in the\n' + 'pattern must be smaller than or equal to the number of ' + 'elements in\n' + '*__match_args__*; if it is larger, the pattern match attempt ' + 'will\n' + 'raise a "TypeError".\n' + '\n' + 'New in version 3.10.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 634** - Structural Pattern Matching\n' + ' The specification for the Python "match" statement.\n' + '\n' + '\n' 'Special method lookup\n' '=====================\n' '\n' @@ -10317,7 +11214,7 @@ topics = {'assert': 'The "assert" statement\n' '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' - 'str.encode(encoding="utf-8", errors="strict")\n' + "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' 'object. Default\n' @@ -10823,7 +11720,7 @@ topics = {'assert': 'The "assert" statement\n' 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=-1)\n' + 'str.rsplit(sep=None, maxsplit=- 1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -10864,7 +11761,7 @@ topics = {'assert': 'The "assert" statement\n' " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=-1)\n' + 'str.split(sep=None, maxsplit=- 1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11593,9 +12490,31 @@ topics = {'assert': 'The "assert" statement\n' 'the\n' 'exception class, the exception instance and a traceback object (see\n' 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. "sys.exc_info()" values are\n' - 'restored to their previous values (before the call) when returning\n' - 'from a function that handled an exception.\n' + 'program where the exception occurred. The details about the ' + 'exception\n' + 'accessed via "sys.exc_info()" are restored to their previous values\n' + 'when leaving an exception handler:\n' + '\n' + ' >>> print(sys.exc_info())\n' + ' (None, None, None)\n' + ' >>> try:\n' + ' ... raise TypeError\n' + ' ... except:\n' + ' ... print(sys.exc_info())\n' + ' ... try:\n' + ' ... raise ValueError\n' + ' ... except:\n' + ' ... print(sys.exc_info())\n' + ' ... print(sys.exc_info())\n' + ' ...\n' + " (, TypeError(), )\n' + " (, ValueError(), )\n' + " (, TypeError(), )\n' + ' >>> print(sys.exc_info())\n' + ' (None, None, None)\n' '\n' 'The optional "else" clause is executed if the control flow leaves ' 'the\n' @@ -11855,7 +12774,7 @@ topics = {'assert': 'The "assert" statement\n' ' points. All the code points in the range "U+0000 - ' 'U+10FFFF"\n' ' can be represented in a string. Python doesn’t have a ' - '"char"\n' + '*char*\n' ' type; instead, every code point in the string is ' 'represented\n' ' as a string object with length "1". The built-in ' @@ -12115,7 +13034,13 @@ topics = {'assert': 'The "assert" statement\n' '| |\n' ' | | and "\'return\'" for the ' 'return | |\n' - ' | | annotation, if provided. ' + ' | | annotation, if provided. For ' + '| |\n' + ' | | more information on working ' + '| |\n' + ' | | with this attribute, see ' + '| |\n' + ' | | Annotations Best Practices. ' '| |\n' ' ' '+---------------------------+---------------------------------+-------------+\n' @@ -12236,20 +13161,18 @@ topics = {'assert': 'The "assert" statement\n' ' A function or method which uses the "yield" statement (see\n' ' section The yield statement) is called a *generator ' 'function*.\n' - ' Such a function, when called, always returns an iterator ' - 'object\n' - ' which can be used to execute the body of the function: ' - 'calling\n' - ' the iterator’s "iterator.__next__()" method will cause the\n' - ' function to execute until it provides a value using the ' - '"yield"\n' - ' statement. When the function executes a "return" statement ' - 'or\n' - ' falls off the end, a "StopIteration" exception is raised and ' - 'the\n' - ' iterator will have reached the end of the set of values to ' - 'be\n' - ' returned.\n' + ' Such a function, when called, always returns an *iterator*\n' + ' object which can be used to execute the body of the ' + 'function:\n' + ' calling the iterator’s "iterator.__next__()" method will ' + 'cause\n' + ' the function to execute until it provides a value using the\n' + ' "yield" statement. When the function executes a "return"\n' + ' statement or falls off the end, a "StopIteration" exception ' + 'is\n' + ' raised and the iterator will have reached the end of the set ' + 'of\n' + ' values to be returned.\n' '\n' ' Coroutine functions\n' ' A function or method which is defined using "async def" is\n' @@ -12265,9 +13188,9 @@ topics = {'assert': 'The "assert" statement\n' ' which uses the "yield" statement is called a *asynchronous\n' ' generator function*. Such a function, when called, returns ' 'an\n' - ' asynchronous iterator object which can be used in an "async ' - 'for"\n' - ' statement to execute the body of the function.\n' + ' *asynchronous iterator* object which can be used in an ' + '"async\n' + ' for" statement to execute the body of the function.\n' '\n' ' Calling the asynchronous iterator’s "aiterator.__anext__" ' 'method\n' @@ -12340,20 +13263,34 @@ topics = {'assert': 'The "assert" statement\n' ' Attribute assignment updates the module’s namespace dictionary,\n' ' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' '\n' - ' Predefined (writable) attributes: "__name__" is the module’s ' - 'name;\n' - ' "__doc__" is the module’s documentation string, or "None" if\n' - ' unavailable; "__annotations__" (optional) is a dictionary\n' - ' containing *variable annotations* collected during module body\n' - ' execution; "__file__" is the pathname of the file from which ' + ' Predefined (writable) attributes:\n' + '\n' + ' "__name__"\n' + ' The module’s name.\n' + '\n' + ' "__doc__"\n' + ' The module’s documentation string, or "None" if ' + 'unavailable.\n' + '\n' + ' "__file__"\n' + ' The pathname of the file from which the module was loaded, ' + 'if\n' + ' it was loaded from a file. The "__file__" attribute may ' + 'be\n' + ' missing for certain types of modules, such as C modules ' + 'that\n' + ' are statically linked into the interpreter. For ' + 'extension\n' + ' modules loaded dynamically from a shared library, it’s ' 'the\n' - ' module was loaded, if it was loaded from a file. The "__file__"\n' - ' attribute may be missing for certain types of modules, such as ' - 'C\n' - ' modules that are statically linked into the interpreter; for\n' - ' extension modules loaded dynamically from a shared library, it ' - 'is\n' - ' the pathname of the shared library file.\n' + ' pathname of the shared library file.\n' + '\n' + ' "__annotations__"\n' + ' A dictionary containing *variable annotations* collected\n' + ' during module body execution. For best practices on ' + 'working\n' + ' with "__annotations__", please see Annotations Best\n' + ' Practices.\n' '\n' ' Special read-only attribute: "__dict__" is the module’s ' 'namespace\n' @@ -12411,20 +13348,31 @@ topics = {'assert': 'The "assert" statement\n' 'instance\n' ' (see below).\n' '\n' - ' Special attributes: "__name__" is the class name; "__module__" ' - 'is\n' - ' the module name in which the class was defined; "__dict__" is ' - 'the\n' - ' dictionary containing the class’s namespace; "__bases__" is a ' - 'tuple\n' - ' containing the base classes, in the order of their occurrence ' - 'in\n' - ' the base class list; "__doc__" is the class’s documentation ' - 'string,\n' - ' or "None" if undefined; "__annotations__" (optional) is a\n' - ' dictionary containing *variable annotations* collected during ' - 'class\n' - ' body execution.\n' + ' Special attributes:\n' + '\n' + ' "__name__"\n' + ' The class name.\n' + '\n' + ' "__module__"\n' + ' The name of the module in which the class was defined.\n' + '\n' + ' "__dict__"\n' + ' The dictionary containing the class’s namespace.\n' + '\n' + ' "__bases__"\n' + ' A tuple containing the base classes, in the order of ' + 'their\n' + ' occurrence in the base class list.\n' + '\n' + ' "__doc__"\n' + ' The class’s documentation string, or "None" if undefined.\n' + '\n' + ' "__annotations__"\n' + ' A dictionary containing *variable annotations* collected\n' + ' during class body execution. For best practices on ' + 'working\n' + ' with "__annotations__", please see Annotations Best\n' + ' Practices.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -12725,9 +13673,8 @@ topics = {'assert': 'The "assert" statement\n' ' object actually returned is the wrapped object, which is not\n' ' subject to any further transformation. Static method objects ' 'are\n' - ' not themselves callable, although the objects they wrap ' - 'usually\n' - ' are. Static method objects are created by the built-in\n' + ' also callable. Static method objects are created by the ' + 'built-in\n' ' "staticmethod()" constructor.\n' '\n' ' Class method objects\n' @@ -13192,6 +14139,14 @@ topics = {'assert': 'The "assert" statement\n' ' Changed in version 3.8: Dictionary views are now ' 'reversible.\n' '\n' + 'dictview.mapping\n' + '\n' + ' Return a "types.MappingProxyType" that wraps the ' + 'original\n' + ' dictionary to which the view refers.\n' + '\n' + ' New in version 3.10.\n' + '\n' 'Keys views are set-like since their entries are unique and ' 'hashable.\n' 'If all values are hashable, so that "(key, value)" pairs are ' @@ -13237,7 +14192,15 @@ topics = {'assert': 'The "assert" statement\n' " >>> keys & {'eggs', 'bacon', 'salad'}\n" " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" - " {'juice', 'sausage', 'bacon', 'spam'}\n", + " {'juice', 'sausage', 'bacon', 'spam'}\n" + '\n' + ' >>> # get back a read-only proxy for the original ' + 'dictionary\n' + ' >>> values.mapping\n' + " mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, " + "'spam': 500})\n" + " >>> values.mapping['spam']\n" + ' 500\n', 'typesmethods': 'Methods\n' '*******\n' '\n' @@ -13432,6 +14395,14 @@ topics = {'assert': 'The "assert" statement\n' 'Comparisons in\n' 'the language reference.)\n' '\n' + 'Forward and reversed iterators over mutable sequences access ' + 'values\n' + 'using an index. That index will continue to march forward (or\n' + 'backward) even if the underlying sequence is mutated. The ' + 'iterator\n' + 'terminates only when an "IndexError" or a "StopIteration" is\n' + 'encountered (or when the index drops below zero).\n' + '\n' 'Notes:\n' '\n' '1. While the "in" and "not in" operations are used only for ' @@ -14244,8 +15215,10 @@ topics = {'assert': 'The "assert" statement\n' 'usage\n' 'patterns to be encapsulated for convenient reuse.\n' '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' + ' with_stmt ::= "with" ( "(" with_stmt_contents ","? ")" | ' + 'with_stmt_contents ) ":" suite\n' + ' with_stmt_contents ::= with_item ("," with_item)*\n' + ' with_item ::= expression ["as" target]\n' '\n' 'The execution of the "with" statement with one “item” proceeds as\n' 'follows:\n' @@ -14331,8 +15304,20 @@ topics = {'assert': 'The "assert" statement\n' ' with B() as b:\n' ' SUITE\n' '\n' + 'You can also write multi-item context managers in multiple lines if\n' + 'the items are surrounded by parentheses. For example:\n' + '\n' + ' with (\n' + ' A() as a,\n' + ' B() as b,\n' + ' ):\n' + ' SUITE\n' + '\n' 'Changed in version 3.1: Support for multiple context expressions.\n' '\n' + 'Changed in version 3.10: Support for using grouping parentheses to\n' + 'break the statement in multiple lines.\n' + '\n' 'See also:\n' '\n' ' **PEP 343** - The “with” statement\n' diff --git a/contrib/tools/python3/src/Lib/random.py b/contrib/tools/python3/src/Lib/random.py index 1d4b5eb36f1..1310a2d9d0e 100644 --- a/contrib/tools/python3/src/Lib/random.py +++ b/contrib/tools/python3/src/Lib/random.py @@ -48,9 +48,10 @@ General notes on the underlying Mersenne Twister core generator: from warnings import warn as _warn from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin -from math import tau as TWOPI, floor as _floor +from math import tau as TWOPI, floor as _floor, isfinite as _isfinite from os import urandom as _urandom from _collections_abc import Set as _Set, Sequence as _Sequence +from operator import index as _index from itertools import accumulate as _accumulate, repeat as _repeat from bisect import bisect as _bisect import os as _os @@ -96,6 +97,7 @@ LOG4 = _log(4.0) SG_MAGICCONST = 1.0 + _log(4.5) BPF = 53 # Number of bits in a float RECIP_BPF = 2 ** -BPF +_ONE = 1 class Random(_random.Random): @@ -287,7 +289,7 @@ class Random(_random.Random): ## -------------------- integer methods ------------------- - def randrange(self, start, stop=None, step=1): + def randrange(self, start, stop=None, step=_ONE): """Choose a random item from range(start, stop[, step]). This fixes the problem with randint() which includes the @@ -297,38 +299,68 @@ class Random(_random.Random): # This code is a bit messy to make it fast for the # common case while still doing adequate error checking. - istart = int(start) - if istart != start: - raise ValueError("non-integer arg 1 for randrange()") + try: + istart = _index(start) + except TypeError: + istart = int(start) + if istart != start: + _warn('randrange() will raise TypeError in the future', + DeprecationWarning, 2) + raise ValueError("non-integer arg 1 for randrange()") + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) if stop is None: + # We don't check for "step != 1" because it hasn't been + # type checked and converted to an integer yet. + if step is not _ONE: + raise TypeError('Missing a non-None stop argument') if istart > 0: return self._randbelow(istart) raise ValueError("empty range for randrange()") # stop argument supplied. - istop = int(stop) - if istop != stop: - raise ValueError("non-integer stop for randrange()") + try: + istop = _index(stop) + except TypeError: + istop = int(stop) + if istop != stop: + _warn('randrange() will raise TypeError in the future', + DeprecationWarning, 2) + raise ValueError("non-integer stop for randrange()") + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) width = istop - istart - if step == 1 and width > 0: - return istart + self._randbelow(width) - if step == 1: + try: + istep = _index(step) + except TypeError: + istep = int(step) + if istep != step: + _warn('randrange() will raise TypeError in the future', + DeprecationWarning, 2) + raise ValueError("non-integer step for randrange()") + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) + # Fast path. + if istep == 1: + if width > 0: + return istart + self._randbelow(width) raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width)) # Non-unit step argument supplied. - istep = int(step) - if istep != step: - raise ValueError("non-integer step for randrange()") if istep > 0: n = (width + istep - 1) // istep elif istep < 0: n = (width + istep + 1) // istep else: raise ValueError("zero step for randrange()") - if n <= 0: raise ValueError("empty range for randrange()") - return istart + istep * self._randbelow(n) def randint(self, a, b): @@ -424,13 +456,14 @@ class Random(_random.Random): # too many calls to _randbelow(), making them slower and # causing them to eat more entropy than necessary. - if isinstance(population, _Set): - _warn('Sampling from a set deprecated\n' - 'since Python 3.9 and will be removed in a subsequent version.', - DeprecationWarning, 2) - population = tuple(population) if not isinstance(population, _Sequence): - raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).") + if isinstance(population, _Set): + _warn('Sampling from a set deprecated\n' + 'since Python 3.9 and will be removed in a subsequent version.', + DeprecationWarning, 2) + population = tuple(population) + else: + raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).") n = len(population) if counts is not None: cum_counts = list(_accumulate(counts)) @@ -500,6 +533,8 @@ class Random(_random.Random): total = cum_weights[-1] + 0.0 # convert to float if total <= 0.0: raise ValueError('Total of weights must be greater than zero') + if not _isfinite(total): + raise ValueError('Total of weights must be finite') bisect = _bisect hi = n - 1 return [population[bisect(cum_weights, random() * total, 0, hi)] @@ -690,7 +725,7 @@ class Random(_random.Random): bbb = alpha - LOG4 ccc = alpha + ainv - while 1: + while True: u1 = random() if not 1e-7 < u1 < 0.9999999: continue @@ -757,7 +792,7 @@ class Random(_random.Random): # Jain, pg. 495 u = 1.0 - self.random() - return 1.0 / u ** (1.0 / alpha) + return u ** (-1.0 / alpha) def weibullvariate(self, alpha, beta): """Weibull distribution. @@ -853,7 +888,7 @@ def _test_generator(n, func, args): from time import perf_counter t0 = perf_counter() - data = [func(*args) for i in range(n)] + data = [func(*args) for i in _repeat(None, n)] t1 = perf_counter() xbar = mean(data) diff --git a/contrib/tools/python3/src/Lib/re.py b/contrib/tools/python3/src/Lib/re.py index bfb7b1ccd93..1d82b500639 100644 --- a/contrib/tools/python3/src/Lib/re.py +++ b/contrib/tools/python3/src/Lib/re.py @@ -176,7 +176,6 @@ class RegexFlag(enum.IntFlag): res = f'~{res}' return res __str__ = object.__str__ - globals().update(RegexFlag.__members__) # sre exception diff --git a/contrib/tools/python3/src/Lib/rlcompleter.py b/contrib/tools/python3/src/Lib/rlcompleter.py index 923f5c05411..98b7930b32f 100644 --- a/contrib/tools/python3/src/Lib/rlcompleter.py +++ b/contrib/tools/python3/src/Lib/rlcompleter.py @@ -31,6 +31,7 @@ Notes: import atexit import builtins +import inspect import __main__ __all__ = ["Completer"] @@ -96,7 +97,13 @@ class Completer: def _callable_postfix(self, val, word): if callable(val): - word = word + "(" + word += "(" + try: + if not inspect.signature(val).parameters: + word += ")" + except ValueError: + pass + return word def global_matches(self, text): diff --git a/contrib/tools/python3/src/Lib/runpy.py b/contrib/tools/python3/src/Lib/runpy.py index 7e1e1ac5dde..caba1214262 100644 --- a/contrib/tools/python3/src/Lib/runpy.py +++ b/contrib/tools/python3/src/Lib/runpy.py @@ -16,7 +16,6 @@ import importlib.util import io import types import os -from pkgutil import read_code, get_importer __all__ = [ "run_module", "run_path", @@ -233,6 +232,7 @@ def _get_main_module_details(error=ImportError): def _get_code_from_file(run_name, fname): # Check for a compiled file first + from pkgutil import read_code decoded_path = os.path.abspath(os.fsdecode(fname)) with io.open_code(decoded_path) as f: code = read_code(f) @@ -255,6 +255,7 @@ def run_path(path_name, init_globals=None, run_name=None): if run_name is None: run_name = "" pkg_name = run_name.rpartition(".")[0] + from pkgutil import get_importer importer = get_importer(path_name) # Trying to avoid importing imp so as to not consume the deprecation warning. is_NullImporter = False diff --git a/contrib/tools/python3/src/Lib/sched.py b/contrib/tools/python3/src/Lib/sched.py index ff87874a3a4..14613cf2987 100644 --- a/contrib/tools/python3/src/Lib/sched.py +++ b/contrib/tools/python3/src/Lib/sched.py @@ -26,23 +26,19 @@ has another way to reference private data (besides global variables). import time import heapq from collections import namedtuple +from itertools import count import threading from time import monotonic as _time __all__ = ["scheduler"] -class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')): - __slots__ = [] - def __eq__(s, o): return (s.time, s.priority) == (o.time, o.priority) - def __lt__(s, o): return (s.time, s.priority) < (o.time, o.priority) - def __le__(s, o): return (s.time, s.priority) <= (o.time, o.priority) - def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) - def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) - +Event = namedtuple('Event', 'time, priority, sequence, action, argument, kwargs') Event.time.__doc__ = ('''Numeric type compatible with the return value of the timefunc function passed to the constructor.''') Event.priority.__doc__ = ('''Events scheduled for the same time will be executed in the order of their priority.''') +Event.sequence.__doc__ = ('''A continually increasing sequence number that + separates events if time and priority are equal.''') Event.action.__doc__ = ('''Executing the event means executing action(*argument, **kwargs)''') Event.argument.__doc__ = ('''argument is a sequence holding the positional @@ -61,6 +57,7 @@ class scheduler: self._lock = threading.RLock() self.timefunc = timefunc self.delayfunc = delayfunc + self._sequence_generator = count() def enterabs(self, time, priority, action, argument=(), kwargs=_sentinel): """Enter a new event in the queue at an absolute time. @@ -71,8 +68,10 @@ class scheduler: """ if kwargs is _sentinel: kwargs = {} - event = Event(time, priority, action, argument, kwargs) + with self._lock: + event = Event(time, priority, next(self._sequence_generator), + action, argument, kwargs) heapq.heappush(self._queue, event) return event # The ID @@ -136,7 +135,8 @@ class scheduler: with lock: if not q: break - time, priority, action, argument, kwargs = q[0] + (time, priority, sequence, action, + argument, kwargs) = q[0] now = timefunc() if time > now: delay = True diff --git a/contrib/tools/python3/src/Lib/shelve.py b/contrib/tools/python3/src/Lib/shelve.py index 5d443a0fa8d..e053c397345 100644 --- a/contrib/tools/python3/src/Lib/shelve.py +++ b/contrib/tools/python3/src/Lib/shelve.py @@ -56,7 +56,7 @@ entries in the cache, and empty the cache (d.sync() also synchronizes the persistent dictionary on disk, if feasible). """ -from pickle import Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler from io import BytesIO import collections.abc @@ -85,7 +85,7 @@ class Shelf(collections.abc.MutableMapping): keyencoding="utf-8"): self.dict = dict if protocol is None: - protocol = 3 + protocol = DEFAULT_PROTOCOL self._protocol = protocol self.writeback = writeback self.cache = {} diff --git a/contrib/tools/python3/src/Lib/shutil.py b/contrib/tools/python3/src/Lib/shutil.py index c048cdf9b2c..37bf98df796 100644 --- a/contrib/tools/python3/src/Lib/shutil.py +++ b/contrib/tools/python3/src/Lib/shutil.py @@ -32,16 +32,6 @@ try: except ImportError: _LZMA_SUPPORTED = False -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - _WINDOWS = os.name == 'nt' posix = nt = None if os.name == 'posix': @@ -860,8 +850,14 @@ def _is_immutable(src): def _get_gid(name): """Returns a gid, given a group name.""" - if getgrnam is None or name is None: + if name is None: + return None + + try: + from grp import getgrnam + except ImportError: return None + try: result = getgrnam(name) except KeyError: @@ -872,8 +868,14 @@ def _get_gid(name): def _get_uid(name): """Returns an uid, given a user name.""" - if getpwnam is None or name is None: + if name is None: return None + + try: + from pwd import getpwnam + except ImportError: + return None + try: result = getpwnam(name) except KeyError: diff --git a/contrib/tools/python3/src/Lib/site.py b/contrib/tools/python3/src/Lib/site.py index 19c10bc07ff..844505eb5c9 100644 --- a/contrib/tools/python3/src/Lib/site.py +++ b/contrib/tools/python3/src/Lib/site.py @@ -88,6 +88,11 @@ USER_SITE = None USER_BASE = None +def _trace(message): + if sys.flags.verbose: + print(message, file=sys.stderr) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -100,8 +105,15 @@ def makepath(*paths): def abs_paths(): """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): - if (getattr(getattr(m, '__loader__', None), '__module__', None) not in - ('_frozen_importlib', '_frozen_importlib_external')): + loader_module = None + try: + loader_module = m.__loader__.__module__ + except AttributeError: + try: + loader_module = m.__spec__.loader.__module__ + except AttributeError: + pass + if loader_module not in {'_frozen_importlib', '_frozen_importlib_external'}: continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) @@ -156,14 +168,19 @@ def addpackage(sitedir, name, known_paths): else: reset = False fullname = os.path.join(sitedir, name) + _trace(f"Processing .pth file: {fullname!r}") try: - f = io.TextIOWrapper(io.open_code(fullname)) + # locale encoding is not ideal especially on Windows. But we have used + # it for a long time. setuptools uses the locale encoding too. + f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") except OSError: return with f: for n, line in enumerate(f): if line.startswith("#"): continue + if line.strip() == "": + continue try: if line.startswith(("import ", "import\t")): exec(line) @@ -190,6 +207,7 @@ def addpackage(sitedir, name, known_paths): def addsitedir(sitedir, known_paths=None): """Add 'sitedir' argument to sys.path if missing and handle .pth files in 'sitedir'""" + _trace(f"Adding directory: {sitedir!r}") if known_paths is None: known_paths = _init_pathinfo() reset = True @@ -248,6 +266,10 @@ def _getuserbase(): if env_base: return env_base + # VxWorks has no home directories + if sys.platform == "vxworks": + return None + def joinuser(*args): return os.path.expanduser(os.path.join(*args)) @@ -267,7 +289,8 @@ def _get_path(userbase): version = sys.version_info if os.name == 'nt': - return f'{userbase}\\Python{version[0]}{version[1]}\\site-packages' + ver_nodot = sys.winver.replace('.', '') + return f'{userbase}\\Python{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/python/site-packages' @@ -294,11 +317,14 @@ def getusersitepackages(): If the global variable ``USER_SITE`` is not initialized yet, this function will also set it. """ - global USER_SITE + global USER_SITE, ENABLE_USER_SITE userbase = getuserbase() # this will also set USER_BASE if USER_SITE is None: - USER_SITE = _get_path(userbase) + if userbase is None: + ENABLE_USER_SITE = False # disable user site and return None + else: + USER_SITE = _get_path(userbase) return USER_SITE @@ -310,6 +336,7 @@ def addusersitepackages(known_paths): """ # get the per user site-package path # this call will also make sure USER_BASE and USER_SITE are set + _trace("Processing user site-packages") user_site = getusersitepackages() if ENABLE_USER_SITE and os.path.isdir(user_site): @@ -354,6 +381,7 @@ def getsitepackages(prefixes=None): def addsitepackages(known_paths, prefixes=None): """Add site-packages to sys.path""" + _trace("Processing global site-packages") for sitedir in getsitepackages(prefixes): if os.path.isdir(sitedir): addsitedir(sitedir, known_paths) @@ -611,11 +639,14 @@ def _script(): for dir in sys.path: print(" %r," % (dir,)) print("]") - print("USER_BASE: %r (%s)" % (user_base, - "exists" if os.path.isdir(user_base) else "doesn't exist")) - print("USER_SITE: %r (%s)" % (user_site, - "exists" if os.path.isdir(user_site) else "doesn't exist")) - print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE) + def exists(path): + if path is not None and os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + print(f"USER_BASE: {user_base!r} ({exists(user_base)})") + print(f"USER_SITE: {user_site!r} ({exists(user_site)})") + print(f"ENABLE_USER_SITE: {ENABLE_USER_SITE!r}") sys.exit(0) buffer = [] diff --git a/contrib/tools/python3/src/Lib/smtpd.py b/contrib/tools/python3/src/Lib/smtpd.py index 8f1a22e9378..963e0a7689c 100755 --- a/contrib/tools/python3/src/Lib/smtpd.py +++ b/contrib/tools/python3/src/Lib/smtpd.py @@ -83,8 +83,6 @@ import errno import getopt import time import socket -import asyncore -import asynchat import collections from warnings import warn from email._header_value_parser import get_addr_spec, get_angle_addr @@ -94,6 +92,20 @@ __all__ = [ "MailmanProxy", ] +warn( + 'The smtpd module is deprecated and unmaintained and will be removed ' + 'in Python 3.12. Please see aiosmtpd ' + '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.', + DeprecationWarning, + stacklevel=2) + + +# These are imported after the above warning so that users get the correct +# deprecation warning. +import asyncore +import asynchat + + program = sys.argv[0] __version__ = 'Python SMTP proxy version 0.3' @@ -163,7 +175,7 @@ class SMTPChannel(asynchat.async_chat): # a race condition may occur if the other end is closing # before we can get the peername self.close() - if err.args[0] != errno.ENOTCONN: + if err.errno != errno.ENOTCONN: raise return print('Peer:', repr(self.peer), file=DEBUGSTREAM) diff --git a/contrib/tools/python3/src/Lib/smtplib.py b/contrib/tools/python3/src/Lib/smtplib.py index b1fd45a003d..324a1c19f12 100755 --- a/contrib/tools/python3/src/Lib/smtplib.py +++ b/contrib/tools/python3/src/Lib/smtplib.py @@ -168,7 +168,7 @@ def quotedata(data): """Quote data for email. Double leading '.', and change Unix newline '\\n', or Mac '\\r' into - Internet CRLF end-of-line. + internet CRLF end-of-line. """ return re.sub(r'(?m)^\.', '..', re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) @@ -223,7 +223,7 @@ class SMTP: helo_resp = None ehlo_msg = "ehlo" ehlo_resp = None - does_esmtp = 0 + does_esmtp = False default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, @@ -459,7 +459,7 @@ class SMTP: self.ehlo_resp = msg if code != 250: return (code, msg) - self.does_esmtp = 1 + self.does_esmtp = True #parse the ehlo response -ddm assert isinstance(self.ehlo_resp, bytes), repr(self.ehlo_resp) resp = self.ehlo_resp.decode("latin-1").split('\n') @@ -797,7 +797,7 @@ class SMTP: self.helo_resp = None self.ehlo_resp = None self.esmtp_features = {} - self.does_esmtp = 0 + self.does_esmtp = False else: # RFC 3207: # 501 Syntax error (no parameters allowed) diff --git a/contrib/tools/python3/src/Lib/socket.py b/contrib/tools/python3/src/Lib/socket.py index 46fc49ca323..63ba0acc908 100755 --- a/contrib/tools/python3/src/Lib/socket.py +++ b/contrib/tools/python3/src/Lib/socket.py @@ -337,6 +337,7 @@ class socket(_socket.socket): buffer = io.BufferedWriter(raw, buffering) if binary: return buffer + encoding = io.text_encoding(encoding) text = io.TextIOWrapper(buffer, encoding, errors, newline) text.mode = mode return text @@ -377,7 +378,7 @@ class socket(_socket.socket): try: while True: if timeout and not selector_select(timeout): - raise _socket.timeout('timed out') + raise TimeoutError('timed out') if count: blocksize = count - total_sent if blocksize <= 0: @@ -706,7 +707,7 @@ class SocketIO(io.RawIOBase): self._timeout_occurred = True raise except error as e: - if e.args[0] in _blocking_errnos: + if e.errno in _blocking_errnos: return None raise @@ -722,7 +723,7 @@ class SocketIO(io.RawIOBase): return self._sock.send(b) except error as e: # XXX what about EINTR? - if e.args[0] in _blocking_errnos: + if e.errno in _blocking_errnos: return None raise diff --git a/contrib/tools/python3/src/Lib/sqlite3/__init__.py b/contrib/tools/python3/src/Lib/sqlite3/__init__.py index 1e717450c29..0dedf186b1a 100644 --- a/contrib/tools/python3/src/Lib/sqlite3/__init__.py +++ b/contrib/tools/python3/src/Lib/sqlite3/__init__.py @@ -55,3 +55,17 @@ The sqlite3 module is written by Gerhard Häring . """ from sqlite3.dbapi2 import * + + +# bpo-42264: OptimizedUnicode was deprecated in Python 3.10. It's scheduled +# for removal in Python 3.12. +def __getattr__(name): + if name == "OptimizedUnicode": + import warnings + msg = (""" + OptimizedUnicode is deprecated and will be removed in Python 3.12. + Since Python 3.3 it has simply been an alias for 'str'. + """) + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return str + raise AttributeError(f"module 'sqlite3' has no attribute '{name}'") diff --git a/contrib/tools/python3/src/Lib/sqlite3/dbapi2.py b/contrib/tools/python3/src/Lib/sqlite3/dbapi2.py index 991682ce9ef..cfe6225f46e 100644 --- a/contrib/tools/python3/src/Lib/sqlite3/dbapi2.py +++ b/contrib/tools/python3/src/Lib/sqlite3/dbapi2.py @@ -84,6 +84,20 @@ def register_adapters_and_converters(): register_adapters_and_converters() +# bpo-24464: enable_shared_cache was deprecated in Python 3.10. It's +# scheduled for removal in Python 3.12. +def enable_shared_cache(enable): + from _sqlite3 import enable_shared_cache as _old_enable_shared_cache + import warnings + msg = ( + "enable_shared_cache is deprecated and will be removed in Python 3.12. " + "Shared cache is strongly discouraged by the SQLite 3 documentation. " + "If shared cache must be used, open the database in URI mode using" + "the cache=shared query parameter." + ) + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return _old_enable_shared_cache(enable) + # Clean up namespace del(register_adapters_and_converters) diff --git a/contrib/tools/python3/src/Lib/ssl.py b/contrib/tools/python3/src/Lib/ssl.py index e95e4cf5e98..26f7bd0a74b 100644 --- a/contrib/tools/python3/src/Lib/ssl.py +++ b/contrib/tools/python3/src/Lib/ssl.py @@ -253,7 +253,7 @@ if sys.platform == "win32": from _ssl import enum_certificates, enum_crls from socket import socket, SOCK_STREAM, create_connection -from socket import SOL_SOCKET, SO_TYPE +from socket import SOL_SOCKET, SO_TYPE, _GLOBAL_DEFAULT_TIMEOUT import socket as _socket import base64 # for DER-to-PEM translation import errno @@ -381,6 +381,11 @@ def match_hostname(cert, hostname): CertificateError is raised on failure. On success, the function returns nothing. """ + warnings.warn( + "ssl.match_hostname() is deprecated", + category=DeprecationWarning, + stacklevel=2 + ) if not cert: raise ValueError("empty or no certificate, match_hostname needs a " "SSL socket or SSL context with either " @@ -493,7 +498,14 @@ class SSLContext(_SSLContext): sslsocket_class = None # SSLSocket is assigned later. sslobject_class = None # SSLObject is assigned later. - def __new__(cls, protocol=PROTOCOL_TLS, *args, **kwargs): + def __new__(cls, protocol=None, *args, **kwargs): + if protocol is None: + warnings.warn( + "ssl.SSLContext() without protocol argument is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + protocol = PROTOCOL_TLS self = _SSLContext.__new__(cls, protocol) return self @@ -532,6 +544,11 @@ class SSLContext(_SSLContext): ) def set_npn_protocols(self, npn_protocols): + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) protos = bytearray() for protocol in npn_protocols: b = bytes(protocol, 'ascii') @@ -752,12 +769,15 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(PROTOCOL_TLS) - if purpose == Purpose.SERVER_AUTH: # verify certs and host name in client mode + context = SSLContext(PROTOCOL_TLS_CLIENT) context.verify_mode = CERT_REQUIRED context.check_hostname = True + elif purpose == Purpose.CLIENT_AUTH: + context = SSLContext(PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) @@ -773,7 +793,7 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, context.keylog_filename = keylogfile return context -def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, +def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, check_hostname=False, purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): @@ -790,10 +810,18 @@ def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(protocol) + if purpose == Purpose.SERVER_AUTH: + # verify certs and host name in client mode + if protocol is None: + protocol = PROTOCOL_TLS_CLIENT + elif purpose == Purpose.CLIENT_AUTH: + if protocol is None: + protocol = PROTOCOL_TLS_SERVER + else: + raise ValueError(purpose) - if not check_hostname: - context.check_hostname = False + context = SSLContext(protocol) + context.check_hostname = check_hostname if cert_reqs is not None: context.verify_mode = cert_reqs if check_hostname: @@ -927,15 +955,17 @@ class SSLObject: """Return the currently selected NPN protocol as a string, or ``None`` if a next protocol was not negotiated or if NPN is not supported by one of the peers.""" - if _ssl.HAS_NPN: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) def selected_alpn_protocol(self): """Return the currently selected ALPN protocol as a string, or ``None`` if a next protocol was not negotiated or if ALPN is not supported by one of the peers.""" - if _ssl.HAS_ALPN: - return self._sslobj.selected_alpn_protocol() + return self._sslobj.selected_alpn_protocol() def cipher(self): """Return the currently selected cipher as a 3-tuple ``(name, @@ -1144,10 +1174,12 @@ class SSLSocket(socket): @_sslcopydoc def selected_npn_protocol(self): self._checkClosed() - if self._sslobj is None or not _ssl.HAS_NPN: - return None - else: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) + return None @_sslcopydoc def selected_alpn_protocol(self): @@ -1406,7 +1438,11 @@ def wrap_socket(sock, keyfile=None, certfile=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None): - + warnings.warn( + "ssl.wrap_socket() is deprecated, use SSLContext.wrap_socket()", + category=DeprecationWarning, + stacklevel=2 + ) if server_side and not certfile: raise ValueError("certfile must be specified for server-side " "operations") @@ -1484,11 +1520,14 @@ def PEM_cert_to_DER_cert(pem_cert_string): d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)] return base64.decodebytes(d.encode('ASCII', 'strict')) -def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): +def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, + ca_certs=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Retrieve the certificate from the server at the specified address, and return it as a PEM-encoded string. If 'ca_certs' is specified, validate the server cert against it. - If 'ssl_version' is specified, use it in the connection attempt.""" + If 'ssl_version' is specified, use it in the connection attempt. + If 'timeout' is specified, use it in the connection attempt. + """ host, port = addr if ca_certs is not None: @@ -1498,8 +1537,8 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): context = _create_stdlib_context(ssl_version, cert_reqs=cert_reqs, cafile=ca_certs) - with create_connection(addr) as sock: - with context.wrap_socket(sock) as sslsock: + with create_connection(addr, timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as sslsock: dercert = sslsock.getpeercert(True) return DER_cert_to_PEM_cert(dercert) diff --git a/contrib/tools/python3/src/Lib/statistics.py b/contrib/tools/python3/src/Lib/statistics.py index 463ac9e92c6..f66245380ab 100644 --- a/contrib/tools/python3/src/Lib/statistics.py +++ b/contrib/tools/python3/src/Lib/statistics.py @@ -73,6 +73,30 @@ second argument to the four "spread" functions to avoid recalculating it: 2.5 +Statistics for relations between two inputs +------------------------------------------- + +================== ==================================================== +Function Description +================== ==================================================== +covariance Sample covariance for two variables. +correlation Pearson's correlation coefficient for two variables. +linear_regression Intercept and slope for simple linear regression. +================== ==================================================== + +Calculate covariance, Pearson's correlation, and simple linear regression +for two inputs: + +>>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] +>>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3] +>>> covariance(x, y) +0.75 +>>> correlation(x, y) #doctest: +ELLIPSIS +0.31622776601... +>>> linear_regression(x, y) #doctest: +LinearRegression(slope=0.1, intercept=1.5) + + Exceptions ---------- @@ -83,9 +107,12 @@ A single exception is defined: StatisticsError is a subclass of ValueError. __all__ = [ 'NormalDist', 'StatisticsError', + 'correlation', + 'covariance', 'fmean', 'geometric_mean', 'harmonic_mean', + 'linear_regression', 'mean', 'median', 'median_grouped', @@ -106,11 +133,11 @@ import random from fractions import Fraction from decimal import Decimal -from itertools import groupby +from itertools import groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum from operator import itemgetter -from collections import Counter +from collections import Counter, namedtuple # === Exceptions === @@ -120,21 +147,17 @@ class StatisticsError(ValueError): # === Private utilities === -def _sum(data, start=0): - """_sum(data [, start]) -> (type, sum, count) +def _sum(data): + """_sum(data) -> (type, sum, count) Return a high-precision sum of the given numeric data as a fraction, together with the type to be converted to and the count of items. - If optional argument ``start`` is given, it is added to the total. - If ``data`` is empty, ``start`` (defaulting to 0) is returned. - - Examples -------- - >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75) - (, Fraction(11, 1), 5) + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) Some sources of round-off error will be avoided: @@ -157,10 +180,9 @@ def _sum(data, start=0): allowed. """ count = 0 - n, d = _exact_ratio(start) - partials = {d: n} + partials = {} partials_get = partials.get - T = _coerce(int, type(start)) + T = int for typ, values in groupby(data, type): T = _coerce(T, typ) # or raise TypeError for n, d in map(_exact_ratio, values): @@ -173,8 +195,7 @@ def _sum(data, start=0): assert not _isfinite(total) else: # Sum all the partial sums using builtin sum. - # FIXME is this faster if we sum them in order of the denominator? - total = sum(Fraction(n, d) for d, n in sorted(partials.items())) + total = sum(Fraction(n, d) for d, n in partials.items()) return (T, total, count) @@ -225,27 +246,19 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ try: - # Optimise the common case of floats. We expect that the most often - # used numeric type will be builtin floats, so try to make this as - # fast as possible. - if type(x) is float or type(x) is Decimal: - return x.as_integer_ratio() - try: - # x may be an int, Fraction, or Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - try: - # x may be a float or Decimal subclass. - return x.as_integer_ratio() - except AttributeError: - # Just give up? - pass + return x.as_integer_ratio() + except AttributeError: + pass except (OverflowError, ValueError): # float NAN or INF. assert not _isfinite(x) return (x, None) - msg = "can't convert type '{}' to numerator/denominator" - raise TypeError(msg.format(type(x).__name__)) + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) def _convert(value, T): @@ -364,37 +377,36 @@ def geometric_mean(data): 'containing positive numbers') from None -def harmonic_mean(data): +def harmonic_mean(data, weights=None): """Return the harmonic mean of data. - The harmonic mean, sometimes called the subcontrary mean, is the - reciprocal of the arithmetic mean of the reciprocals of the data, - and is often appropriate when averaging quantities which are rates - or ratios, for example speeds. Example: + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - Suppose an investor purchases an equal value of shares in each of - three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. - What is the average P/E ratio for the investor's portfolio? + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. - 3.6 + >>> harmonic_mean([40, 60]) + 48.0 - Using the arithmetic mean would give an average of about 5.167, which - is too high. + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? + + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 If ``data`` is empty, or any element is less than zero, ``harmonic_mean`` will raise ``StatisticsError``. """ - # For a justification for using harmonic mean for P/E ratios, see - # http://fixthepitch.pellucid.com/comps-analysis-the-missing-harmony-of-summary-statistics/ - # http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2621087 if iter(data) is data: data = list(data) errmsg = 'harmonic mean does not support negative values' n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') - elif n == 1: + elif n == 1 and weights is None: x = data[0] if isinstance(x, (numbers.Real, Decimal)): if x < 0: @@ -402,13 +414,23 @@ def harmonic_mean(data): return x else: raise TypeError('unsupported type') + if weights is None: + weights = repeat(1, n) + sum_weights = n + else: + if iter(weights) is weights: + weights = list(weights) + if len(weights) != n: + raise StatisticsError('Number of weights does not match data size') + _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) try: - T, total, count = _sum(1 / x for x in _fail_neg(data, errmsg)) + data = _fail_neg(data, errmsg) + T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 - assert count == n - return _convert(n / total, T) - + if total <= 0: + raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) # FIXME: investigate ways to calculate medians without sorting? Quickselect? def median(data): @@ -683,14 +705,20 @@ def _ss(data, c=None): if c is not None: T, total, count = _sum((x-c)**2 for x in data) return (T, total) - c = mean(data) - T, total, count = _sum((x-c)**2 for x in data) - # The following sum should mathematically equal zero, but due to rounding - # error may not. - U, total2, count2 = _sum((x - c) for x in data) - assert T == U and count == count2 - total -= total2 ** 2 / len(data) - assert not total < 0, 'negative sum of square deviations: %f' % total + T, total, count = _sum(data) + mean_n, mean_d = (total / count).as_integer_ratio() + partials = Counter() + for n, d in map(_exact_ratio, data): + diff_n = n * mean_d - d * mean_n + diff_d = d * mean_d + partials[diff_d * diff_d] += diff_n * diff_n + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + total = sum(Fraction(n, d) for d, n in partials.items()) return (T, total) @@ -794,6 +822,9 @@ def stdev(data, xbar=None): 1.0810874155219827 """ + # Fixme: Despite the exact sum of squared deviations, some inaccuracy + # remain because there are two rounding steps. The first occurs in + # the _convert() step for variance(), the second occurs in math.sqrt(). var = variance(data, xbar) try: return var.sqrt() @@ -810,6 +841,9 @@ def pstdev(data, mu=None): 0.986893273527251 """ + # Fixme: Despite the exact sum of squared deviations, some inaccuracy + # remain because there are two rounding steps. The first occurs in + # the _convert() step for pvariance(), the second occurs in math.sqrt(). var = pvariance(data, mu) try: return var.sqrt() @@ -817,6 +851,119 @@ def pstdev(data, mu=None): return math.sqrt(var) +# === Statistics for relations between two inputs === + +# See https://en.wikipedia.org/wiki/Covariance +# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient +# https://en.wikipedia.org/wiki/Simple_linear_regression + + +def covariance(x, y, /): + """Covariance + + Return the sample covariance of two inputs *x* and *y*. Covariance + is a measure of the joint variability of two inputs. + + >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3] + >>> covariance(x, y) + 0.75 + >>> z = [9, 8, 7, 6, 5, 4, 3, 2, 1] + >>> covariance(x, z) + -7.5 + >>> covariance(z, x) + -7.5 + + """ + n = len(x) + if len(y) != n: + raise StatisticsError('covariance requires that both inputs have same number of data points') + if n < 2: + raise StatisticsError('covariance requires at least two data points') + xbar = fsum(x) / n + ybar = fsum(y) / n + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + return sxy / (n - 1) + + +def correlation(x, y, /): + """Pearson's correlation coefficient + + Return the Pearson's correlation coefficient for two inputs. Pearson's + correlation coefficient *r* takes values between -1 and +1. It measures the + strength and direction of the linear relationship, where +1 means very + strong, positive linear relationship, -1 very strong, negative linear + relationship, and 0 no linear relationship. + + >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] + >>> correlation(x, x) + 1.0 + >>> correlation(x, y) + -1.0 + + """ + n = len(x) + if len(y) != n: + raise StatisticsError('correlation requires that both inputs have same number of data points') + if n < 2: + raise StatisticsError('correlation requires at least two data points') + xbar = fsum(x) / n + ybar = fsum(y) / n + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxx = fsum((xi - xbar) ** 2.0 for xi in x) + syy = fsum((yi - ybar) ** 2.0 for yi in y) + try: + return sxy / sqrt(sxx * syy) + except ZeroDivisionError: + raise StatisticsError('at least one of the inputs is constant') + + +LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept')) + + +def linear_regression(x, y, /): + """Slope and intercept for simple linear regression. + + Return the slope and intercept of simple linear regression + parameters estimated using ordinary least squares. Simple linear + regression describes relationship between an independent variable + *x* and a dependent variable *y* in terms of linear function: + + y = slope * x + intercept + noise + + where *slope* and *intercept* are the regression parameters that are + estimated, and noise represents the variability of the data that was + not explained by the linear regression (it is equal to the + difference between predicted and actual values of the dependent + variable). + + The parameters are returned as a named tuple. + + >>> x = [1, 2, 3, 4, 5] + >>> noise = NormalDist().samples(5, seed=42) + >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] + >>> linear_regression(x, y) #doctest: +ELLIPSIS + LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) + + """ + n = len(x) + if len(y) != n: + raise StatisticsError('linear regression requires that both inputs have same number of data points') + if n < 2: + raise StatisticsError('linear regression requires at least two data points') + xbar = fsum(x) / n + ybar = fsum(y) / n + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxx = fsum((xi - xbar) ** 2.0 for xi in x) + try: + slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) + except ZeroDivisionError: + raise StatisticsError('x is constant') + intercept = ybar - slope * xbar + return LinearRegression(slope=slope, intercept=intercept) + + ## Normal Distribution ##################################################### diff --git a/contrib/tools/python3/src/Lib/subprocess.py b/contrib/tools/python3/src/Lib/subprocess.py index 4effc1d8b3f..ccb46a6337b 100644 --- a/contrib/tools/python3/src/Lib/subprocess.py +++ b/contrib/tools/python3/src/Lib/subprocess.py @@ -5,7 +5,6 @@ # Copyright (c) 2003-2005 by Peter Astrand # # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. r"""Subprocesses with accessible I/O streams @@ -55,13 +54,10 @@ from time import monotonic as _time import types try: - import pwd + import fcntl except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None + fcntl = None + __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", @@ -326,7 +322,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'showrefcount', 'utf8', 'oldparser'): + 'showrefcount', 'utf8'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -664,8 +660,9 @@ def _use_posix_spawn(): # os.posix_spawn() is not available return False - if sys.platform == 'darwin': - # posix_spawn() is a syscall on macOS and properly reports errors + if sys.platform in ('darwin', 'sunos5'): + # posix_spawn() is a syscall on both macOS and Solaris, + # and properly reports errors return True # Check libc name and runtime libc version @@ -697,7 +694,7 @@ def _use_posix_spawn(): _USE_POSIX_SPAWN = _use_posix_spawn() -class Popen(object): +class Popen: """ Execute a child program in a new process. For a complete description of the arguments see the Python documentation. @@ -760,7 +757,7 @@ class Popen(object): startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=(), *, user=None, group=None, extra_groups=None, - encoding=None, errors=None, text=None, umask=-1): + encoding=None, errors=None, text=None, umask=-1, pipesize=-1): """Create new Popen instance.""" _cleanup() # Held while anything is calling waitpid before returncode has been @@ -777,6 +774,11 @@ class Popen(object): if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if pipesize is None: + pipesize = -1 # Restore default + if not isinstance(pipesize, int): + raise TypeError("pipesize must be an integer") + if _mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " @@ -801,6 +803,7 @@ class Popen(object): self.returncode = None self.encoding = encoding self.errors = errors + self.pipesize = pipesize # Validate the combinations of text and universal_newlines if (text is not None and universal_newlines is not None @@ -842,6 +845,13 @@ class Popen(object): self.text_mode = encoding or errors or text or universal_newlines + # PEP 597: We suppress the EncodingWarning in subprocess module + # for now (at Python 3.10), because we focus on files for now. + # This will be changed to encoding = io.text_encoding(encoding) + # in the future. + if self.text_mode and encoding is None: + self.encoding = encoding = "locale" + # How long to resume waiting on a child after the first ^C. # There is no right value for this. The purpose is to be polite # yet remain good for interactive users trying to exit a tool. @@ -865,7 +875,9 @@ class Popen(object): "current platform") elif isinstance(group, str): - if grp is None: + try: + import grp + except ImportError: raise ValueError("The group parameter cannot be a string " "on systems without the grp module") @@ -891,7 +903,9 @@ class Popen(object): gids = [] for extra_group in extra_groups: if isinstance(extra_group, str): - if grp is None: + try: + import grp + except ImportError: raise ValueError("Items in extra_groups cannot be " "strings on systems without the " "grp module") @@ -917,10 +931,11 @@ class Popen(object): "the current platform") elif isinstance(user, str): - if pwd is None: + try: + import pwd + except ImportError: raise ValueError("The user parameter cannot be a string " "on systems without the pwd module") - uid = pwd.getpwnam(user).pw_uid elif isinstance(user, int): uid = user @@ -1577,6 +1592,8 @@ class Popen(object): pass elif stdin == PIPE: p2cread, p2cwrite = os.pipe() + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize) elif stdin == DEVNULL: p2cread = self._get_devnull() elif isinstance(stdin, int): @@ -1589,6 +1606,8 @@ class Popen(object): pass elif stdout == PIPE: c2pread, c2pwrite = os.pipe() + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize) elif stdout == DEVNULL: c2pwrite = self._get_devnull() elif isinstance(stdout, int): @@ -1601,6 +1620,8 @@ class Popen(object): pass elif stderr == PIPE: errread, errwrite = os.pipe() + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize) elif stderr == STDOUT: if c2pwrite != -1: errwrite = c2pwrite diff --git a/contrib/tools/python3/src/Lib/symbol.py b/contrib/tools/python3/src/Lib/symbol.py deleted file mode 100644 index aaac8c91443..00000000000 --- a/contrib/tools/python3/src/Lib/symbol.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Non-terminal symbols of Python grammar (from "graminit.h").""" - -# This file is automatically generated; please don't muck it up! -# -# To update the symbols in this file, 'cd' to the top directory of -# the python source tree after building the interpreter and run: -# -# python3 Tools/scripts/generate_symbol_py.py Include/graminit.h Lib/symbol.py -# -# or just -# -# make regen-symbol - -import warnings - -warnings.warn( - "The symbol module is deprecated and will be removed " - "in future versions of Python", - DeprecationWarning, - stacklevel=2, -) - -#--start constants-- -single_input = 256 -file_input = 257 -eval_input = 258 -decorator = 259 -decorators = 260 -decorated = 261 -async_funcdef = 262 -funcdef = 263 -parameters = 264 -typedargslist = 265 -tfpdef = 266 -varargslist = 267 -vfpdef = 268 -stmt = 269 -simple_stmt = 270 -small_stmt = 271 -expr_stmt = 272 -annassign = 273 -testlist_star_expr = 274 -augassign = 275 -del_stmt = 276 -pass_stmt = 277 -flow_stmt = 278 -break_stmt = 279 -continue_stmt = 280 -return_stmt = 281 -yield_stmt = 282 -raise_stmt = 283 -import_stmt = 284 -import_name = 285 -import_from = 286 -import_as_name = 287 -dotted_as_name = 288 -import_as_names = 289 -dotted_as_names = 290 -dotted_name = 291 -global_stmt = 292 -nonlocal_stmt = 293 -assert_stmt = 294 -compound_stmt = 295 -async_stmt = 296 -if_stmt = 297 -while_stmt = 298 -for_stmt = 299 -try_stmt = 300 -with_stmt = 301 -with_item = 302 -except_clause = 303 -suite = 304 -namedexpr_test = 305 -test = 306 -test_nocond = 307 -lambdef = 308 -lambdef_nocond = 309 -or_test = 310 -and_test = 311 -not_test = 312 -comparison = 313 -comp_op = 314 -star_expr = 315 -expr = 316 -xor_expr = 317 -and_expr = 318 -shift_expr = 319 -arith_expr = 320 -term = 321 -factor = 322 -power = 323 -atom_expr = 324 -atom = 325 -testlist_comp = 326 -trailer = 327 -subscriptlist = 328 -subscript = 329 -sliceop = 330 -exprlist = 331 -testlist = 332 -dictorsetmaker = 333 -classdef = 334 -arglist = 335 -argument = 336 -comp_iter = 337 -sync_comp_for = 338 -comp_for = 339 -comp_if = 340 -encoding_decl = 341 -yield_expr = 342 -yield_arg = 343 -func_body_suite = 344 -func_type_input = 345 -func_type = 346 -typelist = 347 -#--end constants-- - -sym_name = {} -for _name, _value in list(globals().items()): - if type(_value) is type(0): - sym_name[_value] = _name -del _name, _value diff --git a/contrib/tools/python3/src/Lib/symtable.py b/contrib/tools/python3/src/Lib/symtable.py index 521540fe9ee..98db1e2557d 100644 --- a/contrib/tools/python3/src/Lib/symtable.py +++ b/contrib/tools/python3/src/Lib/symtable.py @@ -10,6 +10,11 @@ import weakref __all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] def symtable(code, filename, compile_type): + """ Return the toplevel *SymbolTable* for the source code. + + *filename* is the name of the file with the code + and *compile_type* is the *compile()* mode argument. + """ top = _symtable.symtable(code, filename, compile_type) return _newSymbolTable(top, filename) @@ -55,6 +60,11 @@ class SymbolTable: self._filename) def get_type(self): + """Return the type of the symbol table. + + The values retuned are 'class', 'module' and + 'function'. + """ if self._table.type == _symtable.TYPE_MODULE: return "module" if self._table.type == _symtable.TYPE_FUNCTION: @@ -65,27 +75,51 @@ class SymbolTable: "unexpected type: {0}".format(self._table.type) def get_id(self): + """Return an identifier for the table. + """ return self._table.id def get_name(self): + """Return the table's name. + + This corresponds to the name of the class, function + or 'top' if the table is for a class, function or + global respectively. + """ return self._table.name def get_lineno(self): + """Return the number of the first line in the + block for the table. + """ return self._table.lineno def is_optimized(self): + """Return *True* if the locals in the table + are optimizable. + """ return bool(self._table.type == _symtable.TYPE_FUNCTION) def is_nested(self): + """Return *True* if the block is a nested class + or function.""" return bool(self._table.nested) def has_children(self): + """Return *True* if the block has nested namespaces. + """ return bool(self._table.children) def get_identifiers(self): + """Return a list of names of symbols in the table. + """ return self._table.symbols.keys() def lookup(self, name): + """Lookup a *name* in the table. + + Returns a *Symbol* instance. + """ sym = self._symbols.get(name) if sym is None: flags = self._table.symbols[name] @@ -96,6 +130,9 @@ class SymbolTable: return sym def get_symbols(self): + """Return a list of *Symbol* instances for + names in the table. + """ return [self.lookup(ident) for ident in self.get_identifiers()] def __check_children(self, name): @@ -104,6 +141,8 @@ class SymbolTable: if st.name == name] def get_children(self): + """Return a list of the nested symbol tables. + """ return [_newSymbolTable(st, self._filename) for st in self._table.children] @@ -122,11 +161,15 @@ class Function(SymbolTable): if test_func(self._table.symbols[ident])) def get_parameters(self): + """Return a tuple of parameters to the function. + """ if self.__params is None: self.__params = self.__idents_matching(lambda x:x & DEF_PARAM) return self.__params def get_locals(self): + """Return a tuple of locals in the function. + """ if self.__locals is None: locs = (LOCAL, CELL) test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs @@ -134,6 +177,8 @@ class Function(SymbolTable): return self.__locals def get_globals(self): + """Return a tuple of globals in the function. + """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob @@ -141,11 +186,15 @@ class Function(SymbolTable): return self.__globals def get_nonlocals(self): + """Return a tuple of nonlocals in the function. + """ if self.__nonlocals is None: self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL) return self.__nonlocals def get_frees(self): + """Return a tuple of free variables in the function. + """ if self.__frees is None: is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE self.__frees = self.__idents_matching(is_free) @@ -157,6 +206,8 @@ class Class(SymbolTable): __methods = None def get_methods(self): + """Return a tuple of methods declared in the class. + """ if self.__methods is None: d = {} for st in self._table.children: @@ -178,12 +229,19 @@ class Symbol: return "".format(self.__name) def get_name(self): + """Return a name of a symbol. + """ return self.__name def is_referenced(self): + """Return *True* if the symbol is used in + its block. + """ return bool(self.__flags & _symtable.USE) def is_parameter(self): + """Return *True* if the symbol is a parameter. + """ return bool(self.__flags & DEF_PARAM) def is_global(self): @@ -193,9 +251,12 @@ class Symbol: or (self.__module_scope and self.__flags & DEF_BOUND)) def is_nonlocal(self): + """Return *True* if the symbol is nonlocal.""" return bool(self.__flags & DEF_NONLOCAL) def is_declared_global(self): + """Return *True* if the symbol is declared global + with a global statement.""" return bool(self.__scope == GLOBAL_EXPLICIT) def is_local(self): @@ -205,19 +266,28 @@ class Symbol: or (self.__module_scope and self.__flags & DEF_BOUND)) def is_annotated(self): + """Return *True* if the symbol is annotated. + """ return bool(self.__flags & DEF_ANNOT) def is_free(self): + """Return *True* if a referenced symbol is + not assigned to. + """ return bool(self.__scope == FREE) def is_imported(self): + """Return *True* if the symbol is created from + an import statement. + """ return bool(self.__flags & DEF_IMPORT) def is_assigned(self): + """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) def is_namespace(self): - """Returns true if name binding introduces new namespace. + """Returns *True* if name binding introduces new namespace. If the name is used as the target of a function or class statement, this will be true. @@ -234,7 +304,7 @@ class Symbol: return self.__namespaces def get_namespace(self): - """Returns the single namespace bound to this name. + """Return the single namespace bound to this name. Raises ValueError if the name is bound to multiple namespaces. """ diff --git a/contrib/tools/python3/src/Lib/sysconfig.py b/contrib/tools/python3/src/Lib/sysconfig.py index 55bd06f3a08..8e403a7b00e 100644 --- a/contrib/tools/python3/src/Lib/sysconfig.py +++ b/contrib/tools/python3/src/Lib/sysconfig.py @@ -56,42 +56,73 @@ _INSTALL_SCHEMES = { 'scripts': '{base}/Scripts', 'data': '{base}', }, - # NOTE: When modifying "purelib" scheme, update site._get_path() too. - 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot}', - 'platstdlib': '{userbase}/Python{py_version_nodot}', - 'purelib': '{userbase}/Python{py_version_nodot}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot}/site-packages', - 'include': '{userbase}/Python{py_version_nodot}/Include', - 'scripts': '{userbase}/Python{py_version_nodot}/Scripts', - 'data': '{userbase}', - }, - 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/{platlibdir}/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, + } + + +# NOTE: site.py has copy of this function. +# Sync it when modify this function. +def _getuserbase(): + env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base + + # VxWorks has no home directories + if sys.platform == "vxworks": + return None + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + return joinuser(base, "Python") + + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + f"{sys.version_info[0]}.{sys.version_info[1]}") + + return joinuser("~", ".local") + +_HAS_USER_BASE = (_getuserbase() is not None) + +if _HAS_USER_BASE: + _INSTALL_SCHEMES |= { + # NOTE: When modifying "purelib" scheme, update site._get_path() too. + 'nt_user': { + 'stdlib': '{userbase}/Python{py_version_nodot_plat}', + 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', + 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/Python{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'data': '{userbase}', + }, + 'posix_user': { + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + 'osx_framework_user': { + 'stdlib': '{userbase}/lib/python', + 'platstdlib': '{userbase}/lib/python', + 'purelib': '{userbase}/lib/python/site-packages', + 'platlib': '{userbase}/lib/python/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, } _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', 'scripts', 'data') _PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2] -_PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2] +_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' +_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' _PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) @@ -99,6 +130,12 @@ _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) _CONFIG_VARS = None _USER_BASE = None +# Regexes needed for parsing Makefile (and similar syntaxes, +# like old-style Setup files). +_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" +_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" +_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + def _safe_realpath(path): try: @@ -147,18 +184,24 @@ _PYTHON_BUILD = is_python_build(True) if _PYTHON_BUILD: for scheme in ('posix_prefix', 'posix_home'): - _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include' - _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.' + # On POSIX-y platforms, Python will: + # - Build from .h files in 'headers' (which is only added to the + # scheme when building CPython) + # - Install .h files to 'include' + scheme = _INSTALL_SCHEMES[scheme] + scheme['headers'] = scheme['include'] + scheme['include'] = '{srcdir}/Include' + scheme['platinclude'] = '{projectbase}/.' def _subst_vars(s, local_vars): try: return s.format(**local_vars) - except KeyError: + except KeyError as var: try: return s.format(**os.environ) - except KeyError as var: - raise AttributeError('{%s}' % var) from None + except KeyError: + raise AttributeError(f'{var}') from None def _extend_dict(target_dict, other_dict): target_keys = target_dict.keys() @@ -181,60 +224,62 @@ def _expand_vars(scheme, vars): return res -def _get_default_scheme(): - if os.name == 'posix': - # the default scheme for posix is posix_prefix - return 'posix_prefix' - return os.name - - -# NOTE: site.py has copy of this function. -# Sync it when modify this function. -def _getuserbase(): - env_base = os.environ.get("PYTHONUSERBASE", None) - if env_base: - return env_base +def _get_preferred_schemes(): + if os.name == 'nt': + return { + 'prefix': 'nt', + 'home': 'posix_home', + 'user': 'nt_user', + } + if sys.platform == 'darwin' and sys._framework: + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'osx_framework_user', + } + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'posix_user', + } - def joinuser(*args): - return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": - base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") +def get_preferred_scheme(key): + scheme = _get_preferred_schemes()[key] + if scheme not in _INSTALL_SCHEMES: + raise ValueError( + f"{key!r} returned {scheme!r}, which is not a valid scheme " + f"on this platform" + ) + return scheme - if sys.platform == "darwin" and sys._framework: - return joinuser("~", "Library", sys._framework, - "%d.%d" % sys.version_info[:2]) - return joinuser("~", ".local") +def get_default_scheme(): + return get_preferred_scheme('prefix') -def _parse_makefile(filename, vars=None): +def _parse_makefile(filename, vars=None, keep_unresolved=True): """Parse a Makefile-style file. A dictionary containing name/value pairs is returned. If an optional dictionary is passed in as the second argument, it is used instead of a new dictionary. """ - # Regexes needed for parsing Makefile (and similar syntaxes, - # like old-style Setup files). import re - _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") - _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") - _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") if vars is None: vars = {} done = {} notdone = {} - with open(filename, errors="surrogateescape") as f: + with open(filename, encoding=sys.getfilesystemencoding(), + errors="surrogateescape") as f: lines = f.readlines() for line in lines: if line.startswith('#') or line.strip() == '': continue - m = _variable_rx.match(line) + m = re.match(_variable_rx, line) if m: n, v = m.group(1, 2) v = v.strip() @@ -267,8 +312,8 @@ def _parse_makefile(filename, vars=None): while len(variables) > 0: for name in tuple(variables): value = notdone[name] - m1 = _findvar1_rx.search(value) - m2 = _findvar2_rx.search(value) + m1 = re.search(_findvar1_rx, value) + m2 = re.search(_findvar2_rx, value) if m1 and m2: m = m1 if m1.start() < m2.start() else m2 else: @@ -323,9 +368,12 @@ def _parse_makefile(filename, vars=None): done[name] = value else: + # Adds unresolved variables to the done dict. + # This is disabled when called from distutils.sysconfig + if keep_unresolved: + done[name] = value # bogus variable reference (e.g. "prefix=$/opt/python"); # just drop it since we can't deal - done[name] = value variables.remove(name) # strip spurious spaces @@ -343,21 +391,20 @@ def get_makefile_filename(): if _PYTHON_BUILD: return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") if hasattr(sys, 'abiflags'): - config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) + config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' if hasattr(sys.implementation, '_multiarch'): - config_dir_name += '-%s' % sys.implementation._multiarch + config_dir_name += f'-{sys.implementation._multiarch}' return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') def _get_sysconfigdata_name(): - return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) + multiarch = getattr(sys.implementation, '_multiarch', '') + return os.environ.get( + '_PYTHON_SYSCONFIGDATA_NAME', + f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', + ) def _generate_posix_vars(): @@ -369,19 +416,19 @@ def _generate_posix_vars(): try: _parse_makefile(makefile, vars) except OSError as e: - msg = "invalid Python installation: unable to open %s" % makefile + msg = f"invalid Python installation: unable to open {makefile}" if hasattr(e, "strerror"): - msg = msg + " (%s)" % e.strerror + msg = f"{msg} ({e.strerror})" raise OSError(msg) # load the installed pyconfig.h: config_h = get_config_h_filename() try: - with open(config_h) as f: + with open(config_h, encoding="utf-8") as f: parse_config_h(f, vars) except OSError as e: - msg = "invalid Python installation: unable to open %s" % config_h + msg = f"invalid Python installation: unable to open {config_h}" if hasattr(e, "strerror"): - msg = msg + " (%s)" % e.strerror + msg = f"{msg} ({e.strerror})" raise OSError(msg) # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed @@ -407,7 +454,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) + pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}' if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) @@ -443,6 +490,7 @@ def _init_non_posix(vars): vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) + vars['TZPATH'] = '' # # public APIs @@ -505,7 +553,7 @@ def get_path_names(): return _SCHEME_KEYS -def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): +def get_paths(scheme=get_default_scheme(), vars=None, expand=True): """Return a mapping containing an install scheme. ``scheme`` is the install scheme name. If not provided, it will @@ -517,7 +565,7 @@ def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): return _INSTALL_SCHEMES[scheme] -def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True): +def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): """Return a path corresponding to the scheme. ``scheme`` is the install scheme name. @@ -557,20 +605,24 @@ def get_config_vars(*args): except AttributeError: # sys.abiflags may not be defined on all platforms. _CONFIG_VARS['abiflags'] = '' + try: + _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') + except AttributeError: + _CONFIG_VARS['py_version_nodot_plat'] = '' if os.name == 'nt': _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['TZPATH'] = '' if os.name == 'posix': _init_posix(_CONFIG_VARS) # For backward compatibility, see issue19555 SO = _CONFIG_VARS.get('EXT_SUFFIX') if SO is not None: _CONFIG_VARS['SO'] = SO - # Setting 'userbase' is done below the call to the - # init function to enable using 'get_config_var' in - # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() + if _HAS_USER_BASE: + # Setting 'userbase' is done below the call to the + # init function to enable using 'get_config_var' in + # the init-function. + _CONFIG_VARS['userbase'] = _getuserbase() # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) @@ -667,16 +719,16 @@ def get_platform(): # At least on Linux/Intel, 'machine' is the processor -- # i386, etc. # XXX what about Alpha, SPARC, etc? - return "%s-%s" % (osname, machine) + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" - release = "%d.%s" % (int(release[0]) - 3, release[2:]) + release = f"{int(release[0]) - 3}.{release[2:]}" # We can't use "platform.architecture()[0]" because a # bootstrap problem. We use a dict to get an error # if some suspicious happens. bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += ".%s" % bitness[sys.maxsize] + machine += f".{bitness[sys.maxsize]}" # fall through to standard osname-release-machine representation elif osname[:3] == "aix": from _aix_support import aix_platform @@ -694,18 +746,44 @@ def get_platform(): get_config_vars(), osname, release, machine) - return "%s-%s-%s" % (osname, release, machine) + return f"{osname}-{release}-{machine}" def get_python_version(): return _PY_VERSION_SHORT +def expand_makefile_vars(s, vars): + """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in + 'string' according to 'vars' (a dictionary mapping variable names to + values). Variables not present in 'vars' are silently expanded to the + empty string. The variable values in 'vars' should not contain further + variable expansions; if 'vars' is the output of 'parse_makefile()', + you're fine. Returns a variable-expanded version of 's'. + """ + import re + + # This algorithm does multiple expansion, so if vars['foo'] contains + # "${bar}", it will expand ${foo} to ${bar}, and then expand + # ${bar}... and so forth. This is fine as long as 'vars' comes from + # 'parse_makefile()', which takes care of such expansions eagerly, + # according to make's variable expansion semantics. + + while True: + m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s) + if m: + (beg, end) = m.span() + s = s[0:beg] + vars.get(m.group(1)) + s[end:] + else: + break + return s + + def _print_dict(title, data): for index, (key, value) in enumerate(sorted(data.items())): if index == 0: - print('%s: ' % (title)) - print('\t%s = "%s"' % (key, value)) + print(f'{title}: ') + print(f'\t{key} = "{value}"') def _main(): @@ -713,9 +791,9 @@ def _main(): if '--generate-posix-vars' in sys.argv: _generate_posix_vars() return - print('Platform: "%s"' % get_platform()) - print('Python version: "%s"' % get_python_version()) - print('Current installation scheme: "%s"' % _get_default_scheme()) + print(f'Platform: "{get_platform()}"') + print(f'Python version: "{get_python_version()}"') + print(f'Current installation scheme: "{get_default_scheme()}"') print() _print_dict('Paths', get_paths()) print() diff --git a/contrib/tools/python3/src/Lib/tarfile.py b/contrib/tools/python3/src/Lib/tarfile.py index 9438b08ae39..6ada9a05db9 100755 --- a/contrib/tools/python3/src/Lib/tarfile.py +++ b/contrib/tools/python3/src/Lib/tarfile.py @@ -200,6 +200,7 @@ def itn(n, digits=8, format=DEFAULT_FORMAT): # base-256 representation. This allows values up to (256**(digits-1))-1. # A 0o200 byte indicates a positive number, a 0o377 byte a negative # number. + original_n = n n = int(n) if 0 <= n < 8 ** (digits - 1): s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL @@ -363,7 +364,7 @@ class _Stream: try: import zlib except ImportError: - raise CompressionError("zlib module is not available") + raise CompressionError("zlib module is not available") from None self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": @@ -376,7 +377,7 @@ class _Stream: try: import bz2 except ImportError: - raise CompressionError("bz2 module is not available") + raise CompressionError("bz2 module is not available") from None if mode == "r": self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() @@ -388,7 +389,7 @@ class _Stream: try: import lzma except ImportError: - raise CompressionError("lzma module is not available") + raise CompressionError("lzma module is not available") from None if mode == "r": self.dbuf = b"" self.cmp = lzma.LZMADecompressor() @@ -541,8 +542,8 @@ class _Stream: break try: buf = self.cmp.decompress(buf) - except self.exception: - raise ReadError("invalid compressed data") + except self.exception as e: + raise ReadError("invalid compressed data") from e t.append(buf) c += len(buf) t = b"".join(t) @@ -1173,8 +1174,8 @@ class TarInfo(object): # Fetch the next header and process it. try: next = self.fromtarfile(tarfile) - except HeaderError: - raise SubsequentHeaderError("missing or bad subsequent header") + except HeaderError as e: + raise SubsequentHeaderError(str(e)) from None # Patch the TarInfo object from the next header with # the longname information. @@ -1286,8 +1287,8 @@ class TarInfo(object): # Fetch the next header. try: next = self.fromtarfile(tarfile) - except HeaderError: - raise SubsequentHeaderError("missing or bad subsequent header") + except HeaderError as e: + raise SubsequentHeaderError(str(e)) from None # Process GNU sparse information. if "GNU.sparse.map" in pax_headers: @@ -1542,7 +1543,7 @@ class TarFile(object): self.fileobj.seek(self.offset) break except HeaderError as e: - raise ReadError(str(e)) + raise ReadError(str(e)) from None if self.mode in ("a", "w", "x"): self._loaded = True @@ -1612,17 +1613,20 @@ class TarFile(object): # Find out which *open() is appropriate for opening the file. def not_compressed(comptype): return cls.OPEN_METH[comptype] == 'taropen' + error_msgs = [] for comptype in sorted(cls.OPEN_METH, key=not_compressed): func = getattr(cls, cls.OPEN_METH[comptype]) if fileobj is not None: saved_pos = fileobj.tell() try: return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError): + except (ReadError, CompressionError) as e: + error_msgs.append(f'- method {comptype}: {e!r}') if fileobj is not None: fileobj.seek(saved_pos) continue - raise ReadError("file could not be opened successfully") + error_msgs_summary = '\n'.join(error_msgs) + raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}") elif ":" in mode: filemode, comptype = mode.split(":", 1) @@ -1678,21 +1682,21 @@ class TarFile(object): try: from gzip import GzipFile except ImportError: - raise CompressionError("gzip module is not available") + raise CompressionError("gzip module is not available") from None try: fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) - except OSError: + except OSError as e: if fileobj is not None and mode == 'r': - raise ReadError("not a gzip file") + raise ReadError("not a gzip file") from e raise try: t = cls.taropen(name, mode, fileobj, **kwargs) - except OSError: + except OSError as e: fileobj.close() if mode == 'r': - raise ReadError("not a gzip file") + raise ReadError("not a gzip file") from e raise except: fileobj.close() @@ -1711,16 +1715,16 @@ class TarFile(object): try: from bz2 import BZ2File except ImportError: - raise CompressionError("bz2 module is not available") + raise CompressionError("bz2 module is not available") from None fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) try: t = cls.taropen(name, mode, fileobj, **kwargs) - except (OSError, EOFError): + except (OSError, EOFError) as e: fileobj.close() if mode == 'r': - raise ReadError("not a bzip2 file") + raise ReadError("not a bzip2 file") from e raise except: fileobj.close() @@ -1739,16 +1743,16 @@ class TarFile(object): try: from lzma import LZMAFile, LZMAError except ImportError: - raise CompressionError("lzma module is not available") + raise CompressionError("lzma module is not available") from None fileobj = LZMAFile(fileobj or name, mode, preset=preset) try: t = cls.taropen(name, mode, fileobj, **kwargs) - except (LZMAError, EOFError): + except (LZMAError, EOFError) as e: fileobj.close() if mode == 'r': - raise ReadError("not an lzma file") + raise ReadError("not an lzma file") from e raise except: fileobj.close() @@ -2262,7 +2266,7 @@ class TarFile(object): self._extract_member(self._find_link_target(tarinfo), targetpath) except KeyError: - raise ExtractError("unable to resolve link inside archive") + raise ExtractError("unable to resolve link inside archive") from None def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner @@ -2290,16 +2294,16 @@ class TarFile(object): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) - except OSError: - raise ExtractError("could not change owner") + except OSError as e: + raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): """Set file permissions of targetpath according to tarinfo. """ try: os.chmod(targetpath, tarinfo.mode) - except OSError: - raise ExtractError("could not change mode") + except OSError as e: + raise ExtractError("could not change mode") from e def utime(self, tarinfo, targetpath): """Set modification time of targetpath according to tarinfo. @@ -2308,8 +2312,8 @@ class TarFile(object): return try: os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) - except OSError: - raise ExtractError("could not change modification time") + except OSError as e: + raise ExtractError("could not change modification time") from e #-------------------------------------------------------------------------- def next(self): @@ -2345,20 +2349,20 @@ class TarFile(object): self.offset += BLOCKSIZE continue elif self.offset == 0: - raise ReadError(str(e)) + raise ReadError(str(e)) from None except EmptyHeaderError: if self.offset == 0: - raise ReadError("empty file") + raise ReadError("empty file") from None except TruncatedHeaderError as e: if self.offset == 0: - raise ReadError(str(e)) + raise ReadError(str(e)) from None except SubsequentHeaderError as e: - raise ReadError(str(e)) + raise ReadError(str(e)) from None except Exception as e: try: import zlib if isinstance(e, zlib.error): - raise ReadError(f'zlib error: {e}') + raise ReadError(f'zlib error: {e}') from None else: raise e except ImportError: diff --git a/contrib/tools/python3/src/Lib/tempfile.py b/contrib/tools/python3/src/Lib/tempfile.py index eafce6f25b6..7b6821240f2 100644 --- a/contrib/tools/python3/src/Lib/tempfile.py +++ b/contrib/tools/python3/src/Lib/tempfile.py @@ -103,7 +103,11 @@ def _infer_return_type(*args): "path components.") return_type = str if return_type is None: - return str # tempfile APIs return a str by default. + if tempdir is None or isinstance(tempdir, str): + return str # tempfile APIs return a str by default. + else: + # we could check for bytes but it'll fail later on anyway + return bytes return return_type @@ -147,10 +151,7 @@ class _RandomNameSequence: return self def __next__(self): - c = self.characters - choose = self.rng.choice - letters = [choose(c) for dummy in range(8)] - return ''.join(letters) + return ''.join(self.rng.choices(self.characters, k=8)) def _candidate_tempdir_list(): """Generate a list of candidate temporary directories which @@ -272,17 +273,17 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): # User visible interfaces. def gettempprefix(): - """The default prefix for temporary directories.""" - return template + """The default prefix for temporary directories as string.""" + return _os.fsdecode(template) def gettempprefixb(): """The default prefix for temporary directories as bytes.""" - return _os.fsencode(gettempprefix()) + return _os.fsencode(template) tempdir = None -def gettempdir(): - """Accessor for tempfile.tempdir.""" +def _gettempdir(): + """Private accessor for tempfile.tempdir.""" global tempdir if tempdir is None: _once_lock.acquire() @@ -293,9 +294,13 @@ def gettempdir(): _once_lock.release() return tempdir +def gettempdir(): + """Returns tempfile.tempdir as str.""" + return _os.fsdecode(_gettempdir()) + def gettempdirb(): - """A bytes version of tempfile.gettempdir().""" - return _os.fsencode(gettempdir()) + """Returns tempfile.tempdir as bytes.""" + return _os.fsencode(_gettempdir()) def mkstemp(suffix=None, prefix=None, dir=None, text=False): """User-callable function to create and return a unique temporary @@ -542,6 +547,9 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, if _os.name == 'nt' and delete: flags |= _os.O_TEMPORARY + if "b" not in mode: + encoding = _io.text_encoding(encoding) + (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: file = _io.open(fd, mode, buffering=buffering, @@ -582,6 +590,9 @@ else: """ global _O_TMPFILE_WORKS + if "b" not in mode: + encoding = _io.text_encoding(encoding) + prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags @@ -637,6 +648,7 @@ class SpooledTemporaryFile: if 'b' in mode: self._file = _io.BytesIO() else: + encoding = _io.text_encoding(encoding) self._file = _io.TextIOWrapper(_io.BytesIO(), encoding=encoding, errors=errors, newline=newline) @@ -767,7 +779,7 @@ class SpooledTemporaryFile: return rv -class TemporaryDirectory(object): +class TemporaryDirectory: """Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -779,14 +791,17 @@ class TemporaryDirectory(object): in it are removed. """ - def __init__(self, suffix=None, prefix=None, dir=None): + def __init__(self, suffix=None, prefix=None, dir=None, + ignore_cleanup_errors=False): self.name = mkdtemp(suffix, prefix, dir) + self._ignore_cleanup_errors = ignore_cleanup_errors self._finalizer = _weakref.finalize( self, self._cleanup, self.name, - warn_message="Implicitly cleaning up {!r}".format(self)) + warn_message="Implicitly cleaning up {!r}".format(self), + ignore_errors=self._ignore_cleanup_errors) @classmethod - def _rmtree(cls, name): + def _rmtree(cls, name, ignore_errors=False): def onerror(func, path, exc_info): if issubclass(exc_info[0], PermissionError): def resetperms(path): @@ -805,19 +820,20 @@ class TemporaryDirectory(object): _os.unlink(path) # PermissionError is raised on FreeBSD for directories except (IsADirectoryError, PermissionError): - cls._rmtree(path) + cls._rmtree(path, ignore_errors=ignore_errors) except FileNotFoundError: pass elif issubclass(exc_info[0], FileNotFoundError): pass else: - raise + if not ignore_errors: + raise _shutil.rmtree(name, onerror=onerror) @classmethod - def _cleanup(cls, name, warn_message): - cls._rmtree(name) + def _cleanup(cls, name, warn_message, ignore_errors=False): + cls._rmtree(name, ignore_errors=ignore_errors) _warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -830,7 +846,7 @@ class TemporaryDirectory(object): self.cleanup() def cleanup(self): - if self._finalizer.detach(): - self._rmtree(self.name) + if self._finalizer.detach() or _os.path.exists(self.name): + self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors) __class_getitem__ = classmethod(_types.GenericAlias) diff --git a/contrib/tools/python3/src/Lib/textwrap.py b/contrib/tools/python3/src/Lib/textwrap.py index 30e693c8de0..841de9baecf 100644 --- a/contrib/tools/python3/src/Lib/textwrap.py +++ b/contrib/tools/python3/src/Lib/textwrap.py @@ -215,8 +215,16 @@ class TextWrapper: # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. if self.break_long_words: - cur_line.append(reversed_chunks[-1][:space_left]) - reversed_chunks[-1] = reversed_chunks[-1][space_left:] + end = space_left + chunk = reversed_chunks[-1] + if self.break_on_hyphens and len(chunk) > space_left: + # break after last hyphen, but only if there are + # non-hyphens before it + hyphen = chunk.rfind('-', 0, space_left) + if hyphen > 0 and any(c != '-' for c in chunk[:hyphen]): + end = hyphen + 1 + cur_line.append(chunk[:end]) + reversed_chunks[-1] = chunk[end:] # Otherwise, we have to preserve the long word intact. Only add # it to the current line if there's nothing already there -- diff --git a/contrib/tools/python3/src/Lib/threading.py b/contrib/tools/python3/src/Lib/threading.py index a3cb245ab96..2d897429136 100644 --- a/contrib/tools/python3/src/Lib/threading.py +++ b/contrib/tools/python3/src/Lib/threading.py @@ -28,7 +28,7 @@ __all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size', - 'excepthook', 'ExceptHookArgs'] + 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread @@ -65,6 +65,10 @@ def setprofile(func): global _profile_hook _profile_hook = func +def getprofile(): + """Get the profiler function as set by threading.setprofile().""" + return _profile_hook + def settrace(func): """Set a trace function for all threads started from the threading module. @@ -75,6 +79,10 @@ def settrace(func): global _trace_hook _trace_hook = func +def gettrace(): + """Get the trace function as set by threading.settrace().""" + return _trace_hook + # Synchronization classes Lock = _allocate_lock @@ -380,7 +388,16 @@ class Condition: """ self.notify(len(self._waiters)) - notifyAll = notify_all + def notifyAll(self): + """Wake up all threads waiting on this condition. + + This method is deprecated, use notify_all() instead. + + """ + import warnings + warnings.warn('notifyAll() is deprecated, use notify_all() instead', + DeprecationWarning, stacklevel=2) + self.notify_all() class Semaphore: @@ -530,7 +547,16 @@ class Event: """Return true if and only if the internal flag is true.""" return self._flag - isSet = is_set + def isSet(self): + """Return true if and only if the internal flag is true. + + This method is deprecated, use notify_all() instead. + + """ + import warnings + warnings.warn('isSet() is deprecated, use is_set() instead', + DeprecationWarning, stacklevel=2) + return self.is_set() def set(self): """Set the internal flag to true. @@ -745,10 +771,9 @@ class BrokenBarrierError(RuntimeError): # Helper to generate new thread names -_counter = _count().__next__ -_counter() # Consume 0 so first non-main thread has id 1. -def _newname(template="Thread-%d"): - return template % _counter() +_counter = _count(1).__next__ +def _newname(name_template): + return name_template % _counter() # Active thread administration. # @@ -818,8 +843,19 @@ class Thread: assert group is None, "group argument must be None for now" if kwargs is None: kwargs = {} + if name: + name = str(name) + else: + name = _newname("Thread-%d") + if target is not None: + try: + target_name = target.__name__ + name += f" ({target_name})" + except AttributeError: + pass + self._target = target - self._name = str(name or _newname()) + self._name = name self._args = args self._kwargs = kwargs if daemon is not None: @@ -906,7 +942,7 @@ class Thread: """ try: - if self._target: + if self._target is not None: self._target(*self._args, **self._kwargs) finally: # Avoid a refcycle if the thread is running a function with @@ -1161,15 +1197,47 @@ class Thread: self._daemonic = daemonic def isDaemon(self): + """Return whether this thread is a daemon. + + This method is deprecated, use the daemon attribute instead. + + """ + import warnings + warnings.warn('isDaemon() is deprecated, get the daemon attribute instead', + DeprecationWarning, stacklevel=2) return self.daemon def setDaemon(self, daemonic): + """Set whether this thread is a daemon. + + This method is deprecated, use the .daemon property instead. + + """ + import warnings + warnings.warn('setDaemon() is deprecated, set the daemon attribute instead', + DeprecationWarning, stacklevel=2) self.daemon = daemonic def getName(self): + """Return a string used for identification purposes only. + + This method is deprecated, use the name attribute instead. + + """ + import warnings + warnings.warn('getName() is deprecated, get the name attribute instead', + DeprecationWarning, stacklevel=2) return self.name def setName(self, name): + """Set the name string for this thread. + + This method is deprecated, use the name attribute instead. + + """ + import warnings + warnings.warn('setName() is deprecated, set the name attribute instead', + DeprecationWarning, stacklevel=2) self.name = name @@ -1219,6 +1287,10 @@ except ImportError: stderr.flush() +# Original value of threading.excepthook +__excepthook__ = excepthook + + def _make_invoke_excepthook(): # Create a local namespace to ensure that variables remain alive # when _invoke_excepthook() is called, even if it is called late during @@ -1360,7 +1432,16 @@ def current_thread(): except KeyError: return _DummyThread() -currentThread = current_thread +def currentThread(): + """Return the current Thread object, corresponding to the caller's thread of control. + + This function is deprecated, use current_thread() instead. + + """ + import warnings + warnings.warn('currentThread() is deprecated, use current_thread() instead', + DeprecationWarning, stacklevel=2) + return current_thread() def active_count(): """Return the number of Thread objects currently alive. @@ -1372,7 +1453,16 @@ def active_count(): with _active_limbo_lock: return len(_active) + len(_limbo) -activeCount = active_count +def activeCount(): + """Return the number of Thread objects currently alive. + + This function is deprecated, use active_count() instead. + + """ + import warnings + warnings.warn('activeCount() is deprecated, use active_count() instead', + DeprecationWarning, stacklevel=2) + return active_count() def _enumerate(): # Same as enumerate(), but without the lock. Internal use only. diff --git a/contrib/tools/python3/src/Lib/timeit.py b/contrib/tools/python3/src/Lib/timeit.py index 6c3ec01067f..9dfd454936e 100755 --- a/contrib/tools/python3/src/Lib/timeit.py +++ b/contrib/tools/python3/src/Lib/timeit.py @@ -72,6 +72,7 @@ def inner(_it, _timer{init}): _t0 = _timer() for _i in _it: {stmt} + pass _t1 = _timer() return _t1 - _t0 """ diff --git a/contrib/tools/python3/src/Lib/token.py b/contrib/tools/python3/src/Lib/token.py index 493bf042650..9d0c0bf0fb0 100644 --- a/contrib/tools/python3/src/Lib/token.py +++ b/contrib/tools/python3/src/Lib/token.py @@ -62,12 +62,13 @@ AWAIT = 55 ASYNC = 56 TYPE_IGNORE = 57 TYPE_COMMENT = 58 +SOFT_KEYWORD = 59 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 59 -COMMENT = 60 -NL = 61 -ENCODING = 62 -N_TOKENS = 63 +ERRORTOKEN = 60 +COMMENT = 61 +NL = 62 +ENCODING = 63 +N_TOKENS = 64 # Special definitions for cooperation with parser NT_OFFSET = 256 diff --git a/contrib/tools/python3/src/Lib/tokenize.py b/contrib/tools/python3/src/Lib/tokenize.py index a782f6250dd..7d7736fe985 100644 --- a/contrib/tools/python3/src/Lib/tokenize.py +++ b/contrib/tools/python3/src/Lib/tokenize.py @@ -27,6 +27,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections +import functools from io import TextIOWrapper import itertools as _itertools import re @@ -95,6 +96,7 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache def _compile(expr): return re.compile(expr, re.UNICODE) diff --git a/contrib/tools/python3/src/Lib/trace.py b/contrib/tools/python3/src/Lib/trace.py index c505d8bc72a..2cf3643878d 100755 --- a/contrib/tools/python3/src/Lib/trace.py +++ b/contrib/tools/python3/src/Lib/trace.py @@ -116,7 +116,7 @@ class _Ignore: return 0 def _modname(path): - """Return a plausible module name for the patch.""" + """Return a plausible module name for the path.""" base = os.path.basename(path) filename, ext = os.path.splitext(base) diff --git a/contrib/tools/python3/src/Lib/traceback.py b/contrib/tools/python3/src/Lib/traceback.py index c771a137cd9..d6a010f4157 100644 --- a/contrib/tools/python3/src/Lib/traceback.py +++ b/contrib/tools/python3/src/Lib/traceback.py @@ -84,7 +84,25 @@ _context_message = ( "another exception occurred:\n\n") -def print_exception(etype, value, tb, limit=None, file=None, chain=True): +class _Sentinel: + def __repr__(self): + return "" + +_sentinel = _Sentinel() + +def _parse_value_tb(exc, value, tb): + if (value is _sentinel) != (tb is _sentinel): + raise ValueError("Both or neither of value and tb must be given") + if value is tb is _sentinel: + if exc is not None: + return exc, exc.__traceback__ + else: + return None, None + return value, tb + + +def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ + file=None, chain=True): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -95,17 +113,16 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True): occurred with a caret on the next line indicating the approximate position of the error. """ - # format_exception has ignored etype for some time, and code such as cgitb - # passes in bogus values as a result. For compatibility with such code we - # ignore it here (rather than in the new TracebackException API). + value, tb = _parse_value_tb(exc, value, tb) if file is None: file = sys.stderr - for line in TracebackException( - type(value), value, tb, limit=limit).format(chain=chain): + te = TracebackException(type(value), value, tb, limit=limit, compact=True) + for line in te.format(chain=chain): print(line, file=file, end="") -def format_exception(etype, value, tb, limit=None, chain=True): +def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ + chain=True): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -114,19 +131,15 @@ def format_exception(etype, value, tb, limit=None, chain=True): these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ - # format_exception has ignored etype for some time, and code such as cgitb - # passes in bogus values as a result. For compatibility with such code we - # ignore it here (rather than in the new TracebackException API). - return list(TracebackException( - type(value), value, tb, limit=limit).format(chain=chain)) + value, tb = _parse_value_tb(exc, value, tb) + te = TracebackException(type(value), value, tb, limit=limit, compact=True) + return list(te.format(chain=chain)) -def format_exception_only(etype, value): +def format_exception_only(exc, /, value=_sentinel): """Format the exception part of a traceback. - The arguments are the exception type and value such as given by - sys.last_type and sys.last_value. The return value is a list of - strings, each ending in a newline. + The return value is a list of strings, each ending in a newline. Normally, the list contains a single string; however, for SyntaxError exceptions, it contains several lines that (when @@ -137,7 +150,10 @@ def format_exception_only(etype, value): string in the list. """ - return list(TracebackException(etype, value, None).format_exception_only()) + if value is _sentinel: + value = exc + te = TracebackException(type(value), value, None, compact=True) + return list(te.format_exception_only()) # -- not official API but folk probably use these two functions. @@ -285,9 +301,10 @@ class FrameSummary: @property def line(self): if self._line is None: - self._line = linecache.getline(self.filename, self.lineno).strip() - return self._line - + if self.lineno is None: + return None + self._line = linecache.getline(self.filename, self.lineno) + return self._line.strip() def walk_stack(f): """Walk a stack yielding the frame and line number for each frame. @@ -458,61 +475,29 @@ class TracebackException: occurred. - :attr:`lineno` For syntax errors - the linenumber where the error occurred. + - :attr:`end_lineno` For syntax errors - the end linenumber where the error + occurred. Can be `None` if not present. - :attr:`text` For syntax errors - the text where the error occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. + - :attr:`end_offset` For syntax errors - the offset into the text where the + error occurred. Can be `None` if not present. - :attr:`msg` For syntax errors - the compiler error message. """ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, - lookup_lines=True, capture_locals=False, _seen=None): + lookup_lines=True, capture_locals=False, compact=False, + _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. + is_recursive_call = _seen is not None if _seen is None: _seen = set() _seen.add(id(exc_value)) - # Gracefully handle (the way Python 2.4 and earlier did) the case of - # being called with no type or value (None, None, None). - self._truncated = False - try: - if (exc_value and exc_value.__cause__ is not None - and id(exc_value.__cause__) not in _seen): - cause = TracebackException( - type(exc_value.__cause__), - exc_value.__cause__, - exc_value.__cause__.__traceback__, - limit=limit, - lookup_lines=False, - capture_locals=capture_locals, - _seen=_seen) - else: - cause = None - if (exc_value and exc_value.__context__ is not None - and id(exc_value.__context__) not in _seen): - context = TracebackException( - type(exc_value.__context__), - exc_value.__context__, - exc_value.__context__.__traceback__, - limit=limit, - lookup_lines=False, - capture_locals=capture_locals, - _seen=_seen) - else: - context = None - except RecursionError: - # The recursive call to the constructors above - # may result in a stack overflow for long exception chains, - # so we must truncate. - self._truncated = True - cause = None - context = None - self.__cause__ = cause - self.__context__ = context - self.__suppress_context__ = \ - exc_value.__suppress_context__ if exc_value else False + # TODO: locals. self.stack = StackSummary.extract( walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, @@ -526,11 +511,60 @@ class TracebackException: self.filename = exc_value.filename lno = exc_value.lineno self.lineno = str(lno) if lno is not None else None + end_lno = exc_value.end_lineno + self.end_lineno = str(end_lno) if end_lno is not None else None self.text = exc_value.text self.offset = exc_value.offset + self.end_offset = exc_value.end_offset self.msg = exc_value.msg if lookup_lines: self._load_lines() + self.__suppress_context__ = \ + exc_value.__suppress_context__ if exc_value is not None else False + + # Convert __cause__ and __context__ to `TracebackExceptions`s, use a + # queue to avoid recursion (only the top-level call gets _seen == None) + if not is_recursive_call: + queue = [(self, exc_value)] + while queue: + te, e = queue.pop() + if (e and e.__cause__ is not None + and id(e.__cause__) not in _seen): + cause = TracebackException( + type(e.__cause__), + e.__cause__, + e.__cause__.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + _seen=_seen) + else: + cause = None + + if compact: + need_context = (cause is None and + e is not None and + not e.__suppress_context__) + else: + need_context = True + if (e and e.__context__ is not None + and need_context and id(e.__context__) not in _seen): + context = TracebackException( + type(e.__context__), + e.__context__, + e.__context__.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + _seen=_seen) + else: + context = None + te.__cause__ = cause + te.__context__ = context + if cause: + queue.append((te.__cause__, e.__cause__)) + if context: + queue.append((te.__context__, e.__context__)) @classmethod def from_exception(cls, exc, *args, **kwargs): @@ -541,10 +575,6 @@ class TracebackException: """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: frame.line - if self.__context__: - self.__context__._load_lines() - if self.__cause__: - self.__cause__._load_lines() def __eq__(self, other): if isinstance(other, TracebackException): @@ -602,12 +632,20 @@ class TracebackException: ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) yield ' {}\n'.format(ltext) - # Convert 1-based column offset to 0-based index into stripped text - caret = (self.offset or 0) - 1 - spaces - if caret >= 0: - # non-space whitespace (likes tabs) must be kept for alignment - caretspace = ((c if c.isspace() else ' ') for c in ltext[:caret]) - yield ' {}^\n'.format(''.join(caretspace)) + + if self.offset is not None: + offset = self.offset + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if offset == end_offset or end_offset == -1: + end_offset = offset + 1 + + # Convert 1-based column offset to 0-based index into stripped text + colno = offset - 1 - spaces + end_colno = end_offset - 1 - spaces + if colno >= 0: + # non-space whitespace (likes tabs) must be kept for alignment + caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) + yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) msg = self.msg or "" yield "{}: {}{}\n".format(stype, msg, filename_suffix) @@ -623,19 +661,32 @@ class TracebackException: The message indicating which exception occurred is always the last string in the output. """ - if chain: - if self.__cause__ is not None: - yield from self.__cause__.format(chain=chain) - yield _cause_message - elif (self.__context__ is not None and - not self.__suppress_context__): - yield from self.__context__.format(chain=chain) - yield _context_message - if self._truncated: - yield ( - 'Chained exceptions have been truncated to avoid ' - 'stack overflow in traceback formatting:\n') - if self.stack: - yield 'Traceback (most recent call last):\n' - yield from self.stack.format() - yield from self.format_exception_only() + + output = [] + exc = self + while exc: + if chain: + if exc.__cause__ is not None: + chained_msg = _cause_message + chained_exc = exc.__cause__ + elif (exc.__context__ is not None and + not exc.__suppress_context__): + chained_msg = _context_message + chained_exc = exc.__context__ + else: + chained_msg = None + chained_exc = None + + output.append((chained_msg, exc)) + exc = chained_exc + else: + output.append((None, exc)) + exc = None + + for msg, exc in reversed(output): + if msg is not None: + yield msg + if exc.stack: + yield 'Traceback (most recent call last):\n' + yield from exc.stack.format() + yield from exc.format_exception_only() diff --git a/contrib/tools/python3/src/Lib/turtle.py b/contrib/tools/python3/src/Lib/turtle.py index 9c8f6ced250..f3b320b90ca 100644 --- a/contrib/tools/python3/src/Lib/turtle.py +++ b/contrib/tools/python3/src/Lib/turtle.py @@ -264,12 +264,12 @@ class Vec2D(tuple): def __neg__(self): return Vec2D(-self[0], -self[1]) def __abs__(self): - return (self[0]**2 + self[1]**2)**0.5 + return math.hypot(*self) def rotate(self, angle): """rotate self counterclockwise by angle """ perp = Vec2D(-self[1], self[0]) - angle = angle * math.pi / 180.0 + angle = math.radians(angle) c, s = math.cos(angle), math.sin(angle) return Vec2D(self[0]*c+perp[0]*s, self[1]*c+perp[1]*s) def __getnewargs__(self): @@ -1598,7 +1598,7 @@ class TNavigator(object): >>> turtle.heading() 1.5707963267948966 """ - self._setDegreesPerAU(2*math.pi) + self._setDegreesPerAU(math.tau) def _go(self, distance): """move turtle forward by specified distance""" @@ -1889,7 +1889,7 @@ class TNavigator(object): elif isinstance(x, TNavigator): pos = x._position x, y = pos - self._position - result = round(math.atan2(y, x)*180.0/math.pi, 10) % 360.0 + result = round(math.degrees(math.atan2(y, x)), 10) % 360.0 result /= self._degreesPerAU return (self._angleOffset + self._angleOrient*result) % self._fullcircle @@ -1904,7 +1904,7 @@ class TNavigator(object): 67.0 """ x, y = self._orient - result = round(math.atan2(y, x)*180.0/math.pi, 10) % 360.0 + result = round(math.degrees(math.atan2(y, x)), 10) % 360.0 result /= self._degreesPerAU return (self._angleOffset + self._angleOrient*result) % self._fullcircle @@ -1977,7 +1977,7 @@ class TNavigator(object): steps = 1+int(min(11+abs(radius)/6.0, 59.0)*frac) w = 1.0 * extent / steps w2 = 0.5 * w - l = 2.0 * radius * math.sin(w2*math.pi/180.0*self._degreesPerAU) + l = 2.0 * radius * math.sin(math.radians(w2)*self._degreesPerAU) if radius < 0: l, w, w2 = -l, -w, -w2 tr = self._tracer() @@ -2862,7 +2862,7 @@ class RawTurtle(TPen, TNavigator): >>> turtle.fd(50) """ tilt = -angle * self._degreesPerAU * self._angleOrient - tilt = (tilt * math.pi / 180.0) % (2*math.pi) + tilt = math.radians(tilt) % math.tau self.pen(resizemode="user", tilt=tilt) def tiltangle(self, angle=None): @@ -2887,7 +2887,7 @@ class RawTurtle(TPen, TNavigator): >>> turtle.tiltangle() """ if angle is None: - tilt = -self._tilt * (180.0/math.pi) * self._angleOrient + tilt = -math.degrees(self._tilt) * self._angleOrient return (tilt / self._degreesPerAU) % self._fullcircle else: self.settiltangle(angle) @@ -2941,7 +2941,7 @@ class RawTurtle(TPen, TNavigator): if t11 * t22 - t12 * t21 == 0: raise TurtleGraphicsError("Bad shape transform matrix: must not be singular") self._shapetrafo = (m11, m12, m21, m22) - alfa = math.atan2(-m21, m11) % (2 * math.pi) + alfa = math.atan2(-m21, m11) % math.tau sa, ca = math.sin(alfa), math.cos(alfa) a11, a12, a21, a22 = (ca*m11 - sa*m21, ca*m12 - sa*m22, sa*m11 + ca*m21, sa*m12 + ca*m22) diff --git a/contrib/tools/python3/src/Lib/types.py b/contrib/tools/python3/src/Lib/types.py index f15fda5933a..62122a99486 100644 --- a/contrib/tools/python3/src/Lib/types.py +++ b/contrib/tools/python3/src/Lib/types.py @@ -155,7 +155,12 @@ class DynamicClassAttribute: class's __getattr__ method; this is done by raising AttributeError. This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see Enum for an example). + attributes on the class with the same name. (Enum used this between Python + versions 3.4 - 3.9 .) + + Subclass from this to use a different method of accessing virtual atributes + and still be treated properly by the inspect module. (Enum uses this since + Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): @@ -292,8 +297,11 @@ def coroutine(func): return wrapped - GenericAlias = type(list[int]) +UnionType = type(int | str) +EllipsisType = type(Ellipsis) +NoneType = type(None) +NotImplementedType = type(NotImplemented) __all__ = [n for n in globals() if n[:1] != '_'] diff --git a/contrib/tools/python3/src/Lib/typing.py b/contrib/tools/python3/src/Lib/typing.py index d35a2a5b018..086d0f3f959 100644 --- a/contrib/tools/python3/src/Lib/typing.py +++ b/contrib/tools/python3/src/Lib/typing.py @@ -4,8 +4,10 @@ The typing module: Support for gradual typing as defined by PEP 484. At large scale, the structure of the module is following: * Imports and exports, all public names should be explicitly added to __all__. * Internal helper functions: these should never be used in code outside this module. -* _SpecialForm and its instances (special forms): Any, NoReturn, ClassVar, Union, Optional -* Two classes whose instances can be type arguments in addition to types: ForwardRef and TypeVar +* _SpecialForm and its instances (special forms): + Any, NoReturn, ClassVar, Union, Optional, Concatenate +* Classes whose instances can be type arguments in addition to types: + ForwardRef, TypeVar and ParamSpec * The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str], etc., are instances of either of these classes. @@ -35,11 +37,13 @@ __all__ = [ 'Any', 'Callable', 'ClassVar', + 'Concatenate', 'Final', 'ForwardRef', 'Generic', 'Literal', 'Optional', + 'ParamSpec', 'Protocol', 'Tuple', 'Type', @@ -110,14 +114,19 @@ __all__ = [ 'get_args', 'get_origin', 'get_type_hints', + 'is_typeddict', 'NewType', 'no_type_check', 'no_type_check_decorator', 'NoReturn', 'overload', + 'ParamSpecArgs', + 'ParamSpecKwargs', 'runtime_checkable', 'Text', 'TYPE_CHECKING', + 'TypeAlias', + 'TypeGuard', ] # The pseudo-submodules 're' and 'io' are part of the public @@ -156,17 +165,23 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, Final): + if arg in (Any, NoReturn, Final, TypeAlias): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") - if isinstance(arg, (type, TypeVar, ForwardRef)): + if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec, + ParamSpecArgs, ParamSpecKwargs)): return arg if not callable(arg): raise TypeError(f"{msg} Got {arg!r:.100}.") return arg +def _is_param_expr(arg): + return arg is ... or isinstance(arg, + (tuple, list, ParamSpec, _ConcatenateGenericAlias)) + + def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). @@ -188,17 +203,19 @@ def _type_repr(obj): return repr(obj) -def _collect_type_vars(types): - """Collect all type variable contained in types in order of - first appearance (lexicographic order). For example:: +def _collect_type_vars(types_, typevar_types=None): + """Collect all type variable contained + in types in order of first appearance (lexicographic order). For example:: _collect_type_vars((T, List[S, T])) == (T, S) """ + if typevar_types is None: + typevar_types = TypeVar tvars = [] - for t in types: - if isinstance(t, TypeVar) and t not in tvars: + for t in types_: + if isinstance(t, typevar_types) and t not in tvars: tvars.append(t) - if isinstance(t, (_GenericAlias, GenericAlias)): + if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): tvars.extend([t for t in t.__parameters__ if t not in tvars]) return tuple(tvars) @@ -211,9 +228,27 @@ def _check_generic(cls, parameters, elen): raise TypeError(f"{cls} is not a generic class") alen = len(parameters) if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} parameters for {cls};" + raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" f" actual {alen}, expected {elen}") +def _prepare_paramspec_params(cls, params): + """Prepares the parameters for a Generic containing ParamSpec + variables (internal helper). + """ + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if (len(cls.__parameters__) == 1 + and params and not _is_param_expr(params[0])): + assert isinstance(cls.__parameters__[0], ParamSpec) + return (params,) + else: + _check_generic(cls, params, len(cls.__parameters__)) + _params = [] + # Convert lists to tuples to help other libraries cache the results. + for p, tvar in zip(params, cls.__parameters__): + if isinstance(tvar, ParamSpec) and isinstance(p, list): + p = tuple(p) + _params.append(p) + return tuple(_params) def _deduplicate(params): # Weed out strict duplicates, preserving the first of each occurrence. @@ -236,7 +271,7 @@ def _remove_dups_flatten(parameters): # Flatten out Union[Union[...], ...]. params = [] for p in parameters: - if isinstance(p, _UnionGenericAlias): + if isinstance(p, (_UnionGenericAlias, types.UnionType)): params.extend(p.__args__) elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: params.extend(p[1:]) @@ -290,12 +325,14 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): """ if isinstance(t, ForwardRef): return t._evaluate(globalns, localns, recursive_guard) - if isinstance(t, (_GenericAlias, GenericAlias)): + if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): return GenericAlias(t.__origin__, ev_args) + if isinstance(t, types.UnionType): + return functools.reduce(operator.or_, ev_args) else: return t.copy_with(ev_args) return t @@ -331,6 +368,12 @@ class _SpecialForm(_Final, _root=True): self._name = getitem.__name__ self.__doc__ = getitem.__doc__ + def __getattr__(self, item): + if item in {'__name__', '__qualname__'}: + return self._name + + raise AttributeError(item) + def __mro_entries__(self, bases): raise TypeError(f"Cannot subclass {self!r}") @@ -343,6 +386,12 @@ class _SpecialForm(_Final, _root=True): def __call__(self, *args, **kwds): raise TypeError(f"Cannot instantiate {self!r}") + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + def __instancecheck__(self, obj): raise TypeError(f"{self} cannot be used with isinstance()") @@ -467,6 +516,8 @@ def Union(self, parameters): parameters = _remove_dups_flatten(parameters) if len(parameters) == 1: return parameters[0] + if len(parameters) == 2 and type(None) in parameters: + return _UnionGenericAlias(self, parameters, name="Optional") return _UnionGenericAlias(self, parameters) @_SpecialForm @@ -513,6 +564,95 @@ def Literal(self, *parameters): return _LiteralGenericAlias(self, parameters) +@_SpecialForm +def TypeAlias(self, parameters): + """Special marker indicating that an assignment should + be recognized as a proper type alias definition by type + checkers. + + For example:: + + Predicate: TypeAlias = Callable[..., bool] + + It's invalid when used anywhere except as in the example above. + """ + raise TypeError(f"{self} is not subscriptable") + + +@_SpecialForm +def Concatenate(self, parameters): + """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a + higher order function which adds, removes or transforms parameters of a + callable. + + For example:: + + Callable[Concatenate[int, P], int] + + See PEP 612 for detailed information. + """ + if parameters == (): + raise TypeError("Cannot take a Concatenate of no types.") + if not isinstance(parameters, tuple): + parameters = (parameters,) + if not isinstance(parameters[-1], ParamSpec): + raise TypeError("The last parameter to Concatenate should be a " + "ParamSpec variable.") + msg = "Concatenate[arg, ...]: each arg must be a type." + parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) + return _ConcatenateGenericAlias(self, parameters, + _typevar_types=(TypeVar, ParamSpec), + _paramspec_tvars=True) + + +@_SpecialForm +def TypeGuard(self, parameters): + """Special typing form used to annotate the return type of a user-defined + type guard function. ``TypeGuard`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean. + + ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type guard". + + Sometimes it would be convenient to use a user-defined boolean function + as a type guard. Such a function should use ``TypeGuard[...]`` as its + return type to alert static type checkers to this intention. + + Using ``-> TypeGuard`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the type inside ``TypeGuard``. + + For example:: + + def is_str(val: Union[str, float]): + # "isinstance" type guard + if isinstance(val, str): + # Type of ``val`` is narrowed to ``str`` + ... + else: + # Else, type of ``val`` is narrowed to ``float``. + ... + + Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower + form of ``TypeA`` (it can even be a wider form) and this may lead to + type-unsafe results. The main reason is to allow for things like + narrowing ``List[object]`` to ``List[str]`` even though the latter is not + a subtype of the former, since ``List`` is invariant. The responsibility of + writing type-safe type guards is left to the user. + + ``TypeGuard`` also works with type variables. For more information, see + PEP 647 (User-Defined Type Guards). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) + + class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" @@ -577,8 +717,41 @@ class ForwardRef(_Final, _root=True): def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' +class _TypeVarLike: + """Mixin for TypeVar-like types (TypeVar and ParamSpec).""" + def __init__(self, bound, covariant, contravariant): + """Used to setup TypeVars and ParamSpec's bound, covariant and + contravariant attributes. + """ + if covariant and contravariant: + raise ValueError("Bivariant types are not supported.") + self.__covariant__ = bool(covariant) + self.__contravariant__ = bool(contravariant) + if bound: + self.__bound__ = _type_check(bound, "Bound must be a type.") + else: + self.__bound__ = None + + def __or__(self, right): + return Union[self, right] + + def __ror__(self, left): + return Union[left, self] -class TypeVar(_Final, _Immutable, _root=True): + def __repr__(self): + if self.__covariant__: + prefix = '+' + elif self.__contravariant__: + prefix = '-' + else: + prefix = '~' + return prefix + self.__name__ + + def __reduce__(self): + return self.__name__ + + +class TypeVar( _Final, _Immutable, _TypeVarLike, _root=True): """Type variable. Usage:: @@ -628,20 +801,13 @@ class TypeVar(_Final, _Immutable, _root=True): def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): self.__name__ = name - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) + super().__init__(bound, covariant, contravariant) if constraints and bound is not None: raise TypeError("Constraints cannot be combined with bound=...") if constraints and len(constraints) == 1: raise TypeError("A single constraint is not allowed") msg = "TypeVar(name, constraint, ...): constraints must be types." self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None try: def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling except (AttributeError, ValueError): @@ -649,17 +815,121 @@ class TypeVar(_Final, _Immutable, _root=True): if def_mod != 'typing': self.__module__ = def_mod + +class ParamSpecArgs(_Final, _Immutable, _root=True): + """The args for a ParamSpec object. + + Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. + + ParamSpecArgs objects have a reference back to their ParamSpec: + + P.args.__origin__ is P + + This type is meant for runtime introspection and has no special meaning to + static type checkers. + """ + def __init__(self, origin): + self.__origin__ = origin + def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ + return f"{self.__origin__.__name__}.args" - def __reduce__(self): - return self.__name__ + def __eq__(self, other): + if not isinstance(other, ParamSpecArgs): + return NotImplemented + return self.__origin__ == other.__origin__ + + +class ParamSpecKwargs(_Final, _Immutable, _root=True): + """The kwargs for a ParamSpec object. + + Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. + + ParamSpecKwargs objects have a reference back to their ParamSpec: + + P.kwargs.__origin__ is P + + This type is meant for runtime introspection and has no special meaning to + static type checkers. + """ + def __init__(self, origin): + self.__origin__ = origin + + def __repr__(self): + return f"{self.__origin__.__name__}.kwargs" + + def __eq__(self, other): + if not isinstance(other, ParamSpecKwargs): + return NotImplemented + return self.__origin__ == other.__origin__ + + +class ParamSpec(_Final, _Immutable, _TypeVarLike, _root=True): + """Parameter specification variable. + + Usage:: + + P = ParamSpec('P') + + Parameter specification variables exist primarily for the benefit of static + type checkers. They are used to forward the parameter types of one + callable to another callable, a pattern commonly found in higher order + functions and decorators. They are only valid when used in ``Concatenate``, + or as the first argument to ``Callable``, or as parameters for user-defined + Generics. See class Generic for more information on generic types. An + example for annotating a decorator:: + + T = TypeVar('T') + P = ParamSpec('P') + + def add_logging(f: Callable[P, T]) -> Callable[P, T]: + '''A type-safe decorator to add logging to a function.''' + def inner(*args: P.args, **kwargs: P.kwargs) -> T: + logging.info(f'{f.__name__} was called') + return f(*args, **kwargs) + return inner + + @add_logging + def add_two(x: float, y: float) -> float: + '''Add two numbers together.''' + return x + y + + Parameter specification variables defined with covariant=True or + contravariant=True can be used to declare covariant or contravariant + generic types. These keyword arguments are valid, but their actual semantics + are yet to be decided. See PEP 612 for details. + + Parameter specification variables can be introspected. e.g.: + + P.__name__ == 'T' + P.__bound__ == None + P.__covariant__ == False + P.__contravariant__ == False + + Note that only parameter specification variables defined in global scope can + be pickled. + """ + + __slots__ = ('__name__', '__bound__', '__covariant__', '__contravariant__', + '__dict__') + + @property + def args(self): + return ParamSpecArgs(self) + + @property + def kwargs(self): + return ParamSpecKwargs(self) + + def __init__(self, name, *, bound=None, covariant=False, contravariant=False): + self.__name__ = name + super().__init__(bound, covariant, contravariant) + try: + def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + def_mod = None + if def_mod != 'typing': + self.__module__ = def_mod def _is_dunder(attr): @@ -704,6 +974,9 @@ class _BaseGenericAlias(_Final, _root=True): return tuple(res) def __getattr__(self, attr): + if attr in {'__name__', '__qualname__'}: + return self._name or self.__origin__.__name__ + # We are careful for copy and pickle. # Also for simplicity we don't relay any dunder names if '__origin__' in self.__dict__ and not _is_dunder(attr): @@ -711,7 +984,8 @@ class _BaseGenericAlias(_Final, _root=True): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in ('_name', '_inst', '_nparams'): + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', + '_typevar_types', '_paramspec_tvars'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -723,6 +997,9 @@ class _BaseGenericAlias(_Final, _root=True): raise TypeError("Subscripted generics cannot be used with" " class and instance checks") + def __dir__(self): + return list(set(super().__dir__() + + [attr for attr in dir(self.__origin__) if not _is_dunder(attr)])) # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: @@ -736,14 +1013,18 @@ class _BaseGenericAlias(_Final, _root=True): class _GenericAlias(_BaseGenericAlias, _root=True): - def __init__(self, origin, params, *, inst=True, name=None): + def __init__(self, origin, params, *, inst=True, name=None, + _typevar_types=TypeVar, + _paramspec_tvars=False): super().__init__(origin, inst=inst, name=name) if not isinstance(params, tuple): params = (params,) self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in params) - self.__parameters__ = _collect_type_vars(params) + self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types) + self._typevar_types = _typevar_types + self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -756,6 +1037,12 @@ class _GenericAlias(_BaseGenericAlias, _root=True): def __hash__(self): return hash((self.__origin__, self.__args__)) + def __or__(self, right): + return Union[self, right] + + def __ror__(self, left): + return Union[left, self] + @_tp_cache def __getitem__(self, params): if self.__origin__ in (Generic, Protocol): @@ -763,25 +1050,40 @@ class _GenericAlias(_BaseGenericAlias, _root=True): raise TypeError(f"Cannot subscript already-subscripted {self}") if not isinstance(params, tuple): params = (params,) - msg = "Parameters to generic types must be types." - params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, len(self.__parameters__)) + params = tuple(_type_convert(p) for p in params) + if (self._paramspec_tvars + and any(isinstance(t, ParamSpec) for t in self.__parameters__)): + params = _prepare_paramspec_params(self, params) + else: + _check_generic(self, params, len(self.__parameters__)) subst = dict(zip(self.__parameters__, params)) new_args = [] for arg in self.__args__: - if isinstance(arg, TypeVar): - arg = subst[arg] - elif isinstance(arg, (_GenericAlias, GenericAlias)): + if isinstance(arg, self._typevar_types): + if isinstance(arg, ParamSpec): + arg = subst[arg] + if not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + else: + arg = subst[arg] + elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)): subparams = arg.__parameters__ if subparams: subargs = tuple(subst[x] for x in subparams) arg = arg[subargs] - new_args.append(arg) + # Required to flatten out the args for CallableGenericAlias + if self.__origin__ == collections.abc.Callable and isinstance(arg, tuple): + new_args.extend(arg) + else: + new_args.append(arg) return self.copy_with(tuple(new_args)) def copy_with(self, params): - return self.__class__(self.__origin__, params, name=self._name, inst=self._inst) + return self.__class__(self.__origin__, params, name=self._name, inst=self._inst, + _typevar_types=self._typevar_types, + _paramspec_tvars=self._paramspec_tvars) def __repr__(self): if self._name: @@ -802,6 +1104,9 @@ class _GenericAlias(_BaseGenericAlias, _root=True): return operator.getitem, (origin, args) def __mro_entries__(self, bases): + if isinstance(self.__origin__, _SpecialForm): + raise TypeError(f"Cannot subclass {self!r}") + if self._name: # generic version of an ABC or built-in class return super().__mro_entries__(bases) if self.__origin__ is Generic: @@ -855,19 +1160,25 @@ class _SpecialGenericAlias(_BaseGenericAlias, _root=True): def __reduce__(self): return self._name + def __or__(self, right): + return Union[self, right] + + def __ror__(self, left): + return Union[left, self] class _CallableGenericAlias(_GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' - if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: + args = self.__args__ + if len(args) == 2 and _is_param_expr(args[0]): return super().__repr__() return (f'typing.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([_type_repr(a) for a in args[:-1]])}], ' + f'{_type_repr(args[-1])}]') def __reduce__(self): args = self.__args__ - if not (len(args) == 2 and args[0] is ...): + if not (len(args) == 2 and _is_param_expr(args[0])): args = list(args[:-1]), args[-1] return operator.getitem, (Callable, args) @@ -875,7 +1186,9 @@ class _CallableGenericAlias(_GenericAlias, _root=True): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst) + name=self._name, inst=self._inst, + _typevar_types=(TypeVar, ParamSpec), + _paramspec_tvars=True) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -926,7 +1239,7 @@ class _UnionGenericAlias(_GenericAlias, _root=True): return Union[params] def __eq__(self, other): - if not isinstance(other, _UnionGenericAlias): + if not isinstance(other, (_UnionGenericAlias, types.UnionType)): return NotImplemented return set(self.__args__) == set(other.__args__) @@ -942,6 +1255,18 @@ class _UnionGenericAlias(_GenericAlias, _root=True): return f'typing.Optional[{_type_repr(args[0])}]' return super().__repr__() + def __instancecheck__(self, obj): + return self.__subclasscheck__(type(obj)) + + def __subclasscheck__(self, cls): + for arg in self.__args__: + if issubclass(cls, arg): + return True + + def __reduce__(self): + func, (origin, args) = super().__reduce__() + return func, (Union, args) + def _value_and_type_iter(parameters): return ((p, type(p)) for p in parameters) @@ -959,6 +1284,18 @@ class _LiteralGenericAlias(_GenericAlias, _root=True): return hash(frozenset(_value_and_type_iter(self.__args__))) +class _ConcatenateGenericAlias(_GenericAlias, _root=True): + def copy_with(self, params): + if isinstance(params[-1], (list, tuple)): + return (*params[:-1], *params[-1]) + if isinstance(params[-1], _ConcatenateGenericAlias): + params = (*params[:-1], *params[-1].__args__) + elif not isinstance(params[-1], ParamSpec): + raise TypeError("The last parameter to Concatenate should be a " + "ParamSpec variable.") + return super().copy_with(params) + + class Generic: """Abstract base class for generic types. @@ -989,20 +1326,25 @@ class Generic: if not params and cls is not Tuple: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty") - msg = "Parameters to generic types must be types." - params = tuple(_type_check(p, msg) for p in params) + params = tuple(_type_convert(p) for p in params) if cls in (Generic, Protocol): # Generic and Protocol can only be subscripted with unique type variables. - if not all(isinstance(p, TypeVar) for p in params): + if not all(isinstance(p, (TypeVar, ParamSpec)) for p in params): raise TypeError( - f"Parameters to {cls.__name__}[...] must all be type variables") + f"Parameters to {cls.__name__}[...] must all be type variables " + f"or parameter specification variables.") if len(set(params)) != len(params): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - _check_generic(cls, params, len(cls.__parameters__)) - return _GenericAlias(cls, params) + if any(isinstance(t, ParamSpec) for t in cls.__parameters__): + params = _prepare_paramspec_params(cls, params) + else: + _check_generic(cls, params, len(cls.__parameters__)) + return _GenericAlias(cls, params, + _typevar_types=(TypeVar, ParamSpec), + _paramspec_tvars=True) def __init_subclass__(cls, *args, **kwargs): super().__init_subclass__(*args, **kwargs) @@ -1014,7 +1356,7 @@ class Generic: if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_vars(cls.__orig_bases__) + tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec)) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -1113,20 +1455,26 @@ def _no_init_or_replace_init(self, *args, **kwargs): cls.__init__(self, *args, **kwargs) +def _caller(depth=1, default='__main__'): + try: + return sys._getframe(depth + 1).f_globals.get('__name__', default) + except (AttributeError, ValueError): # For platforms without _getframe() + return None + -def _allow_reckless_class_cheks(): +def _allow_reckless_class_checks(depth=3): """Allow instance and class checks for special stdlib modules. The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ try: - return sys._getframe(3).f_globals['__name__'] in ['abc', 'functools'] + return sys._getframe(depth).f_globals['__name__'] in ['abc', 'functools'] except (AttributeError, ValueError): # For platforms without _getframe(). return True -_PROTO_WHITELIST = { +_PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', @@ -1141,6 +1489,14 @@ class _ProtocolMeta(ABCMeta): def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. + if ( + getattr(cls, '_is_protocol', False) and + not getattr(cls, '_is_runtime_protocol', False) and + not _allow_reckless_class_checks(depth=2) + ): + raise TypeError("Instance and class checks can only be used with" + " @runtime_checkable protocols") + if ((not getattr(cls, '_is_protocol', False) or _is_callable_members_only(cls)) and issubclass(instance.__class__, cls)): @@ -1203,12 +1559,12 @@ class Protocol(Generic, metaclass=_ProtocolMeta): # First, perform various sanity checks. if not getattr(cls, '_is_runtime_protocol', False): - if _allow_reckless_class_cheks(): + if _allow_reckless_class_checks(): return NotImplemented raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") if not _is_callable_members_only(cls): - if _allow_reckless_class_cheks(): + if _allow_reckless_class_checks(): return NotImplemented raise TypeError("Protocols with non-method members" " don't support issubclass()") @@ -1245,8 +1601,8 @@ class Protocol(Generic, metaclass=_ProtocolMeta): # ... otherwise check consistency of bases, and prohibit instantiation. for base in cls.__bases__: if not (base in (object, Generic) or - base.__module__ in _PROTO_WHITELIST and - base.__name__ in _PROTO_WHITELIST[base.__module__] or + base.__module__ in _PROTO_ALLOWLIST and + base.__name__ in _PROTO_ALLOWLIST[base.__module__] or issubclass(base, Generic) and base._is_protocol): raise TypeError('Protocols can only inherit from other' ' protocols, got %r' % base) @@ -1293,6 +1649,11 @@ class _AnnotatedAlias(_GenericAlias, _root=True): def __hash__(self): return hash((self.__origin__, self.__metadata__)) + def __getattr__(self, attr): + if attr in {'__name__', '__qualname__'}: + return 'Annotated' + return super().__getattr__(attr) + class Annotated: """Add context specific metadata to a type. @@ -1431,7 +1792,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): - If no dict arguments are passed, an attempt is made to use the globals from obj (or the respective module's globals for classes), and these are also used as the locals. If the object does not appear - to have globals, an empty dictionary is used. + to have globals, an empty dictionary is used. For classes, the search + order is globals first then locals. - If one dict argument is passed, it is used for both globals and locals. @@ -1447,16 +1809,27 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): hints = {} for base in reversed(obj.__mro__): if globalns is None: - base_globals = sys.modules[base.__module__].__dict__ + base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns ann = base.__dict__.get('__annotations__', {}) + if isinstance(ann, types.GetSetDescriptorType): + ann = {} + base_locals = dict(vars(base)) if localns is None else localns + if localns is None and globalns is None: + # This is surprising, but required. Before Python 3.10, + # get_type_hints only evaluated the globalns of + # a class. To maintain backwards compatibility, we reverse + # the globalns and localns order so that eval() looks into + # *base_globals* first rather than *base_locals*. + # This only affects ForwardRefs. + base_globals, base_locals = base_locals, base_globals for name, value in ann.items(): if value is None: value = type(None) if isinstance(value, str): value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, localns) + value = _eval_type(value, base_globals, base_locals) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -1516,6 +1889,12 @@ def _strip_annotations(t): if stripped_args == t.__args__: return t return GenericAlias(t.__origin__, stripped_args) + if isinstance(t, types.UnionType): + stripped_args = tuple(_strip_annotations(a) for a in t.__args__) + if stripped_args == t.__args__: + return t + return functools.reduce(operator.or_, stripped_args) + return t @@ -1532,13 +1911,17 @@ def get_origin(tp): get_origin(Generic[T]) is Generic get_origin(Union[T, int]) is Union get_origin(List[Tuple[T, T]][int]) == list + get_origin(P.args) is P """ if isinstance(tp, _AnnotatedAlias): return Annotated - if isinstance(tp, (_BaseGenericAlias, GenericAlias)): + if isinstance(tp, (_BaseGenericAlias, GenericAlias, + ParamSpecArgs, ParamSpecKwargs)): return tp.__origin__ if tp is Generic: return Generic + if isinstance(tp, types.UnionType): + return types.UnionType return None @@ -1557,12 +1940,29 @@ def get_args(tp): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, (_GenericAlias, GenericAlias)): res = tp.__args__ - if tp.__origin__ is collections.abc.Callable and res[0] is not Ellipsis: + if (tp.__origin__ is collections.abc.Callable + and not (len(res) == 2 and _is_param_expr(res[0]))): res = (list(res[:-1]), res[-1]) return res + if isinstance(tp, types.UnionType): + return tp.__args__ return () +def is_typeddict(tp): + """Check if an annotation is a TypedDict class + + For example:: + class Film(TypedDict): + title: str + year: int + + is_typeddict(Film) # => True + is_typeddict(Union[list, str]) # => False + """ + return isinstance(tp, _TypedDictMeta) + + def no_type_check(arg): """Decorator to indicate that annotations are not type hints. @@ -2047,11 +2447,11 @@ _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) -def NewType(name, tp): +class NewType: """NewType creates simple unique types with almost zero runtime overhead. NewType(name, tp) is considered a subtype of tp by static type checkers. At runtime, NewType(name, tp) returns - a dummy function that simply returns its argument. Usage:: + a dummy callable that simply returns its argument. Usage:: UserId = NewType('UserId', int) @@ -2066,12 +2466,30 @@ def NewType(name, tp): num = UserId(5) + 1 # type: int """ - def new_type(x): + def __init__(self, name, tp): + self.__qualname__ = name + if '.' in name: + name = name.rpartition('.')[-1] + self.__name__ = name + self.__supertype__ = tp + def_mod = _caller() + if def_mod != 'typing': + self.__module__ = def_mod + + def __repr__(self): + return f'{self.__module__}.{self.__qualname__}' + + def __call__(self, x): return x - new_type.__name__ = name - new_type.__supertype__ = tp - return new_type + def __reduce__(self): + return self.__qualname__ + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] # Python-version-specific alias (Python 2: unicode; Python 3: str) diff --git a/contrib/tools/python3/src/Lib/unittest/_log.py b/contrib/tools/python3/src/Lib/unittest/_log.py index 94e7e758bd9..94868e5bb95 100644 --- a/contrib/tools/python3/src/Lib/unittest/_log.py +++ b/contrib/tools/python3/src/Lib/unittest/_log.py @@ -26,11 +26,11 @@ class _CapturingHandler(logging.Handler): class _AssertLogsContext(_BaseTestCaseContext): - """A context manager used to implement TestCase.assertLogs().""" + """A context manager for assertLogs() and assertNoLogs() """ LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" - def __init__(self, test_case, logger_name, level): + def __init__(self, test_case, logger_name, level, no_logs): _BaseTestCaseContext.__init__(self, test_case) self.logger_name = logger_name if level: @@ -38,6 +38,7 @@ class _AssertLogsContext(_BaseTestCaseContext): else: self.level = logging.INFO self.msg = None + self.no_logs = no_logs def __enter__(self): if isinstance(self.logger_name, logging.Logger): @@ -46,6 +47,7 @@ class _AssertLogsContext(_BaseTestCaseContext): logger = self.logger = logging.getLogger(self.logger_name) formatter = logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() + handler.setLevel(self.level) handler.setFormatter(formatter) self.watcher = handler.watcher self.old_handlers = logger.handlers[:] @@ -54,16 +56,31 @@ class _AssertLogsContext(_BaseTestCaseContext): logger.handlers = [handler] logger.setLevel(self.level) logger.propagate = False + if self.no_logs: + return return handler.watcher def __exit__(self, exc_type, exc_value, tb): self.logger.handlers = self.old_handlers self.logger.propagate = self.old_propagate self.logger.setLevel(self.old_level) + if exc_type is not None: # let unexpected exceptions pass through return False - if len(self.watcher.records) == 0: - self._raiseFailure( - "no logs of level {} or higher triggered on {}" - .format(logging.getLevelName(self.level), self.logger.name)) + + if self.no_logs: + # assertNoLogs + if len(self.watcher.records) > 0: + self._raiseFailure( + "Unexpected logs found: {!r}".format( + self.watcher.output + ) + ) + + else: + # assertLogs + if len(self.watcher.records) == 0: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) diff --git a/contrib/tools/python3/src/Lib/unittest/async_case.py b/contrib/tools/python3/src/Lib/unittest/async_case.py index a2980e797ac..23231199f98 100644 --- a/contrib/tools/python3/src/Lib/unittest/async_case.py +++ b/contrib/tools/python3/src/Lib/unittest/async_case.py @@ -134,7 +134,7 @@ class IsolatedAsyncioTestCase(TestCase): task.cancel() loop.run_until_complete( - asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)) + asyncio.gather(*to_cancel, return_exceptions=True)) for task in to_cancel: if task.cancelled(): diff --git a/contrib/tools/python3/src/Lib/unittest/case.py b/contrib/tools/python3/src/Lib/unittest/case.py index 88f1a408651..61003d0c6ea 100644 --- a/contrib/tools/python3/src/Lib/unittest/case.py +++ b/contrib/tools/python3/src/Lib/unittest/case.py @@ -295,7 +295,6 @@ class _AssertWarnsContext(_AssertRaisesBaseContext): self._raiseFailure("{} not triggered".format(exc_name)) - class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -794,7 +793,16 @@ class TestCase(object): """ # Lazy import to avoid importing logging if it is not needed. from ._log import _AssertLogsContext - return _AssertLogsContext(self, logger, level) + return _AssertLogsContext(self, logger, level, no_logs=False) + + def assertNoLogs(self, logger=None, level=None): + """ Fail unless no log messages of level *level* or higher are emitted + on *logger_name* or its children. + + This method must be used as a context manager. + """ + from ._log import _AssertLogsContext + return _AssertLogsContext(self, logger, level, no_logs=True) def _getAssertEqualityFunc(self, first, second): """Get a detailed comparison function for the types of the two args. @@ -1138,7 +1146,8 @@ class TestCase(object): def assertDictContainsSubset(self, subset, dictionary, msg=None): """Checks whether dictionary is a superset of subset.""" warnings.warn('assertDictContainsSubset is deprecated', - DeprecationWarning) + DeprecationWarning, + stacklevel=2) missing = [] mismatched = [] for key, value in subset.items(): diff --git a/contrib/tools/python3/src/Lib/unittest/mock.py b/contrib/tools/python3/src/Lib/unittest/mock.py index 3f5442ed809..7152f86ed96 100644 --- a/contrib/tools/python3/src/Lib/unittest/mock.py +++ b/contrib/tools/python3/src/Lib/unittest/mock.py @@ -36,6 +36,10 @@ from unittest.util import safe_repr from functools import wraps, partial +class InvalidSpecError(Exception): + """Indicates that an invalid value was used as a mock spec.""" + + _builtins = {name for name in dir(builtins) if not name.startswith('_')} FILTER_DIR = True @@ -631,9 +635,10 @@ class NonCallableMock(Base): elif _is_magic(name): raise AttributeError(name) if not self._mock_unsafe: - if name.startswith(('assert', 'assret')): - raise AttributeError("Attributes cannot start with 'assert' " - "or 'assret'") + if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')): + raise AttributeError( + f"{name!r} is not a valid assertion. Use a spec " + f"for the mock if {name!r} is meant to be an attribute.") result = self._mock_children.get(name) if result is _deleted: @@ -652,10 +657,17 @@ class NonCallableMock(Base): self._mock_children[name] = result elif isinstance(result, _SpecState): - result = create_autospec( - result.spec, result.spec_set, result.instance, - result.parent, result.name - ) + try: + result = create_autospec( + result.spec, result.spec_set, result.instance, + result.parent, result.name + ) + except InvalidSpecError: + target_name = self.__dict__['_mock_name'] or self + raise InvalidSpecError( + f'Cannot autospec attr {name!r} from target ' + f'{target_name!r} as it has already been mocked out. ' + f'[target={self!r}, attr={result.spec!r}]') self._mock_children[name] = result return result @@ -1240,6 +1252,17 @@ def _importer(target): return thing +# _check_spec_arg_typos takes kwargs from commands like patch and checks that +# they don't contain common misspellings of arguments related to autospeccing. +def _check_spec_arg_typos(kwargs_to_check): + typos = ("autospect", "auto_spec", "set_spec") + for typo in typos: + if typo in kwargs_to_check: + raise RuntimeError( + f"{typo!r} might be a typo; use unsafe=True if this is intended" + ) + + class _patch(object): attribute_name = None @@ -1247,7 +1270,7 @@ class _patch(object): def __init__( self, getter, attribute, new, spec, create, - spec_set, autospec, new_callable, kwargs + spec_set, autospec, new_callable, kwargs, *, unsafe=False ): if new_callable is not None: if new is not DEFAULT: @@ -1258,6 +1281,16 @@ class _patch(object): raise ValueError( "Cannot use 'autospec' and 'new_callable' together" ) + if not unsafe: + _check_spec_arg_typos(kwargs) + if _is_instance_mock(spec): + raise InvalidSpecError( + f'Cannot spec attr {attribute!r} as the spec ' + f'has already been mocked out. [spec={spec!r}]') + if _is_instance_mock(spec_set): + raise InvalidSpecError( + f'Cannot spec attr {attribute!r} as the spec_set ' + f'target has already been mocked out. [spec_set={spec_set!r}]') self.getter = getter self.attribute = attribute @@ -1485,6 +1518,18 @@ class _patch(object): if autospec is True: autospec = original + if _is_instance_mock(self.target): + raise InvalidSpecError( + f'Cannot autospec attr {self.attribute!r} as the patch ' + f'target has already been mocked out. ' + f'[target={self.target!r}, attr={autospec!r}]') + if _is_instance_mock(autospec): + target_name = getattr(self.target, '__name__', self.target) + raise InvalidSpecError( + f'Cannot autospec attr {self.attribute!r} from target ' + f'{target_name!r} as it has already been mocked out. ' + f'[target={self.target!r}, attr={autospec!r}]') + new = create_autospec(autospec, spec_set=spec_set, _name=self.attribute, **kwargs) elif kwargs: @@ -1567,7 +1612,7 @@ def _get_target(target): def _patch_object( target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, - new_callable=None, **kwargs + new_callable=None, *, unsafe=False, **kwargs ): """ patch the named member (`attribute`) on an object (`target`) with a mock @@ -1589,7 +1634,7 @@ def _patch_object( getter = lambda: target return _patch( getter, attribute, new, spec, create, - spec_set, autospec, new_callable, kwargs + spec_set, autospec, new_callable, kwargs, unsafe=unsafe ) @@ -1644,7 +1689,7 @@ def _patch_multiple(target, spec=None, create=False, spec_set=None, def patch( target, new=DEFAULT, spec=None, create=False, - spec_set=None, autospec=None, new_callable=None, **kwargs + spec_set=None, autospec=None, new_callable=None, *, unsafe=False, **kwargs ): """ `patch` acts as a function decorator, class decorator or a context @@ -1706,6 +1751,10 @@ def patch( use "as" then the patched object will be bound to the name after the "as"; very useful if `patch` is creating a mock object for you. + Patch will raise a `RuntimeError` if passed some common misspellings of + the arguments autospec and spec_set. Pass the argument `unsafe` with the + value True to disable that check. + `patch` takes arbitrary keyword arguments. These will be passed to `AsyncMock` if the patched object is asynchronous, to `MagicMock` otherwise or to `new_callable` if specified. @@ -1716,7 +1765,7 @@ def patch( getter, attribute = _get_target(target) return _patch( getter, attribute, new, spec, create, - spec_set, autospec, new_callable, kwargs + spec_set, autospec, new_callable, kwargs, unsafe=unsafe ) @@ -2566,7 +2615,7 @@ call = _Call(from_kall=False) def create_autospec(spec, spec_set=False, instance=False, _parent=None, - _name=None, **kwargs): + _name=None, *, unsafe=False, **kwargs): """Create a mock object using another object as a spec. Attributes on the mock will use the corresponding attribute on the `spec` object as their spec. @@ -2582,6 +2631,10 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, spec for an instance object by passing `instance=True`. The returned mock will only be callable if instances of the mock are callable. + `create_autospec` will raise a `RuntimeError` if passed some common + misspellings of the arguments autospec and spec_set. Pass the argument + `unsafe` with the value True to disable that check. + `create_autospec` also takes arbitrary keyword arguments that are passed to the constructor of the created mock.""" if _is_list(spec): @@ -2590,6 +2643,9 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, spec = type(spec) is_type = isinstance(spec, type) + if _is_instance_mock(spec): + raise InvalidSpecError(f'Cannot autospec a Mock object. ' + f'[object={spec!r}]') is_async_func = _is_async_func(spec) _kwargs = {'spec': spec} if spec_set: @@ -2599,6 +2655,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _kwargs = {} if _kwargs and instance: _kwargs['_spec_as_instance'] = True + if not unsafe: + _check_spec_arg_typos(kwargs) _kwargs.update(kwargs) diff --git a/contrib/tools/python3/src/Lib/unittest/result.py b/contrib/tools/python3/src/Lib/unittest/result.py index 528d223850e..3da7005e603 100644 --- a/contrib/tools/python3/src/Lib/unittest/result.py +++ b/contrib/tools/python3/src/Lib/unittest/result.py @@ -175,7 +175,8 @@ class TestResult(object): exctype, value, tb = err tb = self._clean_tracebacks(exctype, value, tb, test) tb_e = traceback.TracebackException( - exctype, value, tb, capture_locals=self.tb_locals) + exctype, value, tb, + capture_locals=self.tb_locals, compact=True) msgLines = list(tb_e.format()) if self.buffer: diff --git a/contrib/tools/python3/src/Lib/urllib/parse.py b/contrib/tools/python3/src/Lib/urllib/parse.py index b7965fe3d2b..b35997bc00c 100644 --- a/contrib/tools/python3/src/Lib/urllib/parse.py +++ b/contrib/tools/python3/src/Lib/urllib/parse.py @@ -754,9 +754,8 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') - pairs = [s1 for s1 in qs.split(separator)] r = [] - for name_value in pairs: + for name_value in qs.split(separator): if not name_value and not strict_parsing: continue nv = name_value.split('=', 1) diff --git a/contrib/tools/python3/src/Lib/urllib/request.py b/contrib/tools/python3/src/Lib/urllib/request.py index 4e289fc67df..34b1b0b0b76 100644 --- a/contrib/tools/python3/src/Lib/urllib/request.py +++ b/contrib/tools/python3/src/Lib/urllib/request.py @@ -202,6 +202,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=cafile, capath=capath) + # send ALPN extension to indicate HTTP/1.1 protocol + context.set_alpn_protocols(['http/1.1']) https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif context: diff --git a/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 b/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 index 9d3646a4fc6..b49d77ba44b 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 +++ b/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 @@ -96,6 +96,11 @@ function global:deactivate ([switch]$NonDestructive) { Remove-Item -Path env:VIRTUAL_ENV } + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force @@ -228,6 +233,7 @@ if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " _OLD_VIRTUAL_PROMPT } + $env:VIRTUAL_ENV_PROMPT = $Prompt } # Clear PYTHONHOME diff --git a/contrib/tools/python3/src/Lib/venv/scripts/common/activate b/contrib/tools/python3/src/Lib/venv/scripts/common/activate index 45af3536aa1..6fbc2b8801d 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/common/activate +++ b/contrib/tools/python3/src/Lib/venv/scripts/common/activate @@ -28,6 +28,7 @@ deactivate () { fi unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT if [ ! "${1:-}" = "nondestructive" ] ; then # Self destruct! unset -f deactivate @@ -56,6 +57,8 @@ if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" PS1="__VENV_PROMPT__${PS1:-}" export PS1 + VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" + export VIRTUAL_ENV_PROMPT fi # This should detect bash and zsh, which have a hash command that must diff --git a/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat b/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat index f61413e2323..5daa45afc9f 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat +++ b/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat @@ -25,6 +25,7 @@ if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% +set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ :END if defined _OLD_CODEPAGE ( diff --git a/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat b/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat index 313c0791173..44dae495370 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat +++ b/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat @@ -17,5 +17,6 @@ if defined _OLD_VIRTUAL_PATH ( set _OLD_VIRTUAL_PATH= set VIRTUAL_ENV= +set VIRTUAL_ENV_PROMPT= :END diff --git a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh index 68a0dc74e1a..d6f697c55ed 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh +++ b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh @@ -3,7 +3,7 @@ # Created by Davide Di Blasi . # Ported to Python 3.3 venv by Andrew Svetlov -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' # Unset irrelevant variables. deactivate nondestructive @@ -18,6 +18,7 @@ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then set prompt = "__VENV_PROMPT__$prompt" + setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" endif alias pydoc python -m pydoc diff --git a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish index 54b9ea5676b..e40a1d71489 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish +++ b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish @@ -20,6 +20,7 @@ function deactivate -d "Exit virtual environment and return to normal shell env end set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT if test "$argv[1]" != "nondestructive" # Self-destruct! functions -e deactivate @@ -61,4 +62,5 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" end diff --git a/contrib/tools/python3/src/Lib/webbrowser.py b/contrib/tools/python3/src/Lib/webbrowser.py index 6023c1e1384..ec3cece48c9 100755 --- a/contrib/tools/python3/src/Lib/webbrowser.py +++ b/contrib/tools/python3/src/Lib/webbrowser.py @@ -1,5 +1,5 @@ #! /usr/bin/env python3 -"""Interfaces for launching and remotely controlling Web browsers.""" +"""Interfaces for launching and remotely controlling web browsers.""" # Maintained by Georg Brandl. import os @@ -532,6 +532,10 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "serenityos": + # SerenityOS webbrowser, simply called "Browser". + register("Browser", None, BackgroundBrowser("Browser")) + if sys.platform[:3] == "win": # First try to use the default Windows browser register("windows-default", WindowsDefault) diff --git a/contrib/tools/python3/src/Lib/xml/etree/ElementPath.py b/contrib/tools/python3/src/Lib/xml/etree/ElementPath.py index 880ea7bd991..a1170b572fe 100644 --- a/contrib/tools/python3/src/Lib/xml/etree/ElementPath.py +++ b/contrib/tools/python3/src/Lib/xml/etree/ElementPath.py @@ -65,8 +65,9 @@ xpath_tokenizer_re = re.compile( r"//?|" r"\.\.|" r"\(\)|" + r"!=|" r"[/.*:\[\]\(\)@=])|" - r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + r"((?:\{[^}]+\})?[^/\[\]\(\)@!=\s]+)|" r"\s+" ) @@ -253,15 +254,19 @@ def prepare_predicate(next, token): if elem.get(key) is not None: yield elem return select - if signature == "@-='": - # [@attribute='value'] + if signature == "@-='" or signature == "@-!='": + # [@attribute='value'] or [@attribute!='value'] key = predicate[1] value = predicate[-1] def select(context, result): for elem in result: if elem.get(key) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if (attr_value := elem.get(key)) is not None and attr_value != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" and not re.match(r"\-?\d+$", predicate[0]): # [tag] tag = predicate[0] @@ -270,8 +275,10 @@ def prepare_predicate(next, token): if elem.find(tag) is not None: yield elem return select - if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])): - # [.='value'] or [tag='value'] + if signature == ".='" or signature == ".!='" or ( + (signature == "-='" or signature == "-!='") + and not re.match(r"\-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] or [.!='value'] or [tag!='value'] tag = predicate[0] value = predicate[-1] if tag: @@ -281,12 +288,22 @@ def prepare_predicate(next, token): if "".join(e.itertext()) == value: yield elem break + def select_negated(context, result): + for elem in result: + for e in elem.iterfind(tag): + if "".join(e.itertext()) != value: + yield elem + break else: def select(context, result): for elem in result: if "".join(elem.itertext()) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if "".join(elem.itertext()) != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" or signature == "-()" or signature == "-()-": # [index] or [last()] or [last()-index] if signature == "-": diff --git a/contrib/tools/python3/src/Lib/xml/etree/ElementTree.py b/contrib/tools/python3/src/Lib/xml/etree/ElementTree.py index dae2251d859..07be8609b83 100644 --- a/contrib/tools/python3/src/Lib/xml/etree/ElementTree.py +++ b/contrib/tools/python3/src/Lib/xml/etree/ElementTree.py @@ -1562,7 +1562,6 @@ class XMLParser: # Configure pyexpat: buffering, new-style attribute handling. parser.buffer_text = 1 parser.ordered_attributes = 1 - parser.specified_attributes = 1 self._doctype = None self.entity = {} try: @@ -1582,7 +1581,6 @@ class XMLParser: for event_name in events_to_report: if event_name == "start": parser.ordered_attributes = 1 - parser.specified_attributes = 1 def handler(tag, attrib_in, event=event_name, append=append, start=self._start): append((event, start(tag, attrib_in))) diff --git a/contrib/tools/python3/src/Lib/xml/sax/handler.py b/contrib/tools/python3/src/Lib/xml/sax/handler.py index 481733d2cbe..e8d417e5194 100644 --- a/contrib/tools/python3/src/Lib/xml/sax/handler.py +++ b/contrib/tools/python3/src/Lib/xml/sax/handler.py @@ -340,3 +340,48 @@ all_properties = [property_lexical_handler, property_xml_string, property_encoding, property_interning_dict] + + +class LexicalHandler: + """Optional SAX2 handler for lexical events. + + This handler is used to obtain lexical information about an XML + document, that is, information about how the document was encoded + (as opposed to what it contains, which is reported to the + ContentHandler), such as comments and CDATA marked section + boundaries. + + To set the LexicalHandler of an XMLReader, use the setProperty + method with the property identifier + 'http://xml.org/sax/properties/lexical-handler'.""" + + def comment(self, content): + """Reports a comment anywhere in the document (including the + DTD and outside the document element). + + content is a string that holds the contents of the comment.""" + + def startDTD(self, name, public_id, system_id): + """Report the start of the DTD declarations, if the document + has an associated DTD. + + A startEntity event will be reported before declaration events + from the external DTD subset are reported, and this can be + used to infer from which subset DTD declarations derive. + + name is the name of the document element type, public_id the + public identifier of the DTD (or None if none were supplied) + and system_id the system identfier of the external subset (or + None if none were supplied).""" + + def endDTD(self): + """Signals the end of DTD declarations.""" + + def startCDATA(self): + """Reports the beginning of a CDATA marked section. + + The contents of the CDATA marked section will be reported + through the characters event.""" + + def endCDATA(self): + """Reports the end of a CDATA marked section.""" diff --git a/contrib/tools/python3/src/Lib/zipfile.py b/contrib/tools/python3/src/Lib/zipfile.py index 1e942a503e8..34d2fa4b864 100644 --- a/contrib/tools/python3/src/Lib/zipfile.py +++ b/contrib/tools/python3/src/Lib/zipfile.py @@ -16,6 +16,7 @@ import sys import threading import time import contextlib +import pathlib try: import zlib # We may need its compression method @@ -2206,13 +2207,12 @@ class CompleteDirs(ZipFile): if not isinstance(source, ZipFile): return cls(source) - # Only allow for FastPath when supplied zipfile is read-only + # Only allow for FastLookup when supplied zipfile is read-only if 'r' not in source.mode: cls = CompleteDirs - res = cls.__new__(cls) - vars(res).update(vars(source)) - return res + source.__class__ = cls + return source class FastLookup(CompleteDirs): @@ -2220,6 +2220,7 @@ class FastLookup(CompleteDirs): ZipFile subclass to ensure implicit dirs exist and are resolved rapidly. """ + def namelist(self): with contextlib.suppress(AttributeError): return self.__names @@ -2251,7 +2252,7 @@ class Path: >>> zf.writestr('a.txt', 'content of a') >>> zf.writestr('b/c.txt', 'content of c') >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'abcde.zip' + >>> zf.filename = 'mem/abcde.zip' Path accepts the zipfile object itself or a filename @@ -2263,9 +2264,9 @@ class Path: >>> a, b = root.iterdir() >>> a - Path('abcde.zip', 'a.txt') + Path('mem/abcde.zip', 'a.txt') >>> b - Path('abcde.zip', 'b/') + Path('mem/abcde.zip', 'b/') name property: @@ -2276,7 +2277,7 @@ class Path: >>> c = b / 'c.txt' >>> c - Path('abcde.zip', 'b/c.txt') + Path('mem/abcde.zip', 'b/c.txt') >>> c.name 'c.txt' @@ -2294,36 +2295,68 @@ class Path: Coercion to string: - >>> str(c) - 'abcde.zip/b/c.txt' + >>> import os + >>> str(c).replace(os.sep, posixpath.sep) + 'mem/abcde.zip/b/c.txt' + + At the root, ``name``, ``filename``, and ``parent`` + resolve to the zipfile. Note these attributes are not + valid and will raise a ``ValueError`` if the zipfile + has no filename. + + >>> root.name + 'abcde.zip' + >>> str(root.filename).replace(os.sep, posixpath.sep) + 'mem/abcde.zip' + >>> str(root.parent) + 'mem' """ __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" def __init__(self, root, at=""): + """ + Construct a Path from a ZipFile or filename. + + Note: When the source is an existing ZipFile object, + its type (__class__) will be mutated to a + specialized type. If the caller wishes to retain the + original type, the caller should either create a + separate ZipFile object or pass a filename. + """ self.root = FastLookup.make(root) self.at = at - def open(self, mode='r', *args, **kwargs): + def open(self, mode='r', *args, pwd=None, **kwargs): """ Open this entry as text or binary following the semantics of ``pathlib.Path.open()`` by passing arguments through to io.TextIOWrapper(). """ - pwd = kwargs.pop('pwd', None) + if self.is_dir(): + raise IsADirectoryError(self) zip_mode = mode[0] + if not self.exists() and zip_mode == 'r': + raise FileNotFoundError(self) stream = self.root.open(self.at, zip_mode, pwd=pwd) if 'b' in mode: if args or kwargs: raise ValueError("encoding args invalid for binary operation") return stream + else: + kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) return io.TextIOWrapper(stream, *args, **kwargs) @property def name(self): - return posixpath.basename(self.at.rstrip("/")) + return pathlib.Path(self.at).name or self.filename.name + + @property + def filename(self): + return pathlib.Path(self.root.filename).joinpath(self.at) def read_text(self, *args, **kwargs): + kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) with self.open('r', *args, **kwargs) as strm: return strm.read() @@ -2335,13 +2368,13 @@ class Path: return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") def _next(self, at): - return Path(self.root, at) + return self.__class__(self.root, at) def is_dir(self): return not self.at or self.at.endswith("/") def is_file(self): - return not self.is_dir() + return self.exists() and not self.is_dir() def exists(self): return self.at in self.root._name_set() @@ -2358,14 +2391,16 @@ class Path: def __repr__(self): return self.__repr.format(self=self) - def joinpath(self, add): - next = posixpath.join(self.at, add) + def joinpath(self, *other): + next = posixpath.join(self.at, *other) return self._next(self.root.resolve_dir(next)) __truediv__ = joinpath @property def parent(self): + if not self.at: + return self.filename.parent parent_at = posixpath.dirname(self.at.rstrip('/')) if parent_at: parent_at += '/' diff --git a/contrib/tools/python3/src/Lib/zipimport.py b/contrib/tools/python3/src/Lib/zipimport.py index 5ef0a17c2a5..25eaee9c0f2 100644 --- a/contrib/tools/python3/src/Lib/zipimport.py +++ b/contrib/tools/python3/src/Lib/zipimport.py @@ -22,6 +22,7 @@ import _io # for open import marshal # for loads import sys # for modules import time # for mktime +import _warnings # For warn() __all__ = ['ZipImportError', 'zipimporter'] @@ -42,7 +43,7 @@ END_CENTRAL_DIR_SIZE = 22 STRING_END_ARCHIVE = b'PK\x05\x06' MAX_COMMENT_LEN = (1 << 16) - 1 -class zipimporter: +class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object Create a new zipimporter instance. 'archivepath' must be a path to @@ -115,7 +116,12 @@ class zipimporter: full path name if it's possibly a portion of a namespace package, or None otherwise. The optional 'path' argument is ignored -- it's there for compatibility with the importer protocol. + + Deprecated since Python 3.10. Use find_spec() instead. """ + _warnings.warn("zipimporter.find_loader() is deprecated and slated for " + "removal in Python 3.12; use find_spec() instead", + DeprecationWarning) mi = _get_module_info(self, fullname) if mi is not None: # This is a module or package. @@ -146,15 +152,46 @@ class zipimporter: instance itself if the module was found, or None if it wasn't. The optional 'path' argument is ignored -- it's there for compatibility with the importer protocol. + + Deprecated since Python 3.10. Use find_spec() instead. """ + _warnings.warn("zipimporter.find_module() is deprecated and slated for " + "removal in Python 3.12; use find_spec() instead", + DeprecationWarning) return self.find_loader(fullname, path)[0] + def find_spec(self, fullname, target=None): + """Create a ModuleSpec for the specified module. + + Returns None if the module cannot be found. + """ + module_info = _get_module_info(self, fullname) + if module_info is not None: + return _bootstrap.spec_from_loader(fullname, self, is_package=module_info) + else: + # Not a module or regular package. See if this is a directory, and + # therefore possibly a portion of a namespace package. + + # We're only interested in the last path component of fullname + # earlier components are recorded in self.prefix. + modpath = _get_module_path(self, fullname) + if _is_dir(self, modpath): + # This is possibly a portion of a namespace + # package. Return the string representing its path, + # without a trailing separator. + path = f'{self.archive}{path_sep}{modpath}' + spec = _bootstrap.ModuleSpec(name=fullname, loader=None, + is_package=True) + spec.submodule_search_locations.append(path) + return spec + else: + return None def get_code(self, fullname): """get_code(fullname) -> code object. Return the code object for the specified module. Raise ZipImportError - if the module couldn't be found. + if the module couldn't be imported. """ code, ispackage, modpath = _get_module_code(self, fullname) return code @@ -184,7 +221,8 @@ class zipimporter: def get_filename(self, fullname): """get_filename(fullname) -> filename string. - Return the filename for the specified module. + Return the filename for the specified module or raise ZipImportError + if it couldn't be imported. """ # Deciding the filename requires working out where the code # would come from if the module was actually loaded @@ -236,8 +274,13 @@ class zipimporter: Load the module specified by 'fullname'. 'fullname' must be the fully qualified (dotted) module name. It returns the imported - module, or raises ZipImportError if it wasn't found. + module, or raises ZipImportError if it could not be imported. + + Deprecated since Python 3.10. Use exec_module() instead. """ + msg = ("zipimport.zipimporter.load_module() is deprecated and slated for " + "removal in Python 3.12; use exec_module() instead") + _warnings.warn(msg, DeprecationWarning) code, ispackage, modpath = _get_module_code(self, fullname) mod = sys.modules.get(fullname) if mod is None or not isinstance(mod, _module_type): @@ -280,11 +323,18 @@ class zipimporter: return None except ZipImportError: return None - if not _ZipImportResourceReader._registered: - from importlib.abc import ResourceReader - ResourceReader.register(_ZipImportResourceReader) - _ZipImportResourceReader._registered = True - return _ZipImportResourceReader(self, fullname) + from importlib.readers import ZipReader + return ZipReader(self, fullname) + + + def invalidate_caches(self): + """Reload the file data of the archive path.""" + try: + self._files = _read_directory(self.archive) + _zip_directory_cache[self.archive] = self._files + except ZipImportError: + _zip_directory_cache.pop(self.archive, None) + self._files = {} def __repr__(self): @@ -580,20 +630,15 @@ def _eq_mtime(t1, t2): # Given the contents of a .py[co] file, unmarshal the data -# and return the code object. Return None if it the magic word doesn't -# match, or if the recorded .py[co] metadata does not match the source, -# (we do this instead of raising an exception as we fall back -# to .py if available and we don't want to mask other errors). +# and return the code object. Raises ImportError it the magic word doesn't +# match, or if the recorded .py[co] metadata does not match the source. def _unmarshal_code(self, pathname, fullpath, fullname, data): exc_details = { 'name': fullname, 'path': fullpath, } - try: - flags = _bootstrap_external._classify_pyc(data, fullname, exc_details) - except ImportError: - return None + flags = _bootstrap_external._classify_pyc(data, fullname, exc_details) hash_based = flags & 0b1 != 0 if hash_based: @@ -607,11 +652,8 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes, ) - try: - _bootstrap_external._validate_hash_pyc( - data, source_hash, fullname, exc_details) - except ImportError: - return None + _bootstrap_external._validate_hash_pyc( + data, source_hash, fullname, exc_details) else: source_mtime, source_size = \ _get_mtime_and_size_of_source(self, fullpath) @@ -697,6 +739,7 @@ def _get_pyc_source(self, path): # 'fullname'. def _get_module_code(self, fullname): path = _get_module_path(self, fullname) + import_error = None for suffix, isbytecode, ispackage in _zip_searchorder: fullpath = path + suffix _bootstrap._verbose_message('trying {}{}{}', self.archive, path_sep, fullpath, verbosity=2) @@ -707,8 +750,12 @@ def _get_module_code(self, fullname): else: modpath = toc_entry[0] data = _get_data(self.archive, toc_entry) + code = None if isbytecode: - code = _unmarshal_code(self, modpath, fullpath, fullname, data) + try: + code = _unmarshal_code(self, modpath, fullpath, fullname, data) + except ImportError as exc: + import_error = exc else: code = _compile_source(modpath, data) if code is None: @@ -718,75 +765,8 @@ def _get_module_code(self, fullname): modpath = toc_entry[0] return code, ispackage, modpath else: - raise ZipImportError(f"can't find module {fullname!r}", name=fullname) - - -class _ZipImportResourceReader: - """Private class used to support ZipImport.get_resource_reader(). - - This class is allowed to reference all the innards and private parts of - the zipimporter. - """ - _registered = False - - def __init__(self, zipimporter, fullname): - self.zipimporter = zipimporter - self.fullname = fullname - - def open_resource(self, resource): - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{resource}' - from io import BytesIO - try: - return BytesIO(self.zipimporter.get_data(path)) - except OSError: - raise FileNotFoundError(path) - - def resource_path(self, resource): - # All resources are in the zip file, so there is no path to the file. - # Raising FileNotFoundError tells the higher level API to extract the - # binary data and create a temporary file. - raise FileNotFoundError - - def is_resource(self, name): - # Maybe we could do better, but if we can get the data, it's a - # resource. Otherwise it isn't. - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{name}' - try: - self.zipimporter.get_data(path) - except OSError: - return False - return True - - def contents(self): - # This is a bit convoluted, because fullname will be a module path, - # but _files is a list of file names relative to the top of the - # archive's namespace. We want to compare file paths to find all the - # names of things inside the module represented by fullname. So we - # turn the module path of fullname into a file path relative to the - # top of the archive, and then we iterate through _files looking for - # names inside that "directory". - from pathlib import Path - fullname_path = Path(self.zipimporter.get_filename(self.fullname)) - relative_path = fullname_path.relative_to(self.zipimporter.archive) - # Don't forget that fullname names a package, so its path will include - # __init__.py, which we want to ignore. - assert relative_path.name == '__init__.py' - package_path = relative_path.parent - subdirs_seen = set() - for filename in self.zipimporter._files: - try: - relative = Path(filename).relative_to(package_path) - except ValueError: - continue - # If the path of the file (which is relative to the top of the zip - # namespace), relative to the package given when the resource - # reader was created, has a parent, then it's a name in a - # subdirectory and thus we skip it. - parent_name = relative.parent.name - if len(parent_name) == 0: - yield relative.name - elif parent_name not in subdirs_seen: - subdirs_seen.add(parent_name) - yield parent_name + if import_error: + msg = f"module load failed: {import_error}" + raise ZipImportError(msg, name=fullname) from import_error + else: + raise ZipImportError(f"can't find module {fullname!r}", name=fullname) diff --git a/contrib/tools/python3/src/Lib/zoneinfo/_common.py b/contrib/tools/python3/src/Lib/zoneinfo/_common.py index 41c898f37e4..4c24f01bd7b 100644 --- a/contrib/tools/python3/src/Lib/zoneinfo/_common.py +++ b/contrib/tools/python3/src/Lib/zoneinfo/_common.py @@ -136,8 +136,7 @@ class _TZifHeader: ] def __init__(self, *args): - assert len(self.__slots__) == len(args) - for attr, val in zip(self.__slots__, args): + for attr, val in zip(self.__slots__, args, strict=True): setattr(self, attr, val) @classmethod -- cgit v1.3