Update Python from 3.11.8 to 3.12.2

1 files changed, 323 insertions, 137 deletions

diff --git a/contrib/tools/python3/src/Lib/importlib/_bootstrap.py b/contrib/tools/python3/src/Lib/importlib/_bootstrap.py
index ee93ebc396..d942045f3d 100644
--- a/contrib/tools/python3/src/Lib/importlib/_bootstrap.py
+++ b/contrib/tools/python3/src/Lib/importlib/_bootstrap.py
@@ -51,17 +51,178 @@ def _new_module(name):
 
 # Module-level locking ########################################################
 
-# A dict mapping module names to weakrefs of _ModuleLock instances
-# Dictionary protected by the global import lock
+# For a list that can have a weakref to it.
+class _List(list):
+    pass
+
+
+# Copied from weakref.py with some simplifications and modifications unique to
+# bootstrapping importlib. Many methods were simply deleting for simplicity, so if they
+# are needed in the future they may work if simply copied back in.
+class _WeakValueDictionary:
+
+    def __init__(self):
+        self_weakref = _weakref.ref(self)
+
+        # Inlined to avoid issues with inheriting from _weakref.ref before _weakref is
+        # set by _setup(). Since there's only one instance of this class, this is
+        # not expensive.
+        class KeyedRef(_weakref.ref):
+
+            __slots__ = "key",
+
+            def __new__(type, ob, key):
+                self = super().__new__(type, ob, type.remove)
+                self.key = key
+                return self
+
+            def __init__(self, ob, key):
+                super().__init__(ob, self.remove)
+
+            @staticmethod
+            def remove(wr):
+                nonlocal self_weakref
+
+                self = self_weakref()
+                if self is not None:
+                    if self._iterating:
+                        self._pending_removals.append(wr.key)
+                    else:
+                        _weakref._remove_dead_weakref(self.data, wr.key)
+
+        self._KeyedRef = KeyedRef
+        self.clear()
+
+    def clear(self):
+        self._pending_removals = []
+        self._iterating = set()
+        self.data = {}
+
+    def _commit_removals(self):
+        pop = self._pending_removals.pop
+        d = self.data
+        while True:
+            try:
+                key = pop()
+            except IndexError:
+                return
+            _weakref._remove_dead_weakref(d, key)
+
+    def get(self, key, default=None):
+        if self._pending_removals:
+            self._commit_removals()
+        try:
+            wr = self.data[key]
+        except KeyError:
+            return default
+        else:
+            if (o := wr()) is None:
+                return default
+            else:
+                return o
+
+    def setdefault(self, key, default=None):
+        try:
+            o = self.data[key]()
+        except KeyError:
+            o = None
+        if o is None:
+            if self._pending_removals:
+                self._commit_removals()
+            self.data[key] = self._KeyedRef(default, key)
+            return default
+        else:
+            return o
+
+
+# A dict mapping module names to weakrefs of _ModuleLock instances.
+# Dictionary protected by the global import lock.
 _module_locks = {}
-# A dict mapping thread ids to _ModuleLock instances
-_blocking_on = {}
+
+# A dict mapping thread IDs to weakref'ed lists of _ModuleLock instances.
+# This maps a thread to the module locks it is blocking on acquiring.  The
+# values are lists because a single thread could perform a re-entrant import
+# and be "in the process" of blocking on locks for more than one module.  A
+# thread can be "in the process" because a thread cannot actually block on
+# acquiring more than one lock but it can have set up bookkeeping that reflects
+# that it intends to block on acquiring more than one lock.
+#
+# The dictionary uses a WeakValueDictionary to avoid keeping unnecessary
+# lists around, regardless of GC runs. This way there's no memory leak if
+# the list is no longer needed (GH-106176).
+_blocking_on = None
+
+
+class _BlockingOnManager:
+    """A context manager responsible to updating ``_blocking_on``."""
+    def __init__(self, thread_id, lock):
+        self.thread_id = thread_id
+        self.lock = lock
+
+    def __enter__(self):
+        """Mark the running thread as waiting for self.lock. via _blocking_on."""
+        # Interactions with _blocking_on are *not* protected by the global
+        # import lock here because each thread only touches the state that it
+        # owns (state keyed on its thread id).  The global import lock is
+        # re-entrant (i.e., a single thread may take it more than once) so it
+        # wouldn't help us be correct in the face of re-entrancy either.
+
+        self.blocked_on = _blocking_on.setdefault(self.thread_id, _List())
+        self.blocked_on.append(self.lock)
+
+    def __exit__(self, *args, **kwargs):
+        """Remove self.lock from this thread's _blocking_on list."""
+        self.blocked_on.remove(self.lock)
 
 
 class _DeadlockError(RuntimeError):
     pass
 
 
+
+def _has_deadlocked(target_id, *, seen_ids, candidate_ids, blocking_on):
+    """Check if 'target_id' is holding the same lock as another thread(s).
+
+    The search within 'blocking_on' starts with the threads listed in
+    'candidate_ids'.  'seen_ids' contains any threads that are considered
+    already traversed in the search.
+
+    Keyword arguments:
+    target_id     -- The thread id to try to reach.
+    seen_ids      -- A set of threads that have already been visited.
+    candidate_ids -- The thread ids from which to begin.
+    blocking_on   -- A dict representing the thread/blocking-on graph.  This may
+                     be the same object as the global '_blocking_on' but it is
+                     a parameter to reduce the impact that global mutable
+                     state has on the result of this function.
+    """
+    if target_id in candidate_ids:
+        # If we have already reached the target_id, we're done - signal that it
+        # is reachable.
+        return True
+
+    # Otherwise, try to reach the target_id from each of the given candidate_ids.
+    for tid in candidate_ids:
+        if not (candidate_blocking_on := blocking_on.get(tid)):
+            # There are no edges out from this node, skip it.
+            continue
+        elif tid in seen_ids:
+            # bpo 38091: the chain of tid's we encounter here eventually leads
+            # to a fixed point or a cycle, but does not reach target_id.
+            # This means we would not actually deadlock.  This can happen if
+            # other threads are at the beginning of acquire() below.
+            return False
+        seen_ids.add(tid)
+
+        # Follow the edges out from this thread.
+        edges = [lock.owner for lock in candidate_blocking_on]
+        if _has_deadlocked(target_id, seen_ids=seen_ids, candidate_ids=edges,
+                blocking_on=blocking_on):
+            return True
+
+    return False
+
+
 class _ModuleLock:
     """A recursive lock implementation which is able to detect deadlocks
     (e.g. thread 1 trying to take locks A then B, and thread 2 trying to
@@ -69,33 +230,76 @@ class _ModuleLock:
     """
 
     def __init__(self, name):
-        self.lock = _thread.allocate_lock()
+        # Create an RLock for protecting the import process for the
+        # corresponding module.  Since it is an RLock, a single thread will be
+        # able to take it more than once.  This is necessary to support
+        # re-entrancy in the import system that arises from (at least) signal
+        # handlers and the garbage collector.  Consider the case of:
+        #
+        #  import foo
+        #  -> ...
+        #     -> importlib._bootstrap._ModuleLock.acquire
+        #        -> ...
+        #           -> <garbage collector>
+        #              -> __del__
+        #                 -> import foo
+        #                    -> ...
+        #                       -> importlib._bootstrap._ModuleLock.acquire
+        #                          -> _BlockingOnManager.__enter__
+        #
+        # If a different thread than the running one holds the lock then the
+        # thread will have to block on taking the lock, which is what we want
+        # for thread safety.
+        self.lock = _thread.RLock()
         self.wakeup = _thread.allocate_lock()
+
+        # The name of the module for which this is a lock.
         self.name = name
+
+        # Can end up being set to None if this lock is not owned by any thread
+        # or the thread identifier for the owning thread.
         self.owner = None
-        self.count = 0
-        self.waiters = 0
+
+        # Represent the number of times the owning thread has acquired this lock
+        # via a list of True.  This supports RLock-like ("re-entrant lock")
+        # behavior, necessary in case a single thread is following a circular
+        # import dependency and needs to take the lock for a single module
+        # more than once.
+        #
+        # Counts are represented as a list of True because list.append(True)
+        # and list.pop() are both atomic and thread-safe in CPython and it's hard
+        # to find another primitive with the same properties.
+        self.count = []
+
+        # This is a count of the number of threads that are blocking on
+        # self.wakeup.acquire() awaiting to get their turn holding this module
+        # lock.  When the module lock is released, if this is greater than
+        # zero, it is decremented and `self.wakeup` is released one time.  The
+        # intent is that this will let one other thread make more progress on
+        # acquiring this module lock.  This repeats until all the threads have
+        # gotten a turn.
+        #
+        # This is incremented in self.acquire() when a thread notices it is
+        # going to have to wait for another thread to finish.
+        #
+        # See the comment above count for explanation of the representation.
+        self.waiters = []
 
     def has_deadlock(self):
-        # Deadlock avoidance for concurrent circular imports.
-        me = _thread.get_ident()
-        tid = self.owner
-        seen = set()
-        while True:
-            lock = _blocking_on.get(tid)
-            if lock is None:
-                return False
-            tid = lock.owner
-            if tid == me:
-                return True
-            if tid in seen:
-                # bpo 38091: the chain of tid's we encounter here
-                # eventually leads to a fixpoint or a cycle, but
-                # does not reach 'me'.  This means we would not
-                # actually deadlock.  This can happen if other
-                # threads are at the beginning of acquire() below.
-                return False
-            seen.add(tid)
+        # To avoid deadlocks for concurrent or re-entrant circular imports,
+        # look at _blocking_on to see if any threads are blocking
+        # on getting the import lock for any module for which the import lock
+        # is held by this thread.
+        return _has_deadlocked(
+            # Try to find this thread.
+            target_id=_thread.get_ident(),
+            seen_ids=set(),
+            # Start from the thread that holds the import lock for this
+            # module.
+            candidate_ids=[self.owner],
+            # Use the global "blocking on" state.
+            blocking_on=_blocking_on,
+        )
 
     def acquire(self):
         """
@@ -104,39 +308,82 @@ class _ModuleLock:
         Otherwise, the lock is always acquired and True is returned.
         """
         tid = _thread.get_ident()
-        _blocking_on[tid] = self
-        try:
+        with _BlockingOnManager(tid, self):
             while True:
+                # Protect interaction with state on self with a per-module
+                # lock.  This makes it safe for more than one thread to try to
+                # acquire the lock for a single module at the same time.
                 with self.lock:
-                    if self.count == 0 or self.owner == tid:
+                    if self.count == [] or self.owner == tid:
+                        # If the lock for this module is unowned then we can
+                        # take the lock immediately and succeed.  If the lock
+                        # for this module is owned by the running thread then
+                        # we can also allow the acquire to succeed.  This
+                        # supports circular imports (thread T imports module A
+                        # which imports module B which imports module A).
                         self.owner = tid
-                        self.count += 1
+                        self.count.append(True)
                         return True
+
+                    # At this point we know the lock is held (because count !=
+                    # 0) by another thread (because owner != tid).  We'll have
+                    # to get in line to take the module lock.
+
+                    # But first, check to see if this thread would create a
+                    # deadlock by acquiring this module lock.  If it would
+                    # then just stop with an error.
+                    #
+                    # It's not clear who is expected to handle this error.
+                    # There is one handler in _lock_unlock_module but many
+                    # times this method is called when entering the context
+                    # manager _ModuleLockManager instead - so _DeadlockError
+                    # will just propagate up to application code.
+                    #
+                    # This seems to be more than just a hypothetical -
+                    # https://stackoverflow.com/questions/59509154
+                    # https://github.com/encode/django-rest-framework/issues/7078
                     if self.has_deadlock():
-                        raise _DeadlockError('deadlock detected by %r' % self)
+                        raise _DeadlockError(f'deadlock detected by {self!r}')
+
+                    # Check to see if we're going to be able to acquire the
+                    # lock.  If we are going to have to wait then increment
+                    # the waiters so `self.release` will know to unblock us
+                    # later on.  We do this part non-blockingly so we don't
+                    # get stuck here before we increment waiters.  We have
+                    # this extra acquire call (in addition to the one below,
+                    # outside the self.lock context manager) to make sure
+                    # self.wakeup is held when the next acquire is called (so
+                    # we block).  This is probably needlessly complex and we
+                    # should just take self.wakeup in the return codepath
+                    # above.
                     if self.wakeup.acquire(False):
-                        self.waiters += 1
-                # Wait for a release() call
+                        self.waiters.append(None)
+
+                # Now take the lock in a blocking fashion.  This won't
+                # complete until the thread holding this lock
+                # (self.owner) calls self.release.
                 self.wakeup.acquire()
+
+                # Taking the lock has served its purpose (making us wait), so we can
+                # give it up now.  We'll take it w/o blocking again on the
+                # next iteration around this 'while' loop.
                 self.wakeup.release()
-        finally:
-            del _blocking_on[tid]
 
     def release(self):
         tid = _thread.get_ident()
         with self.lock:
             if self.owner != tid:
                 raise RuntimeError('cannot release un-acquired lock')
-            assert self.count > 0
-            self.count -= 1
-            if self.count == 0:
+            assert len(self.count) > 0
+            self.count.pop()
+            if not len(self.count):
                 self.owner = None
-                if self.waiters:
-                    self.waiters -= 1
+                if len(self.waiters) > 0:
+                    self.waiters.pop()
                     self.wakeup.release()
 
     def __repr__(self):
-        return '_ModuleLock({!r}) at {}'.format(self.name, id(self))
+        return f'_ModuleLock({self.name!r}) at {id(self)}'
 
 
 class _DummyModuleLock:
@@ -157,7 +404,7 @@ class _DummyModuleLock:
         self.count -= 1
 
     def __repr__(self):
-        return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self))
+        return f'_DummyModuleLock({self.name!r}) at {id(self)}'
 
 
 class _ModuleLockManager:
@@ -253,7 +500,7 @@ def _requires_builtin(fxn):
     """Decorator to verify the named module is built-in."""
     def _requires_builtin_wrapper(self, fullname):
         if fullname not in sys.builtin_module_names:
-            raise ImportError('{!r} is not a built-in module'.format(fullname),
+            raise ImportError(f'{fullname!r} is not a built-in module',
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_builtin_wrapper, fxn)
@@ -264,7 +511,7 @@ def _requires_frozen(fxn):
     """Decorator to verify the named module is frozen."""
     def _requires_frozen_wrapper(self, fullname):
         if not _imp.is_frozen(fullname):
-            raise ImportError('{!r} is not a frozen module'.format(fullname),
+            raise ImportError(f'{fullname!r} is not a frozen module',
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_frozen_wrapper, fxn)
@@ -296,11 +543,6 @@ def _module_repr(module):
     loader = getattr(module, '__loader__', None)
     if spec := getattr(module, "__spec__", None):
         return _module_repr_from_spec(spec)
-    elif hasattr(loader, 'module_repr'):
-        try:
-            return loader.module_repr(module)
-        except Exception:
-            pass
     # Fall through to a catch-all which always succeeds.
     try:
         name = module.__name__
@@ -310,11 +552,11 @@ def _module_repr(module):
         filename = module.__file__
     except AttributeError:
         if loader is None:
-            return '<module {!r}>'.format(name)
+            return f'<module {name!r}>'
         else:
-            return '<module {!r} ({!r})>'.format(name, loader)
+            return f'<module {name!r} ({loader!r})>'
     else:
-        return '<module {!r} from {!r}>'.format(name, filename)
+        return f'<module {name!r} from {filename!r}>'
 
 
 class ModuleSpec:
@@ -368,14 +610,12 @@ class ModuleSpec:
         self._cached = None
 
     def __repr__(self):
-        args = ['name={!r}'.format(self.name),
-                'loader={!r}'.format(self.loader)]
+        args = [f'name={self.name!r}', f'loader={self.loader!r}']
         if self.origin is not None:
-            args.append('origin={!r}'.format(self.origin))
+            args.append(f'origin={self.origin!r}')
         if self.submodule_search_locations is not None:
-            args.append('submodule_search_locations={}'
-                        .format(self.submodule_search_locations))
-        return '{}({})'.format(self.__class__.__name__, ', '.join(args))
+            args.append(f'submodule_search_locations={self.submodule_search_locations}')
+        return f'{self.__class__.__name__}({", ".join(args)})'
 
     def __eq__(self, other):
         smsl = self.submodule_search_locations
@@ -582,18 +822,23 @@ def module_from_spec(spec):
 
 def _module_repr_from_spec(spec):
     """Return the repr to use for the module."""
-    # We mostly replicate _module_repr() using the spec attributes.
     name = '?' if spec.name is None else spec.name
     if spec.origin is None:
-        if spec.loader is None:
-            return '<module {!r}>'.format(name)
+        loader = spec.loader
+        if loader is None:
+            return f'<module {name!r}>'
+        elif (
+            _bootstrap_external is not None
+            and isinstance(loader, _bootstrap_external.NamespaceLoader)
+        ):
+            return f'<module {name!r} (namespace) from {list(loader._path)}>'
         else:
-            return '<module {!r} ({!r})>'.format(name, spec.loader)
+            return f'<module {name!r} ({loader!r})>'
     else:
         if spec.has_location:
-            return '<module {!r} from {!r}>'.format(name, spec.origin)
+            return f'<module {name!r} from {spec.origin!r}>'
         else:
-            return '<module {!r} ({})>'.format(spec.name, spec.origin)
+            return f'<module {spec.name!r} ({spec.origin})>'
 
 
 # Used by importlib.reload() and _load_module_shim().
@@ -602,7 +847,7 @@ def _exec(spec, module):
     name = spec.name
     with _ModuleLockManager(name):
         if sys.modules.get(name) is not module:
-            msg = 'module {!r} not in sys.modules'.format(name)
+            msg = f'module {name!r} not in sys.modules'
             raise ImportError(msg, name=name)
         try:
             if spec.loader is None:
@@ -734,17 +979,6 @@ class BuiltinImporter:
 
     _ORIGIN = "built-in"
 
-    @staticmethod
-    def module_repr(module):
-        """Return repr for the module.
-
-        The method is deprecated.  The import machinery does the job itself.
-
-        """
-        _warnings.warn("BuiltinImporter.module_repr() is deprecated and "
-                       "slated for removal in Python 3.12", DeprecationWarning)
-        return f'<module {module.__name__!r} ({BuiltinImporter._ORIGIN})>'
-
     @classmethod
     def find_spec(cls, fullname, path=None, target=None):
         if _imp.is_builtin(fullname):
@@ -752,26 +986,11 @@ class BuiltinImporter:
         else:
             return None
 
-    @classmethod
-    def find_module(cls, fullname, path=None):
-        """Find the built-in module.
-
-        If 'path' is ever specified then the search is considered a failure.
-
-        This method is deprecated.  Use find_spec() instead.
-
-        """
-        _warnings.warn("BuiltinImporter.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
-        spec = cls.find_spec(fullname, path)
-        return spec.loader if spec is not None else None
-
     @staticmethod
     def create_module(spec):
         """Create a built-in module"""
         if spec.name not in sys.builtin_module_names:
-            raise ImportError('{!r} is not a built-in module'.format(spec.name),
+            raise ImportError(f'{spec.name!r} is not a built-in module',
                               name=spec.name)
         return _call_with_frames_removed(_imp.create_builtin, spec)
 
@@ -812,17 +1031,6 @@ class FrozenImporter:
 
     _ORIGIN = "frozen"
 
-    @staticmethod
-    def module_repr(m):
-        """Return repr for the module.
-
-        The method is deprecated.  The import machinery does the job itself.
-
-        """
-        _warnings.warn("FrozenImporter.module_repr() is deprecated and "
-                       "slated for removal in Python 3.12", DeprecationWarning)
-        return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
-
     @classmethod
     def _fix_up_module(cls, module):
         spec = module.__spec__
@@ -947,18 +1155,6 @@ class FrozenImporter:
             spec.submodule_search_locations.insert(0, pkgdir)
         return spec
 
-    @classmethod
-    def find_module(cls, fullname, path=None):
-        """Find a frozen module.
-
-        This method is deprecated.  Use find_spec() instead.
-
-        """
-        _warnings.warn("FrozenImporter.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
-        return cls if _imp.is_frozen(fullname) else None
-
     @staticmethod
     def create_module(spec):
         """Set __file__, if able."""
@@ -1038,17 +1234,7 @@ def _resolve_name(name, package, level):
     if len(bits) < level:
         raise ImportError('attempted relative import beyond top-level package')
     base = bits[0]
-    return '{}.{}'.format(base, name) if name else base
-
-
-def _find_spec_legacy(finder, name, path):
-    msg = (f"{_object_name(finder)}.find_spec() not found; "
-                           "falling back to find_module()")
-    _warnings.warn(msg, ImportWarning)
-    loader = finder.find_module(name, path)
-    if loader is None:
-        return None
-    return spec_from_loader(name, loader)
+    return f'{base}.{name}' if name else base
 
 
 def _find_spec(name, path, target=None):
@@ -1071,9 +1257,7 @@ def _find_spec(name, path, target=None):
             try:
                 find_spec = finder.find_spec
             except AttributeError:
-                spec = _find_spec_legacy(finder, name, path)
-                if spec is None:
-                    continue
+                continue
             else:
                 spec = find_spec(name, path, target)
         if spec is not None:
@@ -1101,7 +1285,7 @@ def _find_spec(name, path, target=None):
 def _sanity_check(name, package, level):
     """Verify arguments are "sane"."""
     if not isinstance(name, str):
-        raise TypeError('module name must be str, not {}'.format(type(name)))
+        raise TypeError(f'module name must be str, not {type(name)}')
     if level < 0:
         raise ValueError('level must be >= 0')
     if level > 0:
@@ -1131,13 +1315,13 @@ def _find_and_load_unlocked(name, import_):
         try:
             path = parent_module.__path__
         except AttributeError:
-            msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
+            msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package'
             raise ModuleNotFoundError(msg, name=name) from None
         parent_spec = parent_module.__spec__
         child = name.rpartition('.')[2]
     spec = _find_spec(name, path)
     if spec is None:
-        raise ModuleNotFoundError(_ERR_MSG.format(name), name=name)
+        raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name)
     else:
         if parent_spec:
             # Temporarily add child we are currently importing to parent's
@@ -1182,8 +1366,7 @@ def _find_and_load(name, import_):
         _lock_unlock_module(name)
 
     if module is None:
-        message = ('import of {} halted; '
-                   'None in sys.modules'.format(name))
+        message = f'import of {name} halted; None in sys.modules'
         raise ModuleNotFoundError(message, name=name)
 
     return module
@@ -1227,7 +1410,7 @@ def _handle_fromlist(module, fromlist, import_, *, recursive=False):
                 _handle_fromlist(module, module.__all__, import_,
                                  recursive=True)
         elif not hasattr(module, x):
-            from_name = '{}.{}'.format(module.__name__, x)
+            from_name = f'{module.__name__}.{x}'
             try:
                 _call_with_frames_removed(import_, from_name)
             except ModuleNotFoundError as exc:
@@ -1254,7 +1437,7 @@ def _calc___package__(globals):
         if spec is not None and package != spec.parent:
             _warnings.warn("__package__ != __spec__.parent "
                            f"({package!r} != {spec.parent!r})",
-                           ImportWarning, stacklevel=3)
+                           DeprecationWarning, stacklevel=3)
         return package
     elif spec is not None:
         return spec.parent
@@ -1320,7 +1503,7 @@ def _setup(sys_module, _imp_module):
     modules, those two modules must be explicitly passed in.
 
     """
-    global _imp, sys
+    global _imp, sys, _blocking_on
     _imp = _imp_module
     sys = sys_module
 
@@ -1348,6 +1531,9 @@ def _setup(sys_module, _imp_module):
             builtin_module = sys.modules[builtin_name]
         setattr(self_module, builtin_name, builtin_module)
 
+    # Instantiation requires _weakref to have been set.
+    _blocking_on = _WeakValueDictionary()
+
 
 def _install(sys_module, _imp_module):
     """Install importers for builtin and frozen modules"""