diff options
| author | AlexSm <alex@ydb.tech> | 2024-03-05 10:40:59 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-03-05 12:40:59 +0300 |
| commit | 1ac13c847b5358faba44dbb638a828e24369467b (patch) | |
| tree | 07672b4dd3604ad3dee540a02c6494cb7d10dc3d /contrib/tools/python3/Lib/importlib | |
| parent | ffcca3e7f7958ddc6487b91d3df8c01054bd0638 (diff) | |
| download | ydb-1ac13c847b5358faba44dbb638a828e24369467b.tar.gz | |
Library import 16 (#2433)
Co-authored-by: robot-piglet <robot-piglet@yandex-team.com>
Co-authored-by: deshevoy <deshevoy@yandex-team.com>
Co-authored-by: robot-contrib <robot-contrib@yandex-team.com>
Co-authored-by: thegeorg <thegeorg@yandex-team.com>
Co-authored-by: robot-ya-builder <robot-ya-builder@yandex-team.com>
Co-authored-by: svidyuk <svidyuk@yandex-team.com>
Co-authored-by: shadchin <shadchin@yandex-team.com>
Co-authored-by: robot-ratatosk <robot-ratatosk@yandex-team.com>
Co-authored-by: innokentii <innokentii@yandex-team.com>
Co-authored-by: arkady-e1ppa <arkady-e1ppa@yandex-team.com>
Co-authored-by: snermolaev <snermolaev@yandex-team.com>
Co-authored-by: dimdim11 <dimdim11@yandex-team.com>
Co-authored-by: kickbutt <kickbutt@yandex-team.com>
Co-authored-by: abdullinsaid <abdullinsaid@yandex-team.com>
Co-authored-by: korsunandrei <korsunandrei@yandex-team.com>
Co-authored-by: petrk <petrk@yandex-team.com>
Co-authored-by: miroslav2 <miroslav2@yandex-team.com>
Co-authored-by: serjflint <serjflint@yandex-team.com>
Co-authored-by: akhropov <akhropov@yandex-team.com>
Co-authored-by: prettyboy <prettyboy@yandex-team.com>
Co-authored-by: ilikepugs <ilikepugs@yandex-team.com>
Co-authored-by: hiddenpath <hiddenpath@yandex-team.com>
Co-authored-by: mikhnenko <mikhnenko@yandex-team.com>
Co-authored-by: spreis <spreis@yandex-team.com>
Co-authored-by: andreyshspb <andreyshspb@yandex-team.com>
Co-authored-by: dimaandreev <dimaandreev@yandex-team.com>
Co-authored-by: rashid <rashid@yandex-team.com>
Co-authored-by: robot-ydb-importer <robot-ydb-importer@yandex-team.com>
Co-authored-by: r-vetrov <r-vetrov@yandex-team.com>
Co-authored-by: ypodlesov <ypodlesov@yandex-team.com>
Co-authored-by: zaverden <zaverden@yandex-team.com>
Co-authored-by: vpozdyayev <vpozdyayev@yandex-team.com>
Co-authored-by: robot-cozmo <robot-cozmo@yandex-team.com>
Co-authored-by: v-korovin <v-korovin@yandex-team.com>
Co-authored-by: arikon <arikon@yandex-team.com>
Co-authored-by: khoden <khoden@yandex-team.com>
Co-authored-by: psydmm <psydmm@yandex-team.com>
Co-authored-by: robot-javacom <robot-javacom@yandex-team.com>
Co-authored-by: dtorilov <dtorilov@yandex-team.com>
Co-authored-by: sennikovmv <sennikovmv@yandex-team.com>
Co-authored-by: hcpp <hcpp@ydb.tech>
Diffstat (limited to 'contrib/tools/python3/Lib/importlib')
24 files changed, 6418 insertions, 0 deletions
diff --git a/contrib/tools/python3/Lib/importlib/__init__.py b/contrib/tools/python3/Lib/importlib/__init__.py new file mode 100644 index 0000000000..707c081cb2 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/__init__.py @@ -0,0 +1,138 @@ +"""A pure Python implementation of import.""" +__all__ = ['__import__', 'import_module', 'invalidate_caches', 'reload'] + +# Bootstrap help ##################################################### + +# Until bootstrapping is complete, DO NOT import any modules that attempt +# to import importlib._bootstrap (directly or indirectly). Since this +# partially initialised package would be present in sys.modules, those +# modules would get an uninitialised copy of the source version, instead +# of a fully initialised version (either the frozen one or the one +# initialised below if the frozen one is not available). +import _imp # Just the builtin component, NOT the full Python module +import sys + +try: + import _frozen_importlib as _bootstrap +except ImportError: + from . import _bootstrap + _bootstrap._setup(sys, _imp) +else: + # importlib._bootstrap is the built-in import, ensure we don't create + # a second copy of the module. + _bootstrap.__name__ = 'importlib._bootstrap' + _bootstrap.__package__ = 'importlib' + try: + _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py') + except NameError: + # __file__ is not guaranteed to be defined, e.g. if this code gets + # frozen by a tool like cx_Freeze. + pass + sys.modules['importlib._bootstrap'] = _bootstrap + +try: + import _frozen_importlib_external as _bootstrap_external +except ImportError: + from . import _bootstrap_external + _bootstrap_external._set_bootstrap_module(_bootstrap) + _bootstrap._bootstrap_external = _bootstrap_external +else: + _bootstrap_external.__name__ = 'importlib._bootstrap_external' + _bootstrap_external.__package__ = 'importlib' + try: + _bootstrap_external.__file__ = __file__.replace('__init__.py', '_bootstrap_external.py') + except NameError: + # __file__ is not guaranteed to be defined, e.g. if this code gets + # frozen by a tool like cx_Freeze. + pass + sys.modules['importlib._bootstrap_external'] = _bootstrap_external + +# To simplify imports in test code +_pack_uint32 = _bootstrap_external._pack_uint32 +_unpack_uint32 = _bootstrap_external._unpack_uint32 + +# Fully bootstrapped at this point, import whatever you like, circular +# dependencies and startup overhead minimisation permitting :) + +import warnings + + +# Public API ######################################################### + +from ._bootstrap import __import__ + + +def invalidate_caches(): + """Call the invalidate_caches() method on all meta path finders stored in + sys.meta_path (where implemented).""" + for finder in sys.meta_path: + if hasattr(finder, 'invalidate_caches'): + finder.invalidate_caches() + + +def import_module(name, package=None): + """Import a module. + + The 'package' argument is required when performing a relative import. It + specifies the package to use as the anchor point from which to resolve the + relative import to an absolute import. + + """ + level = 0 + if name.startswith('.'): + if not package: + raise TypeError("the 'package' argument is required to perform a " + f"relative import for {name!r}") + for character in name: + if character != '.': + break + level += 1 + return _bootstrap._gcd_import(name[level:], package, level) + + +_RELOADING = {} + + +def reload(module): + """Reload the module and return it. + + The module must have been successfully imported before. + + """ + try: + name = module.__spec__.name + except AttributeError: + try: + name = module.__name__ + except AttributeError: + raise TypeError("reload() argument must be a module") + + if sys.modules.get(name) is not module: + raise ImportError(f"module {name} not in sys.modules", name=name) + if name in _RELOADING: + return _RELOADING[name] + _RELOADING[name] = module + try: + parent_name = name.rpartition('.')[0] + if parent_name: + try: + parent = sys.modules[parent_name] + except KeyError: + raise ImportError(f"parent {parent_name!r} not in sys.modules", + name=parent_name) from None + else: + pkgpath = parent.__path__ + else: + pkgpath = None + target = module + spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target) + if spec is None: + raise ModuleNotFoundError(f"spec not found for the module {name!r}", name=name) + _bootstrap._exec(spec, module) + # The module may have replaced itself in sys.modules! + return sys.modules[name] + finally: + try: + del _RELOADING[name] + except KeyError: + pass diff --git a/contrib/tools/python3/Lib/importlib/_abc.py b/contrib/tools/python3/Lib/importlib/_abc.py new file mode 100644 index 0000000000..693b466112 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/_abc.py @@ -0,0 +1,39 @@ +"""Subset of importlib.abc used to reduce importlib.util imports.""" +from . import _bootstrap +import abc + + +class Loader(metaclass=abc.ABCMeta): + + """Abstract base class for import loaders.""" + + def create_module(self, spec): + """Return a module to initialize and into which to load. + + This method should raise ImportError if anything prevents it + from creating a new module. It may return None to indicate + that the spec should create the new module. + """ + # By default, defer to default semantics for the new module. + return None + + # We don't define exec_module() here since that would break + # hasattr checks we do to support backward compatibility. + + def load_module(self, fullname): + """Return the loaded module. + + The module must be added to sys.modules and have import-related + attributes set properly. The fullname is a str. + + ImportError is raised on failure. + + This method is deprecated in favor of loader.exec_module(). If + exec_module() exists then it is used to provide a backwards-compatible + functionality for this method. + + """ + if not hasattr(self, 'exec_module'): + raise ImportError + # Warning implemented in _load_module_shim(). + return _bootstrap._load_module_shim(self, fullname) diff --git a/contrib/tools/python3/Lib/importlib/_bootstrap.py b/contrib/tools/python3/Lib/importlib/_bootstrap.py new file mode 100644 index 0000000000..d942045f3d --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/_bootstrap.py @@ -0,0 +1,1551 @@ +"""Core implementation of import. + +This module is NOT meant to be directly imported! It has been designed such +that it can be bootstrapped into Python as the implementation of import. As +such it requires the injection of specific modules and attributes in order to +work. One should use importlib as the public-facing version of this module. + +""" +# +# IMPORTANT: Whenever making changes to this module, be sure to run a top-level +# `make regen-importlib` followed by `make` in order to get the frozen version +# of the module updated. Not doing so will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module +# in the early stages of compilation. +# + +# See importlib._setup() for what is injected into the global namespace. + +# When editing this code be aware that code executed at import time CANNOT +# reference any injected objects! This includes not only global code but also +# anything specified at the class level. + +def _object_name(obj): + try: + return obj.__qualname__ + except AttributeError: + return type(obj).__qualname__ + +# Bootstrap-related code ###################################################### + +# Modules injected manually by _setup() +_thread = None +_warnings = None +_weakref = None + +# Import done by _install_external_importers() +_bootstrap_external = None + + +def _wrap(new, old): + """Simple substitute for functools.update_wrapper.""" + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) + new.__dict__.update(old.__dict__) + + +def _new_module(name): + return type(sys)(name) + + +# Module-level locking ######################################################## + +# For a list that can have a weakref to it. +class _List(list): + pass + + +# Copied from weakref.py with some simplifications and modifications unique to +# bootstrapping importlib. Many methods were simply deleting for simplicity, so if they +# are needed in the future they may work if simply copied back in. +class _WeakValueDictionary: + + def __init__(self): + self_weakref = _weakref.ref(self) + + # Inlined to avoid issues with inheriting from _weakref.ref before _weakref is + # set by _setup(). Since there's only one instance of this class, this is + # not expensive. + class KeyedRef(_weakref.ref): + + __slots__ = "key", + + def __new__(type, ob, key): + self = super().__new__(type, ob, type.remove) + self.key = key + return self + + def __init__(self, ob, key): + super().__init__(ob, self.remove) + + @staticmethod + def remove(wr): + nonlocal self_weakref + + self = self_weakref() + if self is not None: + if self._iterating: + self._pending_removals.append(wr.key) + else: + _weakref._remove_dead_weakref(self.data, wr.key) + + self._KeyedRef = KeyedRef + self.clear() + + def clear(self): + self._pending_removals = [] + self._iterating = set() + self.data = {} + + def _commit_removals(self): + pop = self._pending_removals.pop + d = self.data + while True: + try: + key = pop() + except IndexError: + return + _weakref._remove_dead_weakref(d, key) + + def get(self, key, default=None): + if self._pending_removals: + self._commit_removals() + try: + wr = self.data[key] + except KeyError: + return default + else: + if (o := wr()) is None: + return default + else: + return o + + def setdefault(self, key, default=None): + try: + o = self.data[key]() + except KeyError: + o = None + if o is None: + if self._pending_removals: + self._commit_removals() + self.data[key] = self._KeyedRef(default, key) + return default + else: + return o + + +# A dict mapping module names to weakrefs of _ModuleLock instances. +# Dictionary protected by the global import lock. +_module_locks = {} + +# A dict mapping thread IDs to weakref'ed lists of _ModuleLock instances. +# This maps a thread to the module locks it is blocking on acquiring. The +# values are lists because a single thread could perform a re-entrant import +# and be "in the process" of blocking on locks for more than one module. A +# thread can be "in the process" because a thread cannot actually block on +# acquiring more than one lock but it can have set up bookkeeping that reflects +# that it intends to block on acquiring more than one lock. +# +# The dictionary uses a WeakValueDictionary to avoid keeping unnecessary +# lists around, regardless of GC runs. This way there's no memory leak if +# the list is no longer needed (GH-106176). +_blocking_on = None + + +class _BlockingOnManager: + """A context manager responsible to updating ``_blocking_on``.""" + def __init__(self, thread_id, lock): + self.thread_id = thread_id + self.lock = lock + + def __enter__(self): + """Mark the running thread as waiting for self.lock. via _blocking_on.""" + # Interactions with _blocking_on are *not* protected by the global + # import lock here because each thread only touches the state that it + # owns (state keyed on its thread id). The global import lock is + # re-entrant (i.e., a single thread may take it more than once) so it + # wouldn't help us be correct in the face of re-entrancy either. + + self.blocked_on = _blocking_on.setdefault(self.thread_id, _List()) + self.blocked_on.append(self.lock) + + def __exit__(self, *args, **kwargs): + """Remove self.lock from this thread's _blocking_on list.""" + self.blocked_on.remove(self.lock) + + +class _DeadlockError(RuntimeError): + pass + + + +def _has_deadlocked(target_id, *, seen_ids, candidate_ids, blocking_on): + """Check if 'target_id' is holding the same lock as another thread(s). + + The search within 'blocking_on' starts with the threads listed in + 'candidate_ids'. 'seen_ids' contains any threads that are considered + already traversed in the search. + + Keyword arguments: + target_id -- The thread id to try to reach. + seen_ids -- A set of threads that have already been visited. + candidate_ids -- The thread ids from which to begin. + blocking_on -- A dict representing the thread/blocking-on graph. This may + be the same object as the global '_blocking_on' but it is + a parameter to reduce the impact that global mutable + state has on the result of this function. + """ + if target_id in candidate_ids: + # If we have already reached the target_id, we're done - signal that it + # is reachable. + return True + + # Otherwise, try to reach the target_id from each of the given candidate_ids. + for tid in candidate_ids: + if not (candidate_blocking_on := blocking_on.get(tid)): + # There are no edges out from this node, skip it. + continue + elif tid in seen_ids: + # bpo 38091: the chain of tid's we encounter here eventually leads + # to a fixed point or a cycle, but does not reach target_id. + # This means we would not actually deadlock. This can happen if + # other threads are at the beginning of acquire() below. + return False + seen_ids.add(tid) + + # Follow the edges out from this thread. + edges = [lock.owner for lock in candidate_blocking_on] + if _has_deadlocked(target_id, seen_ids=seen_ids, candidate_ids=edges, + blocking_on=blocking_on): + return True + + return False + + +class _ModuleLock: + """A recursive lock implementation which is able to detect deadlocks + (e.g. thread 1 trying to take locks A then B, and thread 2 trying to + take locks B then A). + """ + + def __init__(self, name): + # Create an RLock for protecting the import process for the + # corresponding module. Since it is an RLock, a single thread will be + # able to take it more than once. This is necessary to support + # re-entrancy in the import system that arises from (at least) signal + # handlers and the garbage collector. Consider the case of: + # + # import foo + # -> ... + # -> importlib._bootstrap._ModuleLock.acquire + # -> ... + # -> <garbage collector> + # -> __del__ + # -> import foo + # -> ... + # -> importlib._bootstrap._ModuleLock.acquire + # -> _BlockingOnManager.__enter__ + # + # If a different thread than the running one holds the lock then the + # thread will have to block on taking the lock, which is what we want + # for thread safety. + self.lock = _thread.RLock() + self.wakeup = _thread.allocate_lock() + + # The name of the module for which this is a lock. + self.name = name + + # Can end up being set to None if this lock is not owned by any thread + # or the thread identifier for the owning thread. + self.owner = None + + # Represent the number of times the owning thread has acquired this lock + # via a list of True. This supports RLock-like ("re-entrant lock") + # behavior, necessary in case a single thread is following a circular + # import dependency and needs to take the lock for a single module + # more than once. + # + # Counts are represented as a list of True because list.append(True) + # and list.pop() are both atomic and thread-safe in CPython and it's hard + # to find another primitive with the same properties. + self.count = [] + + # This is a count of the number of threads that are blocking on + # self.wakeup.acquire() awaiting to get their turn holding this module + # lock. When the module lock is released, if this is greater than + # zero, it is decremented and `self.wakeup` is released one time. The + # intent is that this will let one other thread make more progress on + # acquiring this module lock. This repeats until all the threads have + # gotten a turn. + # + # This is incremented in self.acquire() when a thread notices it is + # going to have to wait for another thread to finish. + # + # See the comment above count for explanation of the representation. + self.waiters = [] + + def has_deadlock(self): + # To avoid deadlocks for concurrent or re-entrant circular imports, + # look at _blocking_on to see if any threads are blocking + # on getting the import lock for any module for which the import lock + # is held by this thread. + return _has_deadlocked( + # Try to find this thread. + target_id=_thread.get_ident(), + seen_ids=set(), + # Start from the thread that holds the import lock for this + # module. + candidate_ids=[self.owner], + # Use the global "blocking on" state. + blocking_on=_blocking_on, + ) + + def acquire(self): + """ + Acquire the module lock. If a potential deadlock is detected, + a _DeadlockError is raised. + Otherwise, the lock is always acquired and True is returned. + """ + tid = _thread.get_ident() + with _BlockingOnManager(tid, self): + while True: + # Protect interaction with state on self with a per-module + # lock. This makes it safe for more than one thread to try to + # acquire the lock for a single module at the same time. + with self.lock: + if self.count == [] or self.owner == tid: + # If the lock for this module is unowned then we can + # take the lock immediately and succeed. If the lock + # for this module is owned by the running thread then + # we can also allow the acquire to succeed. This + # supports circular imports (thread T imports module A + # which imports module B which imports module A). + self.owner = tid + self.count.append(True) + return True + + # At this point we know the lock is held (because count != + # 0) by another thread (because owner != tid). We'll have + # to get in line to take the module lock. + + # But first, check to see if this thread would create a + # deadlock by acquiring this module lock. If it would + # then just stop with an error. + # + # It's not clear who is expected to handle this error. + # There is one handler in _lock_unlock_module but many + # times this method is called when entering the context + # manager _ModuleLockManager instead - so _DeadlockError + # will just propagate up to application code. + # + # This seems to be more than just a hypothetical - + # https://stackoverflow.com/questions/59509154 + # https://github.com/encode/django-rest-framework/issues/7078 + if self.has_deadlock(): + raise _DeadlockError(f'deadlock detected by {self!r}') + + # Check to see if we're going to be able to acquire the + # lock. If we are going to have to wait then increment + # the waiters so `self.release` will know to unblock us + # later on. We do this part non-blockingly so we don't + # get stuck here before we increment waiters. We have + # this extra acquire call (in addition to the one below, + # outside the self.lock context manager) to make sure + # self.wakeup is held when the next acquire is called (so + # we block). This is probably needlessly complex and we + # should just take self.wakeup in the return codepath + # above. + if self.wakeup.acquire(False): + self.waiters.append(None) + + # Now take the lock in a blocking fashion. This won't + # complete until the thread holding this lock + # (self.owner) calls self.release. + self.wakeup.acquire() + + # Taking the lock has served its purpose (making us wait), so we can + # give it up now. We'll take it w/o blocking again on the + # next iteration around this 'while' loop. + self.wakeup.release() + + def release(self): + tid = _thread.get_ident() + with self.lock: + if self.owner != tid: + raise RuntimeError('cannot release un-acquired lock') + assert len(self.count) > 0 + self.count.pop() + if not len(self.count): + self.owner = None + if len(self.waiters) > 0: + self.waiters.pop() + self.wakeup.release() + + def __repr__(self): + return f'_ModuleLock({self.name!r}) at {id(self)}' + + +class _DummyModuleLock: + """A simple _ModuleLock equivalent for Python builds without + multi-threading support.""" + + def __init__(self, name): + self.name = name + self.count = 0 + + def acquire(self): + self.count += 1 + return True + + def release(self): + if self.count == 0: + raise RuntimeError('cannot release un-acquired lock') + self.count -= 1 + + def __repr__(self): + return f'_DummyModuleLock({self.name!r}) at {id(self)}' + + +class _ModuleLockManager: + + def __init__(self, name): + self._name = name + self._lock = None + + def __enter__(self): + self._lock = _get_module_lock(self._name) + self._lock.acquire() + + def __exit__(self, *args, **kwargs): + self._lock.release() + + +# The following two functions are for consumption by Python/import.c. + +def _get_module_lock(name): + """Get or create the module lock for a given module name. + + Acquire/release internally the global import lock to protect + _module_locks.""" + + _imp.acquire_lock() + try: + try: + lock = _module_locks[name]() + except KeyError: + lock = None + + if lock is None: + if _thread is None: + lock = _DummyModuleLock(name) + else: + lock = _ModuleLock(name) + + def cb(ref, name=name): + _imp.acquire_lock() + try: + # bpo-31070: Check if another thread created a new lock + # after the previous lock was destroyed + # but before the weakref callback was called. + if _module_locks.get(name) is ref: + del _module_locks[name] + finally: + _imp.release_lock() + + _module_locks[name] = _weakref.ref(lock, cb) + finally: + _imp.release_lock() + + return lock + + +def _lock_unlock_module(name): + """Acquires then releases the module lock for a given module name. + + This is used to ensure a module is completely initialized, in the + event it is being imported by another thread. + """ + lock = _get_module_lock(name) + try: + lock.acquire() + except _DeadlockError: + # Concurrent circular import, we'll accept a partially initialized + # module object. + pass + else: + lock.release() + +# Frame stripping magic ############################################### +def _call_with_frames_removed(f, *args, **kwds): + """remove_importlib_frames in import.c will always remove sequences + of importlib frames that end with a call to this function + + Use it instead of a normal call in places where including the importlib + frames introduces unwanted noise into the traceback (e.g. when executing + module code) + """ + return f(*args, **kwds) + + +def _verbose_message(message, *args, verbosity=1): + """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" + if sys.flags.verbose >= verbosity: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + + +def _requires_builtin(fxn): + """Decorator to verify the named module is built-in.""" + def _requires_builtin_wrapper(self, fullname): + if fullname not in sys.builtin_module_names: + raise ImportError(f'{fullname!r} is not a built-in module', + name=fullname) + return fxn(self, fullname) + _wrap(_requires_builtin_wrapper, fxn) + return _requires_builtin_wrapper + + +def _requires_frozen(fxn): + """Decorator to verify the named module is frozen.""" + def _requires_frozen_wrapper(self, fullname): + if not _imp.is_frozen(fullname): + raise ImportError(f'{fullname!r} is not a frozen module', + name=fullname) + return fxn(self, fullname) + _wrap(_requires_frozen_wrapper, fxn) + return _requires_frozen_wrapper + + +# Typically used by loader classes as a method replacement. +def _load_module_shim(self, fullname): + """Load the specified module into sys.modules and return it. + + This method is deprecated. Use loader.exec_module() instead. + + """ + msg = ("the load_module() method is deprecated and slated for removal in " + "Python 3.12; use exec_module() instead") + _warnings.warn(msg, DeprecationWarning) + spec = spec_from_loader(fullname, self) + if fullname in sys.modules: + module = sys.modules[fullname] + _exec(spec, module) + return sys.modules[fullname] + else: + return _load(spec) + +# Module specifications ####################################################### + +def _module_repr(module): + """The implementation of ModuleType.__repr__().""" + loader = getattr(module, '__loader__', None) + if spec := getattr(module, "__spec__", None): + return _module_repr_from_spec(spec) + # Fall through to a catch-all which always succeeds. + try: + name = module.__name__ + except AttributeError: + name = '?' + try: + filename = module.__file__ + except AttributeError: + if loader is None: + return f'<module {name!r}>' + else: + return f'<module {name!r} ({loader!r})>' + else: + return f'<module {name!r} from {filename!r}>' + + +class ModuleSpec: + """The specification for a module, used for loading. + + A module's spec is the source for information about the module. For + data associated with the module, including source, use the spec's + loader. + + `name` is the absolute name of the module. `loader` is the loader + to use when loading the module. `parent` is the name of the + package the module is in. The parent is derived from the name. + + `is_package` determines if the module is considered a package or + not. On modules this is reflected by the `__path__` attribute. + + `origin` is the specific location used by the loader from which to + load the module, if that information is available. When filename is + set, origin will match. + + `has_location` indicates that a spec's "origin" reflects a location. + When this is True, `__file__` attribute of the module is set. + + `cached` is the location of the cached bytecode file, if any. It + corresponds to the `__cached__` attribute. + + `submodule_search_locations` is the sequence of path entries to + search when importing submodules. If set, is_package should be + True--and False otherwise. + + Packages are simply modules that (may) have submodules. If a spec + has a non-None value in `submodule_search_locations`, the import + system will consider modules loaded from the spec as packages. + + Only finders (see importlib.abc.MetaPathFinder and + importlib.abc.PathEntryFinder) should modify ModuleSpec instances. + + """ + + def __init__(self, name, loader, *, origin=None, loader_state=None, + is_package=None): + self.name = name + self.loader = loader + self.origin = origin + self.loader_state = loader_state + self.submodule_search_locations = [] if is_package else None + self._uninitialized_submodules = [] + + # file-location attributes + self._set_fileattr = False + self._cached = None + + def __repr__(self): + args = [f'name={self.name!r}', f'loader={self.loader!r}'] + if self.origin is not None: + args.append(f'origin={self.origin!r}') + if self.submodule_search_locations is not None: + args.append(f'submodule_search_locations={self.submodule_search_locations}') + return f'{self.__class__.__name__}({", ".join(args)})' + + def __eq__(self, other): + smsl = self.submodule_search_locations + try: + return (self.name == other.name and + self.loader == other.loader and + self.origin == other.origin and + smsl == other.submodule_search_locations and + self.cached == other.cached and + self.has_location == other.has_location) + except AttributeError: + return NotImplemented + + @property + def cached(self): + if self._cached is None: + if self.origin is not None and self._set_fileattr: + if _bootstrap_external is None: + raise NotImplementedError + self._cached = _bootstrap_external._get_cached(self.origin) + return self._cached + + @cached.setter + def cached(self, cached): + self._cached = cached + + @property + def parent(self): + """The name of the module's parent.""" + if self.submodule_search_locations is None: + return self.name.rpartition('.')[0] + else: + return self.name + + @property + def has_location(self): + return self._set_fileattr + + @has_location.setter + def has_location(self, value): + self._set_fileattr = bool(value) + + +def spec_from_loader(name, loader, *, origin=None, is_package=None): + """Return a module spec based on various loader methods.""" + if origin is None: + origin = getattr(loader, '_ORIGIN', None) + + if not origin and hasattr(loader, 'get_filename'): + if _bootstrap_external is None: + raise NotImplementedError + spec_from_file_location = _bootstrap_external.spec_from_file_location + + if is_package is None: + return spec_from_file_location(name, loader=loader) + search = [] if is_package else None + return spec_from_file_location(name, loader=loader, + submodule_search_locations=search) + + if is_package is None: + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + is_package = None # aka, undefined + else: + # the default + is_package = False + + return ModuleSpec(name, loader, origin=origin, is_package=is_package) + + +def _spec_from_module(module, loader=None, origin=None): + # This function is meant for use in _setup(). + try: + spec = module.__spec__ + except AttributeError: + pass + else: + if spec is not None: + return spec + + name = module.__name__ + if loader is None: + try: + loader = module.__loader__ + except AttributeError: + # loader will stay None. + pass + try: + location = module.__file__ + except AttributeError: + location = None + if origin is None: + if loader is not None: + origin = getattr(loader, '_ORIGIN', None) + if not origin and location is not None: + origin = location + try: + cached = module.__cached__ + except AttributeError: + cached = None + try: + submodule_search_locations = list(module.__path__) + except AttributeError: + submodule_search_locations = None + + spec = ModuleSpec(name, loader, origin=origin) + spec._set_fileattr = False if location is None else (origin == location) + spec.cached = cached + spec.submodule_search_locations = submodule_search_locations + return spec + + +def _init_module_attrs(spec, module, *, override=False): + # The passed-in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + # __name__ + if (override or getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + # __loader__ + if override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + if _bootstrap_external is None: + raise NotImplementedError + NamespaceLoader = _bootstrap_external.NamespaceLoader + + loader = NamespaceLoader.__new__(NamespaceLoader) + loader._path = spec.submodule_search_locations + spec.loader = loader + # While the docs say that module.__file__ is not set for + # built-in modules, and the code below will avoid setting it if + # spec.has_location is false, this is incorrect for namespace + # packages. Namespace packages have no location, but their + # __spec__.origin is None, and thus their module.__file__ + # should also be None for consistency. While a bit of a hack, + # this is the best place to ensure this consistency. + # + # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module + # and bpo-32305 + module.__file__ = None + try: + module.__loader__ = loader + except AttributeError: + pass + # __package__ + if override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass + # __path__ + if override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: + # XXX We should extend __path__ if it's already a list. + try: + module.__path__ = spec.submodule_search_locations + except AttributeError: + pass + # __file__/__cached__ + if spec.has_location: + if override or getattr(module, '__file__', None) is None: + try: + module.__file__ = spec.origin + except AttributeError: + pass + + if override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: + try: + module.__cached__ = spec.cached + except AttributeError: + pass + return module + + +def module_from_spec(spec): + """Create a module based on the provided spec.""" + # Typically loaders will not implement create_module(). + module = None + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` then it means default + # module creation should be used. + module = spec.loader.create_module(spec) + elif hasattr(spec.loader, 'exec_module'): + raise ImportError('loaders that define exec_module() ' + 'must also define create_module()') + if module is None: + module = _new_module(spec.name) + _init_module_attrs(spec, module) + return module + + +def _module_repr_from_spec(spec): + """Return the repr to use for the module.""" + name = '?' if spec.name is None else spec.name + if spec.origin is None: + loader = spec.loader + if loader is None: + return f'<module {name!r}>' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'<module {name!r} (namespace) from {list(loader._path)}>' + else: + return f'<module {name!r} ({loader!r})>' + else: + if spec.has_location: + return f'<module {name!r} from {spec.origin!r}>' + else: + return f'<module {spec.name!r} ({spec.origin})>' + + +# Used by importlib.reload() and _load_module_shim(). +def _exec(spec, module): + """Execute the spec's specified module in an existing module's namespace.""" + name = spec.name + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = f'module {name!r} not in sys.modules' + raise ImportError(msg, name=name) + try: + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # Namespace package. + _init_module_attrs(spec, module, override=True) + else: + _init_module_attrs(spec, module, override=True) + if not hasattr(spec.loader, 'exec_module'): + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) + spec.loader.load_module(name) + else: + spec.loader.exec_module(module) + finally: + # Update the order of insertion into sys.modules for module + # clean-up at shutdown. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + return module + + +def _load_backward_compatible(spec): + # It is assumed that all callers have been warned about using load_module() + # appropriately before calling this function. + try: + spec.loader.load_module(spec.name) + except: + if spec.name in sys.modules: + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + raise + # The module must be in sys.modules at this point! + # Move it to the end of sys.modules. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + if getattr(module, '__loader__', None) is None: + try: + module.__loader__ = spec.loader + except AttributeError: + pass + if getattr(module, '__package__', None) is None: + try: + # Since module.__path__ may not line up with + # spec.submodule_search_paths, we can't necessarily rely + # on spec.parent here. + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = spec.name.rpartition('.')[0] + except AttributeError: + pass + if getattr(module, '__spec__', None) is None: + try: + module.__spec__ = spec + except AttributeError: + pass + return module + +def _load_unlocked(spec): + # A helper for direct use by the import system. + if spec.loader is not None: + # Not a namespace package. + if not hasattr(spec.loader, 'exec_module'): + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) + return _load_backward_compatible(spec) + + module = module_from_spec(spec) + + # This must be done before putting the module in sys.modules + # (otherwise an optimization shortcut in import.c becomes + # wrong). + spec._initializing = True + try: + sys.modules[spec.name] = module + try: + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # A namespace package so do nothing. + else: + spec.loader.exec_module(module) + except: + try: + del sys.modules[spec.name] + except KeyError: + pass + raise + # Move the module to the end of sys.modules. + # We don't ensure that the import-related module attributes get + # set in the sys.modules replacement case. Such modules are on + # their own. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + _verbose_message('import {!r} # {!r}', spec.name, spec.loader) + finally: + spec._initializing = False + + return module + +# A method used during testing of _load_unlocked() and by +# _load_module_shim(). +def _load(spec): + """Return a new module object, loaded by the spec's loader. + + The module is not added to its parent. + + If a module is already in sys.modules, that existing module gets + clobbered. + + """ + with _ModuleLockManager(spec.name): + return _load_unlocked(spec) + + +# Loaders ##################################################################### + +class BuiltinImporter: + + """Meta path import for built-in modules. + + All methods are either class or static methods to avoid the need to + instantiate the class. + + """ + + _ORIGIN = "built-in" + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if _imp.is_builtin(fullname): + return spec_from_loader(fullname, cls, origin=cls._ORIGIN) + else: + return None + + @staticmethod + def create_module(spec): + """Create a built-in module""" + if spec.name not in sys.builtin_module_names: + raise ImportError(f'{spec.name!r} is not a built-in module', + name=spec.name) + return _call_with_frames_removed(_imp.create_builtin, spec) + + @staticmethod + def exec_module(module): + """Exec a built-in module""" + _call_with_frames_removed(_imp.exec_builtin, module) + + @classmethod + @_requires_builtin + def get_code(cls, fullname): + """Return None as built-in modules do not have code objects.""" + return None + + @classmethod + @_requires_builtin + def get_source(cls, fullname): + """Return None as built-in modules do not have source code.""" + return None + + @classmethod + @_requires_builtin + def is_package(cls, fullname): + """Return False as built-in modules are never packages.""" + return False + + load_module = classmethod(_load_module_shim) + + +class FrozenImporter: + + """Meta path import for frozen modules. + + All methods are either class or static methods to avoid the need to + instantiate the class. + + """ + + _ORIGIN = "frozen" + + @classmethod + def _fix_up_module(cls, module): + spec = module.__spec__ + state = spec.loader_state + if state is None: + # The module is missing FrozenImporter-specific values. + + # Fix up the spec attrs. + origname = vars(module).pop('__origname__', None) + assert origname, 'see PyImport_ImportFrozenModuleObject()' + ispkg = hasattr(module, '__path__') + assert _imp.is_frozen_package(module.__name__) == ispkg, ispkg + filename, pkgdir = cls._resolve_filename(origname, spec.name, ispkg) + spec.loader_state = type(sys.implementation)( + filename=filename, + origname=origname, + ) + __path__ = spec.submodule_search_locations + if ispkg: + assert __path__ == [], __path__ + if pkgdir: + spec.submodule_search_locations.insert(0, pkgdir) + else: + assert __path__ is None, __path__ + + # Fix up the module attrs (the bare minimum). + assert not hasattr(module, '__file__'), module.__file__ + if filename: + try: + module.__file__ = filename + except AttributeError: + pass + if ispkg: + if module.__path__ != __path__: + assert module.__path__ == [], module.__path__ + module.__path__.extend(__path__) + else: + # These checks ensure that _fix_up_module() is only called + # in the right places. + __path__ = spec.submodule_search_locations + ispkg = __path__ is not None + # Check the loader state. + assert sorted(vars(state)) == ['filename', 'origname'], state + if state.origname: + # The only frozen modules with "origname" set are stdlib modules. + (__file__, pkgdir, + ) = cls._resolve_filename(state.origname, spec.name, ispkg) + assert state.filename == __file__, (state.filename, __file__) + if pkgdir: + assert __path__ == [pkgdir], (__path__, pkgdir) + else: + assert __path__ == ([] if ispkg else None), __path__ + else: + __file__ = None + assert state.filename is None, state.filename + assert __path__ == ([] if ispkg else None), __path__ + # Check the file attrs. + if __file__: + assert hasattr(module, '__file__') + assert module.__file__ == __file__, (module.__file__, __file__) + else: + assert not hasattr(module, '__file__'), module.__file__ + if ispkg: + assert hasattr(module, '__path__') + assert module.__path__ == __path__, (module.__path__, __path__) + else: + assert not hasattr(module, '__path__'), module.__path__ + assert not spec.has_location + + @classmethod + def _resolve_filename(cls, fullname, alias=None, ispkg=False): + if not fullname or not getattr(sys, '_stdlib_dir', None): + return None, None + try: + sep = cls._SEP + except AttributeError: + sep = cls._SEP = '\\' if sys.platform == 'win32' else '/' + + if fullname != alias: + if fullname.startswith('<'): + fullname = fullname[1:] + if not ispkg: + fullname = f'{fullname}.__init__' + else: + ispkg = False + relfile = fullname.replace('.', sep) + if ispkg: + pkgdir = f'{sys._stdlib_dir}{sep}{relfile}' + filename = f'{pkgdir}{sep}__init__.py' + else: + pkgdir = None + filename = f'{sys._stdlib_dir}{sep}{relfile}.py' + return filename, pkgdir + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + info = _call_with_frames_removed(_imp.find_frozen, fullname) + if info is None: + return None + # We get the marshaled data in exec_module() (the loader + # part of the importer), instead of here (the finder part). + # The loader is the usual place to get the data that will + # be loaded into the module. (For example, see _LoaderBasics + # in _bootstra_external.py.) Most importantly, this importer + # is simpler if we wait to get the data. + # However, getting as much data in the finder as possible + # to later load the module is okay, and sometimes important. + # (That's why ModuleSpec.loader_state exists.) This is + # especially true if it avoids throwing away expensive data + # the loader would otherwise duplicate later and can be done + # efficiently. In this case it isn't worth it. + _, ispkg, origname = info + spec = spec_from_loader(fullname, cls, + origin=cls._ORIGIN, + is_package=ispkg) + filename, pkgdir = cls._resolve_filename(origname, fullname, ispkg) + spec.loader_state = type(sys.implementation)( + filename=filename, + origname=origname, + ) + if pkgdir: + spec.submodule_search_locations.insert(0, pkgdir) + return spec + + @staticmethod + def create_module(spec): + """Set __file__, if able.""" + module = _new_module(spec.name) + try: + filename = spec.loader_state.filename + except AttributeError: + pass + else: + if filename: + module.__file__ = filename + return module + + @staticmethod + def exec_module(module): + spec = module.__spec__ + name = spec.name + code = _call_with_frames_removed(_imp.get_frozen_object, name) + exec(code, module.__dict__) + + @classmethod + def load_module(cls, fullname): + """Load a frozen module. + + This method is deprecated. Use exec_module() instead. + + """ + # Warning about deprecation implemented in _load_module_shim(). + module = _load_module_shim(cls, fullname) + info = _imp.find_frozen(fullname) + assert info is not None + _, ispkg, origname = info + module.__origname__ = origname + vars(module).pop('__file__', None) + if ispkg: + module.__path__ = [] + cls._fix_up_module(module) + return module + + @classmethod + @_requires_frozen + def get_code(cls, fullname): + """Return the code object for the frozen module.""" + return _imp.get_frozen_object(fullname) + + @classmethod + @_requires_frozen + def get_source(cls, fullname): + """Return None as frozen modules do not have source code.""" + return None + + @classmethod + @_requires_frozen + def is_package(cls, fullname): + """Return True if the frozen module is a package.""" + return _imp.is_frozen_package(fullname) + + +# Import itself ############################################################### + +class _ImportLockContext: + + """Context manager for the import lock.""" + + def __enter__(self): + """Acquire the import lock.""" + _imp.acquire_lock() + + def __exit__(self, exc_type, exc_value, exc_traceback): + """Release the import lock regardless of any raised exceptions.""" + _imp.release_lock() + + +def _resolve_name(name, package, level): + """Resolve a relative module name to an absolute one.""" + bits = package.rsplit('.', level - 1) + if len(bits) < level: + raise ImportError('attempted relative import beyond top-level package') + base = bits[0] + return f'{base}.{name}' if name else base + + +def _find_spec(name, path, target=None): + """Find a module's spec.""" + meta_path = sys.meta_path + if meta_path is None: + # PyImport_Cleanup() is running or has been called. + raise ImportError("sys.meta_path is None, Python is likely " + "shutting down") + + if not meta_path: + _warnings.warn('sys.meta_path is empty', ImportWarning) + + # We check sys.modules here for the reload case. While a passed-in + # target will usually indicate a reload there is no guarantee, whereas + # sys.modules provides one. + is_reload = name in sys.modules + for finder in meta_path: + with _ImportLockContext(): + try: + find_spec = finder.find_spec + except AttributeError: + continue + else: + spec = find_spec(name, path, target) + if spec is not None: + # The parent import may have already imported this module. + if not is_reload and name in sys.modules: + module = sys.modules[name] + try: + __spec__ = module.__spec__ + except AttributeError: + # We use the found spec since that is the one that + # we would have used if the parent module hadn't + # beaten us to the punch. + return spec + else: + if __spec__ is None: + return spec + else: + return __spec__ + else: + return spec + else: + return None + + +def _sanity_check(name, package, level): + """Verify arguments are "sane".""" + if not isinstance(name, str): + raise TypeError(f'module name must be str, not {type(name)}') + if level < 0: + raise ValueError('level must be >= 0') + if level > 0: + if not isinstance(package, str): + raise TypeError('__package__ not set to a string') + elif not package: + raise ImportError('attempted relative import with no known parent ' + 'package') + if not name and level == 0: + raise ValueError('Empty module name') + + +_ERR_MSG_PREFIX = 'No module named ' +_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' + +def _find_and_load_unlocked(name, import_): + path = None + parent = name.rpartition('.')[0] + parent_spec = None + if parent: + if parent not in sys.modules: + _call_with_frames_removed(import_, parent) + # Crazy side-effects! + if name in sys.modules: + return sys.modules[name] + parent_module = sys.modules[parent] + try: + path = parent_module.__path__ + except AttributeError: + msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' + raise ModuleNotFoundError(msg, name=name) from None + parent_spec = parent_module.__spec__ + child = name.rpartition('.')[2] + spec = _find_spec(name, path) + if spec is None: + raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) + else: + if parent_spec: + # Temporarily add child we are currently importing to parent's + # _uninitialized_submodules for circular import tracking. + parent_spec._uninitialized_submodules.append(child) + try: + module = _load_unlocked(spec) + finally: + if parent_spec: + parent_spec._uninitialized_submodules.pop() + if parent: + # Set the module as an attribute on its parent. + parent_module = sys.modules[parent] + try: + setattr(parent_module, child, module) + except AttributeError: + msg = f"Cannot set an attribute on {parent!r} for child module {child!r}" + _warnings.warn(msg, ImportWarning) + return module + + +_NEEDS_LOADING = object() + + +def _find_and_load(name, import_): + """Find and load the module.""" + + # Optimization: we avoid unneeded module locking if the module + # already exists in sys.modules and is fully initialized. + module = sys.modules.get(name, _NEEDS_LOADING) + if (module is _NEEDS_LOADING or + getattr(getattr(module, "__spec__", None), "_initializing", False)): + with _ModuleLockManager(name): + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return _find_and_load_unlocked(name, import_) + + # Optimization: only call _bootstrap._lock_unlock_module() if + # module.__spec__._initializing is True. + # NOTE: because of this, initializing must be set *before* + # putting the new module in sys.modules. + _lock_unlock_module(name) + + if module is None: + message = f'import of {name} halted; None in sys.modules' + raise ModuleNotFoundError(message, name=name) + + return module + + +def _gcd_import(name, package=None, level=0): + """Import and return the module based on its name, the package the call is + being made from, and the level adjustment. + + This function represents the greatest common denominator of functionality + between import_module and __import__. This includes setting __package__ if + the loader did not. + + """ + _sanity_check(name, package, level) + if level > 0: + name = _resolve_name(name, package, level) + return _find_and_load(name, _gcd_import) + + +def _handle_fromlist(module, fromlist, import_, *, recursive=False): + """Figure out what __import__ should return. + + The import_ parameter is a callable which takes the name of module to + import. It is required to decouple the function from assuming importlib's + import implementation is desired. + + """ + # The hell that is fromlist ... + # If a package was imported, try to import stuff from fromlist. + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module.__name__ + '.__all__' + else: + where = "``from list''" + raise TypeError(f"Item in {where} must be str, " + f"not {type(x).__name__}") + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, + recursive=True) + elif not hasattr(module, x): + from_name = f'{module.__name__}.{x}' + try: + _call_with_frames_removed(import_, from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if (exc.name == from_name and + sys.modules.get(from_name, _NEEDS_LOADING) is not None): + continue + raise + return module + + +def _calc___package__(globals): + """Calculate what __package__ should be. + + __package__ is not guaranteed to be defined or could be set to None + to represent that its proper value is unknown. + + """ + package = globals.get('__package__') + spec = globals.get('__spec__') + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn("__package__ != __spec__.parent " + f"({package!r} != {spec.parent!r})", + DeprecationWarning, stacklevel=3) + return package + elif spec is not None: + return spec.parent + else: + _warnings.warn("can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, stacklevel=3) + package = globals['__name__'] + if '__path__' not in globals: + package = package.rpartition('.')[0] + return package + + +def __import__(name, globals=None, locals=None, fromlist=(), level=0): + """Import a module. + + The 'globals' argument is used to infer where the import is occurring from + to handle relative imports. The 'locals' argument is ignored. The + 'fromlist' argument specifies what should exist as attributes on the module + being imported (e.g. ``from module import <fromlist>``). The 'level' + argument represents the package location to import from in a relative + import (e.g. ``from ..pkg import mod`` would have a 'level' of 2). + + """ + if level == 0: + module = _gcd_import(name) + else: + globals_ = globals if globals is not None else {} + package = _calc___package__(globals_) + module = _gcd_import(name, package, level) + if not fromlist: + # Return up to the first dot in 'name'. This is complicated by the fact + # that 'name' may be relative. + if level == 0: + return _gcd_import(name.partition('.')[0]) + elif not name: + return module + else: + # Figure out where to slice the module's name up to the first dot + # in 'name'. + cut_off = len(name) - len(name.partition('.')[0]) + # Slice end needs to be positive to alleviate need to special-case + # when ``'.' not in name``. + return sys.modules[module.__name__[:len(module.__name__)-cut_off]] + elif hasattr(module, '__path__'): + return _handle_fromlist(module, fromlist, _gcd_import) + else: + return module + + +def _builtin_from_name(name): + spec = BuiltinImporter.find_spec(name) + if spec is None: + raise ImportError('no built-in module named ' + name) + return _load_unlocked(spec) + + +def _setup(sys_module, _imp_module): + """Setup importlib by importing needed built-in modules and injecting them + into the global namespace. + + As sys is needed for sys.modules access and _imp is needed to load built-in + modules, those two modules must be explicitly passed in. + + """ + global _imp, sys, _blocking_on + _imp = _imp_module + sys = sys_module + + # Set up the spec for existing builtin/frozen modules. + module_type = type(sys) + for name, module in sys.modules.items(): + if isinstance(module, module_type): + if name in sys.builtin_module_names: + loader = BuiltinImporter + elif _imp.is_frozen(name): + loader = FrozenImporter + else: + continue + spec = _spec_from_module(module, loader) + _init_module_attrs(spec, module) + if loader is FrozenImporter: + loader._fix_up_module(module) + + # Directly load built-in modules needed during bootstrap. + self_module = sys.modules[__name__] + for builtin_name in ('_thread', '_warnings', '_weakref'): + if builtin_name not in sys.modules: + builtin_module = _builtin_from_name(builtin_name) + else: + builtin_module = sys.modules[builtin_name] + setattr(self_module, builtin_name, builtin_module) + + # Instantiation requires _weakref to have been set. + _blocking_on = _WeakValueDictionary() + + +def _install(sys_module, _imp_module): + """Install importers for builtin and frozen modules""" + _setup(sys_module, _imp_module) + + sys.meta_path.append(BuiltinImporter) + sys.meta_path.append(FrozenImporter) + + +def _install_external_importers(): + """Install importers that require external filesystem access""" + global _bootstrap_external + import _frozen_importlib_external + _bootstrap_external = _frozen_importlib_external + _frozen_importlib_external._install(sys.modules[__name__]) diff --git a/contrib/tools/python3/Lib/importlib/_bootstrap_external.py b/contrib/tools/python3/Lib/importlib/_bootstrap_external.py new file mode 100644 index 0000000000..e6f75a9f6f --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/_bootstrap_external.py @@ -0,0 +1,1742 @@ +"""Core implementation of path-based import. + +This module is NOT meant to be directly imported! It has been designed such +that it can be bootstrapped into Python as the implementation of import. As +such it requires the injection of specific modules and attributes in order to +work. One should use importlib as the public-facing version of this module. + +""" +# IMPORTANT: Whenever making changes to this module, be sure to run a top-level +# `make regen-importlib` followed by `make` in order to get the frozen version +# of the module updated. Not doing so will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module in the early +# stages of compilation. +# + +# See importlib._setup() for what is injected into the global namespace. + +# When editing this code be aware that code executed at import time CANNOT +# reference any injected objects! This includes not only global code but also +# anything specified at the class level. + +# Module injected manually by _set_bootstrap_module() +_bootstrap = None + +# Import builtin modules +import _imp +import _io +import sys +import _warnings +import marshal + + +_MS_WINDOWS = (sys.platform == 'win32') +if _MS_WINDOWS: + import nt as _os + import winreg +else: + import posix as _os + + +if _MS_WINDOWS: + path_separators = ['\\', '/'] +else: + path_separators = ['/'] +# Assumption made in _path_join() +assert all(len(sep) == 1 for sep in path_separators) +path_sep = path_separators[0] +path_sep_tuple = tuple(path_separators) +path_separators = ''.join(path_separators) +_pathseps_with_colon = {f':{s}' for s in path_separators} + + +# Bootstrap-related code ###################################################### +_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) + + +def _make_relax_case(): + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY): + key = 'PYTHONCASEOK' + else: + key = b'PYTHONCASEOK' + + def _relax_case(): + """True if filenames must be checked case-insensitively and ignore environment flags are not set.""" + return not sys.flags.ignore_environment and key in _os.environ + else: + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return False + return _relax_case + +_relax_case = _make_relax_case() + + +def _pack_uint32(x): + """Convert a 32-bit integer to little-endian.""" + return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') + + +def _unpack_uint32(data): + """Convert 4 bytes in little-endian to an integer.""" + assert len(data) == 4 + return int.from_bytes(data, 'little') + +def _unpack_uint16(data): + """Convert 2 bytes in little-endian to an integer.""" + assert len(data) == 2 + return int.from_bytes(data, 'little') + + +if _MS_WINDOWS: + def _path_join(*path_parts): + """Replacement for os.path.join().""" + if not path_parts: + return "" + if len(path_parts) == 1: + return path_parts[0] + root = "" + path = [] + for new_root, tail in map(_os._path_splitroot, path_parts): + if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple): + root = new_root.rstrip(path_separators) or root + path = [path_sep + tail] + elif new_root.endswith(':'): + if root.casefold() != new_root.casefold(): + # Drive relative paths have to be resolved by the OS, so we reset the + # tail but do not add a path_sep prefix. + root = new_root + path = [tail] + else: + path.append(tail) + else: + root = new_root or root + path.append(tail) + path = [p.rstrip(path_separators) for p in path if p] + if len(path) == 1 and not path[0]: + # Avoid losing the root's trailing separator when joining with nothing + return root + path_sep + return root + path_sep.join(path) + +else: + def _path_join(*path_parts): + """Replacement for os.path.join().""" + return path_sep.join([part.rstrip(path_separators) + for part in path_parts if part]) + + +def _path_split(path): + """Replacement for os.path.split().""" + i = max(path.rfind(p) for p in path_separators) + if i < 0: + return '', path + return path[:i], path[i + 1:] + + +def _path_stat(path): + """Stat the path. + + Made a separate function to make it easier to override in experiments + (e.g. cache stat results). + + """ + return _os.stat(path) + + +def _path_is_mode_type(path, mode): + """Test whether the path is the specified mode type.""" + try: + stat_info = _path_stat(path) + except OSError: + return False + return (stat_info.st_mode & 0o170000) == mode + + +def _path_isfile(path): + """Replacement for os.path.isfile.""" + return _path_is_mode_type(path, 0o100000) + + +def _path_isdir(path): + """Replacement for os.path.isdir.""" + if not path: + path = _os.getcwd() + return _path_is_mode_type(path, 0o040000) + + +if _MS_WINDOWS: + def _path_isabs(path): + """Replacement for os.path.isabs.""" + if not path: + return False + root = _os._path_splitroot(path)[0].replace('/', '\\') + return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\')) + +else: + def _path_isabs(path): + """Replacement for os.path.isabs.""" + return path.startswith(path_separators) + + +def _path_abspath(path): + """Replacement for os.path.abspath.""" + if not _path_isabs(path): + for sep in path_separators: + path = path.removeprefix(f".{sep}") + return _path_join(_os.getcwd(), path) + else: + return path + + +def _write_atomic(path, data, mode=0o666): + """Best-effort function to write data to a path atomically. + Be prepared to handle a FileExistsError if concurrent writing of the + temporary file is attempted.""" + # id() is used to generate a pseudo-random filename. + path_tmp = f'{path}.{id(path)}' + fd = _os.open(path_tmp, + _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666) + try: + # We first write data to a temporary file, and then use os.replace() to + # perform an atomic rename. + with _io.FileIO(fd, 'wb') as file: + file.write(data) + _os.replace(path_tmp, path) + except OSError: + try: + _os.unlink(path_tmp) + except OSError: + pass + raise + + +_code_type = type(_write_atomic.__code__) + + +# Finder/loader utility code ############################################### + +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) +# Python 2.7a0: 62181 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 2.7a0 62191 (introduce SETUP_WITH) +# Python 2.7a0 62201 (introduce BUILD_SET) +# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT + #3021) +# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) +# Python 3.1a1: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE + #4715) +# Python 3.2a1: 3160 (add SETUP_WITH #6101) +# tag: cpython-32 +# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) +# tag: cpython-32 +# Python 3.2a3 3180 (add DELETE_DEREF #4617) +# Python 3.3a1 3190 (__class__ super closure changed) +# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) +# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) +# Python 3.3a2 3220 (changed PEP 380 implementation #14230) +# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) +# Python 3.4a1 3250 (evaluate positional default arguments before +# keyword-only defaults #16967) +# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override +# free vars #17853) +# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) +# Python 3.4a1 3280 (remove implicit class argument) +# Python 3.4a4 3290 (changes to __qualname__ computation #19301) +# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) +# Python 3.4rc2 3310 (alter __qualname__ computation #20625) +# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) +# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) +# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) +# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) +# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) +# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) +# Python 3.6a2 3370 (16 bit wordcode #26647) +# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) +# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE +# #27095) +# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) +# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes +# #27985) +# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL + #27213) +# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) +# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) +# Python 3.6rc1 3379 (more thorough __class__ validation #23722) +# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) +# Python 3.7a2 3391 (update GET_AITER #31709) +# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) +# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) +# Python 3.7b5 3394 (restored docstring as the first stmt in the body; +# this might affected the first line number #32911) +# Python 3.8a1 3400 (move frame block handling to compiler #17611) +# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) +# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) +# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict +# comprehensions #35224) +# Python 3.8b2 3412 (Swap the position of positional args and positional +# only args in ast.arguments #37593) +# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) +# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) +# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) +# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) +# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) +# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) +# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) +# Python 3.10a1 3430 (Make 'annotations' future by default) +# Python 3.10a1 3431 (New line number table format -- PEP 626) +# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) +# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) +# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) +# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). +# Python 3.10b1 3436 (Add GEN_START bytecode #43683) +# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) +# Python 3.10b1 3438 Safer line number table handling. +# Python 3.10b1 3439 (Add ROT_N) +# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) +# Python 3.11a1 3451 (Add CALL_METHOD_KW) +# Python 3.11a1 3452 (drop nlocals from marshaled code objects) +# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) +# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) +# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) +# Python 3.11a1 3456 (interleave cell args bpo-43693) +# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) +# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) +# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) +# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) +# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) +# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change +# MATCH_CLASS and MATCH_KEYS, and add COPY) +# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the +# active exception) +# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into +# BINARY_OP) +# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) +# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) +# Python 3.11a4 3467 (Change CALL_xxx opcodes) +# Python 3.11a4 3468 (Add SEND opcode) +# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) +# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) +# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) +# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) +# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) +# Python 3.11a4 3474 (Add RESUME opcode) +# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) +# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) +# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and +# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) +# Python 3.11a5 3478 (New CALL opcodes) +# Python 3.11a5 3479 (Add PUSH_NULL opcode) +# Python 3.11a5 3480 (New CALL opcodes, second iteration) +# Python 3.11a5 3481 (Use inline cache for BINARY_OP) +# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) +# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) +# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and +# STORE_ATTR) +# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) +# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) +# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) +# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) +# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) +# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) +# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, +# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) +# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) +# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) +# Python 3.11a7 3494 (New location info table) +# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) +# Python 3.12a1 3500 (Remove PRECALL opcode) +# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) +# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) +# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) +# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) +# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) +# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) +# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) +# Python 3.12a1 3508 (Add CLEANUP_THROW) +# Python 3.12a1 3509 (Conditional jumps only jump forward) +# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) +# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) +# Python 3.12a2 3512 (Remove all unused consts from code objects) +# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) +# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) +# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) +# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) +# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) +# Python 3.12a6 3518 (Add RETURN_CONST instruction) +# Python 3.12a6 3519 (Modify SEND instruction) +# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) +# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) +# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) +# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) +# Python 3.12b1 3525 (Shrink the CALL caches) +# Python 3.12b1 3526 (Add instrumentation support) +# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) +# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) +# Python 3.12b1 3529 (Inline list/dict/set comprehensions) +# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) +# Python 3.12b1 3531 (Add PEP 695 changes) + +# Python 3.13 will start with 3550 + +# Please don't copy-paste the same pre-release tag for new entries above!!! +# You should always use the *upcoming* tag. For example, if 3.12a6 came out +# a week ago, I should put "Python 3.12a7" next to my new magic number. + +# MAGIC must change whenever the bytecode emitted by the compiler may no +# longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). +# +# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. +# +# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array +# in PC/launcher.c must also be updated. + +MAGIC_NUMBER = (3531).to_bytes(2, 'little') + b'\r\n' + +_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c + +_PYCACHE = '__pycache__' +_OPT = 'opt-' + +SOURCE_SUFFIXES = ['.py'] +if _MS_WINDOWS: + SOURCE_SUFFIXES.append('.pyw') + +EXTENSION_SUFFIXES = _imp.extension_suffixes() + +BYTECODE_SUFFIXES = ['.pyc'] +# Deprecated. +DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES + +def cache_from_source(path, debug_override=None, *, optimization=None): + """Given the path to a .py file, return the path to its .pyc file. + + The .py file does not need to exist; this simply returns the path to the + .pyc file calculated as if the .py file were imported. + + The 'optimization' parameter controls the presumed optimization level of + the bytecode file. If 'optimization' is not None, the string representation + of the argument is taken and verified to be alphanumeric (else ValueError + is raised). + + The debug_override parameter is deprecated. If debug_override is not None, + a True value is the same as setting 'optimization' to the empty string + while a False value is equivalent to setting 'optimization' to '1'. + + If sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + if debug_override is not None: + _warnings.warn('the debug_override parameter is deprecated; use ' + "'optimization' instead", DeprecationWarning) + if optimization is not None: + message = 'debug_override or optimization must be set to None' + raise TypeError(message) + optimization = '' if debug_override else 1 + path = _os.fspath(path) + head, tail = _path_split(path) + base, sep, rest = tail.rpartition('.') + tag = sys.implementation.cache_tag + if tag is None: + raise NotImplementedError('sys.implementation.cache_tag is None') + almost_filename = ''.join([(base if base else rest), sep, tag]) + if optimization is None: + if sys.flags.optimize == 0: + optimization = '' + else: + optimization = sys.flags.optimize + optimization = str(optimization) + if optimization != '': + if not optimization.isalnum(): + raise ValueError(f'{optimization!r} is not alphanumeric') + almost_filename = f'{almost_filename}.{_OPT}{optimization}' + filename = almost_filename + BYTECODE_SUFFIXES[0] + if sys.pycache_prefix is not None: + # We need an absolute path to the py file to avoid the possibility of + # collisions within sys.pycache_prefix, if someone has two different + # `foo/bar.py` on their system and they import both of them using the + # same sys.pycache_prefix. Let's say sys.pycache_prefix is + # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first + # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative + # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an + # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`. + head = _path_abspath(head) + + # Strip initial drive from a Windows path. We know we have an absolute + # path here, so the second part of the check rules out a POSIX path that + # happens to contain a colon at the second character. + if head[1] == ':' and head[0] not in path_separators: + head = head[2:] + + # Strip initial path separator from `head` to complete the conversion + # back to a root-relative path before joining. + return _path_join( + sys.pycache_prefix, + head.lstrip(path_separators), + filename, + ) + return _path_join(head, _PYCACHE, filename) + + +def source_from_cache(path): + """Given the path to a .pyc. file, return the path to its .py file. + + The .pyc file does not need to exist; this simply returns the path to + the .py file calculated to correspond to the .pyc file. If path does + not conform to PEP 3147/488 format, ValueError will be raised. If + sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + if sys.implementation.cache_tag is None: + raise NotImplementedError('sys.implementation.cache_tag is None') + path = _os.fspath(path) + head, pycache_filename = _path_split(path) + found_in_pycache_prefix = False + if sys.pycache_prefix is not None: + stripped_path = sys.pycache_prefix.rstrip(path_separators) + if head.startswith(stripped_path + path_sep): + head = head[len(stripped_path):] + found_in_pycache_prefix = True + if not found_in_pycache_prefix: + head, pycache = _path_split(head) + if pycache != _PYCACHE: + raise ValueError(f'{_PYCACHE} not bottom-level directory in ' + f'{path!r}') + dot_count = pycache_filename.count('.') + if dot_count not in {2, 3}: + raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}') + elif dot_count == 3: + optimization = pycache_filename.rsplit('.', 2)[-2] + if not optimization.startswith(_OPT): + raise ValueError("optimization portion of filename does not start " + f"with {_OPT!r}") + opt_level = optimization[len(_OPT):] + if not opt_level.isalnum(): + raise ValueError(f"optimization level {optimization!r} is not an " + "alphanumeric value") + base_filename = pycache_filename.partition('.')[0] + return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) + + +def _get_sourcefile(bytecode_path): + """Convert a bytecode file path to a source path (if possible). + + This function exists purely for backwards-compatibility for + PyImport_ExecCodeModuleWithFilenames() in the C API. + + """ + if len(bytecode_path) == 0: + return None + rest, _, extension = bytecode_path.rpartition('.') + if not rest or extension.lower()[-3:-1] != 'py': + return bytecode_path + try: + source_path = source_from_cache(bytecode_path) + except (NotImplementedError, ValueError): + source_path = bytecode_path[:-1] + return source_path if _path_isfile(source_path) else bytecode_path + + +def _get_cached(filename): + if filename.endswith(tuple(SOURCE_SUFFIXES)): + try: + return cache_from_source(filename) + except NotImplementedError: + pass + elif filename.endswith(tuple(BYTECODE_SUFFIXES)): + return filename + else: + return None + + +def _calc_mode(path): + """Calculate the mode permissions for a bytecode file.""" + try: + mode = _path_stat(path).st_mode + except OSError: + mode = 0o666 + # We always ensure write access so we can update cached files + # later even when the source files are read-only on Windows (#6074) + mode |= 0o200 + return mode + + +def _check_name(method): + """Decorator to verify that the module being requested matches the one the + loader can handle. + + The first argument (self) must define _name which the second argument is + compared against. If the comparison fails then ImportError is raised. + + """ + def _check_name_wrapper(self, name=None, *args, **kwargs): + if name is None: + name = self.name + elif self.name != name: + raise ImportError('loader for %s cannot handle %s' % + (self.name, name), name=name) + return method(self, name, *args, **kwargs) + + # FIXME: @_check_name is used to define class methods before the + # _bootstrap module is set by _set_bootstrap_module(). + if _bootstrap is not None: + _wrap = _bootstrap._wrap + else: + def _wrap(new, old): + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) + new.__dict__.update(old.__dict__) + + _wrap(_check_name_wrapper, method) + return _check_name_wrapper + + +def _classify_pyc(data, name, exc_details): + """Perform basic validity checking of a pyc header and return the flags field, + which determines how the pyc should be further validated against the source. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required, though.) + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + ImportError is raised when the magic number is incorrect or when the flags + field is invalid. EOFError is raised when the data is found to be truncated. + + """ + magic = data[:4] + if magic != MAGIC_NUMBER: + message = f'bad magic number in {name!r}: {magic!r}' + _bootstrap._verbose_message('{}', message) + raise ImportError(message, **exc_details) + if len(data) < 16: + message = f'reached EOF while reading pyc header of {name!r}' + _bootstrap._verbose_message('{}', message) + raise EOFError(message) + flags = _unpack_uint32(data[4:8]) + # Only the first two flags are defined. + if flags & ~0b11: + message = f'invalid flags {flags!r} in {name!r}' + raise ImportError(message, **exc_details) + return flags + + +def _validate_timestamp_pyc(data, source_mtime, source_size, name, + exc_details): + """Validate a pyc against the source last-modified time. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_mtime* is the last modified timestamp of the source file. + + *source_size* is None or the size of the source file in bytes. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF): + message = f'bytecode is stale for {name!r}' + _bootstrap._verbose_message('{}', message) + raise ImportError(message, **exc_details) + if (source_size is not None and + _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)): + raise ImportError(f'bytecode is stale for {name!r}', **exc_details) + + +def _validate_hash_pyc(data, source_hash, name, exc_details): + """Validate a hash-based pyc by checking the real source hash against the one in + the pyc header. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_hash* is the importlib.util.source_hash() of the source file. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if data[8:16] != source_hash: + raise ImportError( + f'hash in bytecode doesn\'t match hash of source {name!r}', + **exc_details, + ) + + +def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): + """Compile bytecode as found in a pyc.""" + code = marshal.loads(data) + if isinstance(code, _code_type): + _bootstrap._verbose_message('code object from {!r}', bytecode_path) + if source_path is not None: + _imp._fix_co_filename(code, source_path) + return code + else: + raise ImportError(f'Non-code object in {bytecode_path!r}', + name=name, path=bytecode_path) + + +def _code_to_timestamp_pyc(code, mtime=0, source_size=0): + "Produce the data for a timestamp-based pyc." + data = bytearray(MAGIC_NUMBER) + data.extend(_pack_uint32(0)) + data.extend(_pack_uint32(mtime)) + data.extend(_pack_uint32(source_size)) + data.extend(marshal.dumps(code)) + return data + + +def _code_to_hash_pyc(code, source_hash, checked=True): + "Produce the data for a hash-based pyc." + data = bytearray(MAGIC_NUMBER) + flags = 0b1 | checked << 1 + data.extend(_pack_uint32(flags)) + assert len(source_hash) == 8 + data.extend(source_hash) + data.extend(marshal.dumps(code)) + return data + + +def decode_source(source_bytes): + """Decode bytes representing source code and return the string. + + Universal newline support is used in the decoding. + """ + import tokenize # To avoid bootstrap issues. + source_bytes_readline = _io.BytesIO(source_bytes).readline + encoding = tokenize.detect_encoding(source_bytes_readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) + + +# Module specifications ####################################################### + +_POPULATE = object() + + +def spec_from_file_location(name, location=None, *, loader=None, + submodule_search_locations=_POPULATE): + """Return a module spec based on a file location. + + To indicate that the module is a package, set + submodule_search_locations to a list of directory paths. An + empty list is sufficient, though its not otherwise useful to the + import system. + + The loader must take a spec as its only __init__() arg. + + """ + if location is None: + # The caller may simply want a partially populated location- + # oriented spec. So we set the location to a bogus value and + # fill in as much as we can. + location = '<unknown>' + if hasattr(loader, 'get_filename'): + # ExecutionLoader + try: + location = loader.get_filename(name) + except ImportError: + pass + else: + location = _os.fspath(location) + try: + location = _path_abspath(location) + except OSError: + pass + + # If the location is on the filesystem, but doesn't actually exist, + # we could return None here, indicating that the location is not + # valid. However, we don't have a good way of testing since an + # indirect location (e.g. a zip file or URL) will look like a + # non-existent file relative to the filesystem. + + spec = _bootstrap.ModuleSpec(name, loader, origin=location) + spec._set_fileattr = True + + # Pick a loader if one wasn't provided. + if loader is None: + for loader_class, suffixes in _get_supported_file_loaders(): + if location.endswith(tuple(suffixes)): + loader = loader_class(name, location) + spec.loader = loader + break + else: + return None + + # Set submodule_search_paths appropriately. + if submodule_search_locations is _POPULATE: + # Check the loader. + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + spec.submodule_search_locations = [] + else: + spec.submodule_search_locations = submodule_search_locations + if spec.submodule_search_locations == []: + if location: + dirname = _path_split(location)[0] + spec.submodule_search_locations.append(dirname) + + return spec + + +def _bless_my_loader(module_globals): + """Helper function for _warnings.c + + See GH#97850 for details. + """ + # 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and + # that use case only has the module globals. This function could be + # extended to accept either that or a module object. However, in the + # latter case, it would be better to raise certain exceptions when looking + # at a module, which should have either a __loader__ or __spec__.loader. + # For backward compatibility, it is possible that we'll get an empty + # dictionary for the module globals, and that cannot raise an exception. + if not isinstance(module_globals, dict): + return None + + missing = object() + loader = module_globals.get('__loader__', None) + spec = module_globals.get('__spec__', missing) + + if loader is None: + if spec is missing: + # If working with a module: + # raise AttributeError('Module globals is missing a __spec__') + return None + elif spec is None: + raise ValueError('Module globals is missing a __spec__.loader') + + spec_loader = getattr(spec, 'loader', missing) + + if spec_loader in (missing, None): + if loader is None: + exc = AttributeError if spec_loader is missing else ValueError + raise exc('Module globals is missing a __spec__.loader') + _warnings.warn( + 'Module globals is missing a __spec__.loader', + DeprecationWarning) + spec_loader = loader + + assert spec_loader is not None + if loader is not None and loader != spec_loader: + _warnings.warn( + 'Module globals; __loader__ != __spec__.loader', + DeprecationWarning) + return loader + + return spec_loader + + +# Loaders ##################################################################### + +class WindowsRegistryFinder: + + """Meta path finder for modules declared in the Windows registry.""" + + REGISTRY_KEY = ( + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}') + REGISTRY_KEY_DEBUG = ( + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}\\Debug') + DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES) + + @staticmethod + def _open_registry(key): + try: + return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key) + except OSError: + return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) + + @classmethod + def _search_registry(cls, fullname): + if cls.DEBUG_BUILD: + registry_key = cls.REGISTRY_KEY_DEBUG + else: + registry_key = cls.REGISTRY_KEY + key = registry_key.format(fullname=fullname, + sys_version='%d.%d' % sys.version_info[:2]) + try: + with cls._open_registry(key) as hkey: + filepath = winreg.QueryValue(hkey, '') + except OSError: + return None + return filepath + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + filepath = cls._search_registry(fullname) + if filepath is None: + return None + try: + _path_stat(filepath) + except OSError: + return None + for loader, suffixes in _get_supported_file_loaders(): + if filepath.endswith(tuple(suffixes)): + spec = _bootstrap.spec_from_loader(fullname, + loader(fullname, filepath), + origin=filepath) + return spec + + +class _LoaderBasics: + + """Base class of common code needed by both SourceLoader and + SourcelessFileLoader.""" + + def is_package(self, fullname): + """Concrete implementation of InspectLoader.is_package by checking if + the path returned by get_filename has a filename of '__init__.py'.""" + filename = _path_split(self.get_filename(fullname))[1] + filename_base = filename.rsplit('.', 1)[0] + tail_name = fullname.rpartition('.')[2] + return filename_base == '__init__' and tail_name != '__init__' + + def create_module(self, spec): + """Use default semantics for module creation.""" + + def exec_module(self, module): + """Execute the module.""" + code = self.get_code(module.__name__) + if code is None: + raise ImportError(f'cannot load module {module.__name__!r} when ' + 'get_code() returns None') + _bootstrap._call_with_frames_removed(exec, code, module.__dict__) + + def load_module(self, fullname): + """This method is deprecated.""" + # Warning implemented in _load_module_shim(). + return _bootstrap._load_module_shim(self, fullname) + + +class SourceLoader(_LoaderBasics): + + def path_mtime(self, path): + """Optional method that returns the modification time (an int) for the + specified path (a str). + + Raises OSError when the path cannot be handled. + """ + raise OSError + + def path_stats(self, path): + """Optional method returning a metadata dict for the specified + path (a str). + + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + + Implementing this method allows the loader to read bytecode files. + Raises OSError when the path cannot be handled. + """ + return {'mtime': self.path_mtime(path)} + + def _cache_bytecode(self, source_path, cache_path, data): + """Optional method which writes data (bytes) to a file path (a str). + + Implementing this method allows for the writing of bytecode files. + + The source path is needed in order to correctly transfer permissions + """ + # For backwards compatibility, we delegate to set_data() + return self.set_data(cache_path, data) + + def set_data(self, path, data): + """Optional method which writes data (bytes) to a file path (a str). + + Implementing this method allows for the writing of bytecode files. + """ + + + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except OSError as exc: + raise ImportError('source not available through get_data()', + name=fullname) from exc + return decode_source(source_bytes) + + def source_to_code(self, data, path, *, _optimize=-1): + """Return the code object compiled from source. + + The 'data' argument can be any object type that compile() supports. + """ + return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', + dont_inherit=True, optimize=_optimize) + + def get_code(self, fullname): + """Concrete implementation of InspectLoader.get_code. + + Reading of bytecode requires path_stats to be implemented. To write + bytecode, set_data must also be implemented. + + """ + source_path = self.get_filename(fullname) + source_mtime = None + source_bytes = None + source_hash = None + hash_based = False + check_source = True + try: + bytecode_path = cache_from_source(source_path) + except NotImplementedError: + bytecode_path = None + else: + try: + st = self.path_stats(source_path) + except OSError: + pass + else: + source_mtime = int(st['mtime']) + try: + data = self.get_data(bytecode_path) + except OSError: + pass + else: + exc_details = { + 'name': fullname, + 'path': bytecode_path, + } + try: + flags = _classify_pyc(data, fullname, exc_details) + bytes_data = memoryview(data)[16:] + hash_based = flags & 0b1 != 0 + if hash_based: + check_source = flags & 0b10 != 0 + if (_imp.check_hash_based_pycs != 'never' and + (check_source or + _imp.check_hash_based_pycs == 'always')): + source_bytes = self.get_data(source_path) + source_hash = _imp.source_hash( + _RAW_MAGIC_NUMBER, + source_bytes, + ) + _validate_hash_pyc(data, source_hash, fullname, + exc_details) + else: + _validate_timestamp_pyc( + data, + source_mtime, + st['size'], + fullname, + exc_details, + ) + except (ImportError, EOFError): + pass + else: + _bootstrap._verbose_message('{} matches {}', bytecode_path, + source_path) + return _compile_bytecode(bytes_data, name=fullname, + bytecode_path=bytecode_path, + source_path=source_path) + if source_bytes is None: + source_bytes = self.get_data(source_path) + code_object = self.source_to_code(source_bytes, source_path) + _bootstrap._verbose_message('code object from {}', source_path) + if (not sys.dont_write_bytecode and bytecode_path is not None and + source_mtime is not None): + if hash_based: + if source_hash is None: + source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_bytes) + data = _code_to_hash_pyc(code_object, source_hash, check_source) + else: + data = _code_to_timestamp_pyc(code_object, source_mtime, + len(source_bytes)) + try: + self._cache_bytecode(source_path, bytecode_path, data) + except NotImplementedError: + pass + return code_object + + +class FileLoader: + + """Base file loader class which implements the loader protocol methods that + require file system usage.""" + + def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" + self.name = fullname + self.path = path + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + + @_check_name + def load_module(self, fullname): + """Load a module from a file. + + This method is deprecated. Use exec_module() instead. + + """ + # The only reason for this method is for the name check. + # Issue #14857: Avoid the zero-argument form of super so the implementation + # of that form can be updated without breaking the frozen module. + return super(FileLoader, self).load_module(fullname) + + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self.path + + def get_data(self, path): + """Return the data from path as raw bytes.""" + if isinstance(self, (SourceLoader, ExtensionFileLoader)): + with _io.open_code(str(path)) as file: + return file.read() + else: + with _io.FileIO(path, 'r') as file: + return file.read() + + @_check_name + def get_resource_reader(self, module): + from importlib.readers import FileReader + return FileReader(self) + + +class SourceFileLoader(FileLoader, SourceLoader): + + """Concrete implementation of SourceLoader using the file system.""" + + def path_stats(self, path): + """Return the metadata for the path.""" + st = _path_stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} + + def _cache_bytecode(self, source_path, bytecode_path, data): + # Adapt between the two APIs + mode = _calc_mode(source_path) + return self.set_data(bytecode_path, data, _mode=mode) + + def set_data(self, path, data, *, _mode=0o666): + """Write bytes data to a file.""" + parent, filename = _path_split(path) + path_parts = [] + # Figure out what directories are missing. + while parent and not _path_isdir(parent): + parent, part = _path_split(parent) + path_parts.append(part) + # Create needed directories. + for part in reversed(path_parts): + parent = _path_join(parent, part) + try: + _os.mkdir(parent) + except FileExistsError: + # Probably another Python process already created the dir. + continue + except OSError as exc: + # Could be a permission error, read-only filesystem: just forget + # about writing the data. + _bootstrap._verbose_message('could not create {!r}: {!r}', + parent, exc) + return + try: + _write_atomic(path, data, _mode) + _bootstrap._verbose_message('created {!r}', path) + except OSError as exc: + # Same as above: just don't write the bytecode. + _bootstrap._verbose_message('could not create {!r}: {!r}', path, + exc) + + +class SourcelessFileLoader(FileLoader, _LoaderBasics): + + """Loader which handles sourceless file imports.""" + + def get_code(self, fullname): + path = self.get_filename(fullname) + data = self.get_data(path) + # Call _classify_pyc to do basic validation of the pyc but ignore the + # result. There's no source to check against. + exc_details = { + 'name': fullname, + 'path': path, + } + _classify_pyc(data, fullname, exc_details) + return _compile_bytecode( + memoryview(data)[16:], + name=fullname, + bytecode_path=path, + ) + + def get_source(self, fullname): + """Return None as there is no source code.""" + return None + + +class ExtensionFileLoader(FileLoader, _LoaderBasics): + + """Loader for extension modules. + + The constructor is designed to work with FileFinder. + + """ + + def __init__(self, name, path): + self.name = name + self.path = path + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + + def create_module(self, spec): + """Create an uninitialized extension module""" + module = _bootstrap._call_with_frames_removed( + _imp.create_dynamic, spec) + _bootstrap._verbose_message('extension module {!r} loaded from {!r}', + spec.name, self.path) + return module + + def exec_module(self, module): + """Initialize an extension module""" + _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) + _bootstrap._verbose_message('extension module {!r} executed from {!r}', + self.name, self.path) + + def is_package(self, fullname): + """Return True if the extension module is a package.""" + file_name = _path_split(self.path)[1] + return any(file_name == '__init__' + suffix + for suffix in EXTENSION_SUFFIXES) + + def get_code(self, fullname): + """Return None as an extension module cannot create a code object.""" + return None + + def get_source(self, fullname): + """Return None as extension modules have no source code.""" + return None + + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self.path + + +class _NamespacePath: + """Represents a namespace package's path. It uses the module name + to find its parent module, and from there it looks up the parent's + __path__. When this changes, the module's own path is recomputed, + using path_finder. For top-level modules, the parent module's path + is sys.path.""" + + # When invalidate_caches() is called, this epoch is incremented + # https://bugs.python.org/issue45703 + _epoch = 0 + + def __init__(self, name, path, path_finder): + self._name = name + self._path = path + self._last_parent_path = tuple(self._get_parent_path()) + self._last_epoch = self._epoch + self._path_finder = path_finder + + def _find_parent_path_names(self): + """Returns a tuple of (parent-module-name, parent-path-attr-name)""" + parent, dot, me = self._name.rpartition('.') + if dot == '': + # This is a top-level module. sys.path contains the parent path. + return 'sys', 'path' + # Not a top-level module. parent-module.__path__ contains the + # parent path. + return parent, '__path__' + + def _get_parent_path(self): + parent_module_name, path_attr_name = self._find_parent_path_names() + return getattr(sys.modules[parent_module_name], path_attr_name) + + def _recalculate(self): + # If the parent's path has changed, recalculate _path + parent_path = tuple(self._get_parent_path()) # Make a copy + if parent_path != self._last_parent_path or self._epoch != self._last_epoch: + spec = self._path_finder(self._name, parent_path) + # Note that no changes are made if a loader is returned, but we + # do remember the new parent path + if spec is not None and spec.loader is None: + if spec.submodule_search_locations: + self._path = spec.submodule_search_locations + self._last_parent_path = parent_path # Save the copy + self._last_epoch = self._epoch + return self._path + + def __iter__(self): + return iter(self._recalculate()) + + def __getitem__(self, index): + return self._recalculate()[index] + + def __setitem__(self, index, path): + self._path[index] = path + + def __len__(self): + return len(self._recalculate()) + + def __repr__(self): + return f'_NamespacePath({self._path!r})' + + def __contains__(self, item): + return item in self._recalculate() + + def append(self, item): + self._path.append(item) + + +# This class is actually exposed publicly in a namespace package's __loader__ +# attribute, so it should be available through a non-private name. +# https://github.com/python/cpython/issues/92054 +class NamespaceLoader: + def __init__(self, name, path, path_finder): + self._path = _NamespacePath(name, path, path_finder) + + def is_package(self, fullname): + return True + + def get_source(self, fullname): + return '' + + def get_code(self, fullname): + return compile('', '<string>', 'exec', dont_inherit=True) + + def create_module(self, spec): + """Use default semantics for module creation.""" + + def exec_module(self, module): + pass + + def load_module(self, fullname): + """Load a namespace module. + + This method is deprecated. Use exec_module() instead. + + """ + # The import system never calls this method. + _bootstrap._verbose_message('namespace module loaded with path {!r}', + self._path) + # Warning implemented in _load_module_shim(). + return _bootstrap._load_module_shim(self, fullname) + + def get_resource_reader(self, module): + from importlib.readers import NamespaceReader + return NamespaceReader(self._path) + + +# We use this exclusively in module_from_spec() for backward-compatibility. +_NamespaceLoader = NamespaceLoader + + +# Finders ##################################################################### + +class PathFinder: + + """Meta path finder for sys.path and package __path__ attributes.""" + + @staticmethod + def invalidate_caches(): + """Call the invalidate_caches() method on all path entry finders + stored in sys.path_importer_caches (where implemented).""" + for name, finder in list(sys.path_importer_cache.items()): + # Drop entry if finder name is a relative path. The current + # working directory may have changed. + if finder is None or not _path_isabs(name): + del sys.path_importer_cache[name] + elif hasattr(finder, 'invalidate_caches'): + finder.invalidate_caches() + # Also invalidate the caches of _NamespacePaths + # https://bugs.python.org/issue45703 + _NamespacePath._epoch += 1 + + @staticmethod + def _path_hooks(path): + """Search sys.path_hooks for a finder for 'path'.""" + if sys.path_hooks is not None and not sys.path_hooks: + _warnings.warn('sys.path_hooks is empty', ImportWarning) + for hook in sys.path_hooks: + try: + return hook(path) + except ImportError: + continue + else: + return None + + @classmethod + def _path_importer_cache(cls, path): + """Get the finder for the path entry from sys.path_importer_cache. + + If the path entry is not in the cache, find the appropriate finder + and cache it. If no finder is available, store None. + + """ + if path == '': + try: + path = _os.getcwd() + except FileNotFoundError: + # Don't cache the failure as the cwd can easily change to + # a valid directory later on. + return None + try: + finder = sys.path_importer_cache[path] + except KeyError: + finder = cls._path_hooks(path) + sys.path_importer_cache[path] = finder + return finder + + @classmethod + def _get_spec(cls, fullname, path, target=None): + """Find the loader or namespace_path for this module/package name.""" + # If this ends up being a namespace package, namespace_path is + # the list of paths that will become its __path__ + namespace_path = [] + for entry in path: + if not isinstance(entry, str): + continue + finder = cls._path_importer_cache(entry) + if finder is not None: + spec = finder.find_spec(fullname, target) + if spec is None: + continue + if spec.loader is not None: + return spec + portions = spec.submodule_search_locations + if portions is None: + raise ImportError('spec missing loader') + # This is possibly part of a namespace package. + # Remember these path entries (if any) for when we + # create a namespace package, and continue iterating + # on path. + namespace_path.extend(portions) + else: + spec = _bootstrap.ModuleSpec(fullname, None) + spec.submodule_search_locations = namespace_path + return spec + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + """Try to find a spec for 'fullname' on sys.path or 'path'. + + The search is based on sys.path_hooks and sys.path_importer_cache. + """ + if path is None: + path = sys.path + spec = cls._get_spec(fullname, path, target) + if spec is None: + return None + elif spec.loader is None: + namespace_path = spec.submodule_search_locations + if namespace_path: + # We found at least one namespace path. Return a spec which + # can create the namespace package. + spec.origin = None + spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) + return spec + else: + return None + else: + return spec + + @staticmethod + def find_distributions(*args, **kwargs): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + from importlib.metadata import MetadataPathFinder + return MetadataPathFinder.find_distributions(*args, **kwargs) + + +class FileFinder: + + """File-based finder. + + Interactions with the file system are cached for performance, being + refreshed when the directory the finder is handling has been modified. + + """ + + def __init__(self, path, *loader_details): + """Initialize with the path to search on and a variable number of + 2-tuples containing the loader and the file suffixes the loader + recognizes.""" + loaders = [] + for loader, suffixes in loader_details: + loaders.extend((suffix, loader) for suffix in suffixes) + self._loaders = loaders + # Base (directory) path + if not path or path == '.': + self.path = _os.getcwd() + else: + self.path = _path_abspath(path) + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def invalidate_caches(self): + """Invalidate the directory mtime.""" + self._path_mtime = -1 + + def _get_spec(self, loader_class, fullname, path, smsl, target): + loader = loader_class(fullname, path) + return spec_from_file_location(fullname, path, loader=loader, + submodule_search_locations=smsl) + + def find_spec(self, fullname, target=None): + """Try to find a spec for the specified module. + + Returns the matching spec, or None if not found. + """ + is_namespace = False + tail_module = fullname.rpartition('.')[2] + try: + mtime = _path_stat(self.path or _os.getcwd()).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + # Check if the module is the name of a directory (and thus a package). + if cache_module in cache: + base_path = _path_join(self.path, tail_module) + for suffix, loader_class in self._loaders: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, [base_path], target) + else: + # If a namespace package, return the path if we don't + # find a module in the next section. + is_namespace = _path_isdir(base_path) + # Check for a file w/ a proper suffix exists. + for suffix, loader_class in self._loaders: + try: + full_path = _path_join(self.path, tail_module + suffix) + except ValueError: + return None + _bootstrap._verbose_message('trying {}', full_path, verbosity=2) + if cache_module + suffix in cache: + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, + None, target) + if is_namespace: + _bootstrap._verbose_message('possible namespace for {}', base_path) + spec = _bootstrap.ModuleSpec(fullname, None) + spec.submodule_search_locations = [base_path] + return spec + return None + + def _fill_cache(self): + """Fill the cache of potential modules and packages for this directory.""" + path = self.path + try: + contents = _os.listdir(path or _os.getcwd()) + except (FileNotFoundError, PermissionError, NotADirectoryError): + # Directory has either been removed, turned into a file, or made + # unreadable. + contents = [] + # We store two cached versions, to handle runtime changes of the + # PYTHONCASEOK environment variable. + if not sys.platform.startswith('win'): + self._path_cache = set(contents) + else: + # Windows users can import modules with case-insensitive file + # suffixes (for legacy reasons). Make the suffix lowercase here + # so it's done once instead of for every import. This is safe as + # the specified suffixes to check against are always specified in a + # case-sensitive manner. + lower_suffix_contents = set() + for item in contents: + name, dot, suffix = item.partition('.') + if dot: + new_name = f'{name}.{suffix.lower()}' + else: + new_name = name + lower_suffix_contents.add(new_name) + self._path_cache = lower_suffix_contents + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + self._relaxed_path_cache = {fn.lower() for fn in contents} + + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + + If the path called on the closure is not a directory, ImportError is + raised. + + """ + def path_hook_for_FileFinder(path): + """Path hook for importlib.machinery.FileFinder.""" + if not _path_isdir(path): + raise ImportError('only directories are supported', path=path) + return cls(path, *loader_details) + + return path_hook_for_FileFinder + + def __repr__(self): + return f'FileFinder({self.path!r})' + + +# Import setup ############################################################### + +def _fix_up_module(ns, name, pathname, cpathname=None): + # This function is used by PyImport_ExecCodeModuleObject(). + loader = ns.get('__loader__') + spec = ns.get('__spec__') + if not loader: + if spec: + loader = spec.loader + elif pathname == cpathname: + loader = SourcelessFileLoader(name, pathname) + else: + loader = SourceFileLoader(name, pathname) + if not spec: + spec = spec_from_file_location(name, pathname, loader=loader) + if cpathname: + spec.cached = _path_abspath(cpathname) + try: + ns['__spec__'] = spec + ns['__loader__'] = loader + ns['__file__'] = pathname + ns['__cached__'] = cpathname + except Exception: + # Not important enough to report. + pass + + +def _get_supported_file_loaders(): + """Returns a list of file-based module loaders. + + Each item is a tuple (loader, suffixes). + """ + extensions = ExtensionFileLoader, _imp.extension_suffixes() + source = SourceFileLoader, SOURCE_SUFFIXES + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES + return [extensions, source, bytecode] + + +def _set_bootstrap_module(_bootstrap_module): + global _bootstrap + _bootstrap = _bootstrap_module + + +def _install(_bootstrap_module): + """Install the path-based import components.""" + _set_bootstrap_module(_bootstrap_module) + supported_loaders = _get_supported_file_loaders() + sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) + sys.meta_path.append(PathFinder) diff --git a/contrib/tools/python3/Lib/importlib/abc.py b/contrib/tools/python3/Lib/importlib/abc.py new file mode 100644 index 0000000000..b56fa94eb9 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/abc.py @@ -0,0 +1,239 @@ +"""Abstract base classes related to import.""" +from . import _bootstrap_external +from . import machinery +try: + import _frozen_importlib +except ImportError as exc: + if exc.name != '_frozen_importlib': + raise + _frozen_importlib = None +try: + import _frozen_importlib_external +except ImportError: + _frozen_importlib_external = _bootstrap_external +from ._abc import Loader +import abc +import warnings + +from .resources import abc as _resources_abc + + +__all__ = [ + 'Loader', 'MetaPathFinder', 'PathEntryFinder', + 'ResourceLoader', 'InspectLoader', 'ExecutionLoader', + 'FileLoader', 'SourceLoader', +] + + +def __getattr__(name): + """ + For backwards compatibility, continue to make names + from _resources_abc available through this module. #93963 + """ + if name in _resources_abc.__all__: + obj = getattr(_resources_abc, name) + warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) + globals()[name] = obj + return obj + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + +def _register(abstract_cls, *classes): + for cls in classes: + abstract_cls.register(cls) + if _frozen_importlib is not None: + try: + frozen_cls = getattr(_frozen_importlib, cls.__name__) + except AttributeError: + frozen_cls = getattr(_frozen_importlib_external, cls.__name__) + abstract_cls.register(frozen_cls) + + +class MetaPathFinder(metaclass=abc.ABCMeta): + + """Abstract base class for import finders on sys.meta_path.""" + + # We don't define find_spec() here since that would break + # hasattr checks we do to support backward compatibility. + + def invalidate_caches(self): + """An optional method for clearing the finder's cache, if any. + This method is used by importlib.invalidate_caches(). + """ + +_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, + machinery.PathFinder, machinery.WindowsRegistryFinder) + + +class PathEntryFinder(metaclass=abc.ABCMeta): + + """Abstract base class for path entry finders used by PathFinder.""" + + def invalidate_caches(self): + """An optional method for clearing the finder's cache, if any. + This method is used by PathFinder.invalidate_caches(). + """ + +_register(PathEntryFinder, machinery.FileFinder) + + +class ResourceLoader(Loader): + + """Abstract base class for loaders which can return data from their + back-end storage. + + This ABC represents one of the optional protocols specified by PEP 302. + + """ + + @abc.abstractmethod + def get_data(self, path): + """Abstract method which when implemented should return the bytes for + the specified path. The path must be a str.""" + raise OSError + + +class InspectLoader(Loader): + + """Abstract base class for loaders which support inspection about the + modules they can load. + + This ABC represents one of the optional protocols specified by PEP 302. + + """ + + def is_package(self, fullname): + """Optional method which when implemented should return whether the + module is a package. The fullname is a str. Returns a bool. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + def get_code(self, fullname): + """Method which returns the code object for the module. + + The fullname is a str. Returns a types.CodeType if possible, else + returns None if a code object does not make sense + (e.g. built-in module). Raises ImportError if the module cannot be + found. + """ + source = self.get_source(fullname) + if source is None: + return None + return self.source_to_code(source) + + @abc.abstractmethod + def get_source(self, fullname): + """Abstract method which should return the source code for the + module. The fullname is a str. Returns a str. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + @staticmethod + def source_to_code(data, path='<string>'): + """Compile 'data' into a code object. + + The 'data' argument can be anything that compile() can handle. The'path' + argument should be where the data was retrieved (when applicable).""" + return compile(data, path, 'exec', dont_inherit=True) + + exec_module = _bootstrap_external._LoaderBasics.exec_module + load_module = _bootstrap_external._LoaderBasics.load_module + +_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.NamespaceLoader) + + +class ExecutionLoader(InspectLoader): + + """Abstract base class for loaders that wish to support the execution of + modules as scripts. + + This ABC represents one of the optional protocols specified in PEP 302. + + """ + + @abc.abstractmethod + def get_filename(self, fullname): + """Abstract method which should return the value that __file__ is to be + set to. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + def get_code(self, fullname): + """Method to return the code object for fullname. + + Should return None if not applicable (e.g. built-in module). + Raise ImportError if the module cannot be found. + """ + source = self.get_source(fullname) + if source is None: + return None + try: + path = self.get_filename(fullname) + except ImportError: + return self.source_to_code(source) + else: + return self.source_to_code(source, path) + +_register(ExecutionLoader, machinery.ExtensionFileLoader) + + +class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class partially implementing the ResourceLoader and + ExecutionLoader ABCs.""" + +_register(FileLoader, machinery.SourceFileLoader, + machinery.SourcelessFileLoader) + + +class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class for loading source code (and optionally any + corresponding bytecode). + + To support loading from source code, the abstractmethods inherited from + ResourceLoader and ExecutionLoader need to be implemented. To also support + loading from bytecode, the optional methods specified directly by this ABC + is required. + + Inherited abstractmethods not implemented in this ABC: + + * ResourceLoader.get_data + * ExecutionLoader.get_filename + + """ + + def path_mtime(self, path): + """Return the (int) modification time for the path (str).""" + if self.path_stats.__func__ is SourceLoader.path_stats: + raise OSError + return int(self.path_stats(path)['mtime']) + + def path_stats(self, path): + """Return a metadata dict for the source pointed to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + """ + if self.path_mtime.__func__ is SourceLoader.path_mtime: + raise OSError + return {'mtime': self.path_mtime(path)} + + def set_data(self, path, data): + """Write the bytes to the path (if possible). + + Accepts a str path and data as bytes. + + Any needed intermediary directories are to be created. If for some + reason the file cannot be written because of permissions, fail + silently. + """ + +_register(SourceLoader, machinery.SourceFileLoader) diff --git a/contrib/tools/python3/Lib/importlib/machinery.py b/contrib/tools/python3/Lib/importlib/machinery.py new file mode 100644 index 0000000000..d9a19a13f7 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/machinery.py @@ -0,0 +1,20 @@ +"""The machinery of importlib: finders, loaders, hooks, etc.""" + +from ._bootstrap import ModuleSpec +from ._bootstrap import BuiltinImporter +from ._bootstrap import FrozenImporter +from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, + EXTENSION_SUFFIXES) +from ._bootstrap_external import WindowsRegistryFinder +from ._bootstrap_external import PathFinder +from ._bootstrap_external import FileFinder +from ._bootstrap_external import SourceFileLoader +from ._bootstrap_external import SourcelessFileLoader +from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import NamespaceLoader + + +def all_suffixes(): + """Returns a list of all recognized module suffixes for this process""" + return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES diff --git a/contrib/tools/python3/Lib/importlib/metadata/__init__.py b/contrib/tools/python3/Lib/importlib/metadata/__init__.py new file mode 100644 index 0000000000..82e0ce1b28 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/__init__.py @@ -0,0 +1,965 @@ +import os +import re +import abc +import csv +import sys +import email +import pathlib +import zipfile +import operator +import textwrap +import warnings +import functools +import itertools +import posixpath +import contextlib +import collections +import inspect + +from . import _adapters, _meta +from ._collections import FreezableDefaultDict, Pair +from ._functools import method_cache, pass_none +from ._itertools import always_iterable, unique_everseen +from ._meta import PackageMetadata, SimplePath + +from contextlib import suppress +from importlib import import_module +from importlib.abc import MetaPathFinder +from itertools import starmap +from typing import List, Mapping, Optional, cast + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageMetadata', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'packages_distributions', + 'requires', + 'version', +] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + def __str__(self): + return f"No package metadata was found for {self.name}" + + @property + def name(self): + (name,) = self.args + return name + + +class Sectioned: + """ + A simple entry point config parser for performance + + >>> for item in Sectioned.read(Sectioned._sample): + ... print(item) + Pair(name='sec1', value='# comments ignored') + Pair(name='sec1', value='a = 1') + Pair(name='sec1', value='b = 2') + Pair(name='sec2', value='a = 2') + + >>> res = Sectioned.section_pairs(Sectioned._sample) + >>> item = next(res) + >>> item.name + 'sec1' + >>> item.value + Pair(name='a', value='1') + >>> item = next(res) + >>> item.value + Pair(name='b', value='2') + >>> item = next(res) + >>> item.name + 'sec2' + >>> item.value + Pair(name='a', value='2') + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + # comments ignored + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + @classmethod + def section_pairs(cls, text): + return ( + section._replace(value=Pair.parse(section.value)) + for section in cls.read(text, filter_=cls.valid) + if section.name is not None + ) + + @staticmethod + def read(text, filter_=None): + lines = filter(filter_, map(str.strip, text.splitlines())) + name = None + for value in lines: + section_match = value.startswith('[') and value.endswith(']') + if section_match: + name = value.strip('[]') + continue + yield Pair(name, value) + + @staticmethod + def valid(line): + return line and not line.startswith('#') + + +class DeprecatedTuple: + """ + Provide subscript item access for backward compatibility. + + >>> recwarn = getfixture('recwarn') + >>> ep = EntryPoint(name='name', value='value', group='group') + >>> ep[:] + ('name', 'value', 'group') + >>> ep[0] + 'name' + >>> len(recwarn) + 1 + """ + + # Do not remove prior to 2023-05-01 or Python 3.13 + _warn = functools.partial( + warnings.warn, + "EntryPoint tuple interface is deprecated. Access members by name.", + DeprecationWarning, + stacklevel=2, + ) + + def __getitem__(self, item): + self._warn() + return self._key()[item] + + +class EntryPoint(DeprecatedTuple): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + <https://packaging.python.org/specifications/entry-points/>`_ + for more information. + + >>> ep = EntryPoint( + ... name=None, group=None, value='package.module:attr [extra1, extra2]') + >>> ep.module + 'package.module' + >>> ep.attr + 'attr' + >>> ep.extras + ['extra1', 'extra2'] + """ + + pattern = re.compile( + r'(?P<module>[\w.]+)\s*' + r'(:\s*(?P<attr>[\w.]+)\s*)?' + r'((?P<extras>\[.*\])\s*)?$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + name: str + value: str + group: str + + dist: Optional['Distribution'] = None + + def __init__(self, name, value, group): + vars(self).update(name=name, value=value, group=group) + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + + @property + def extras(self): + match = self.pattern.match(self.value) + return re.findall(r'\w+', match.group('extras') or '') + + def _for(self, dist): + vars(self).update(dist=dist) + return self + + def matches(self, **params): + """ + EntryPoint matches the given parameters. + + >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]') + >>> ep.matches(group='foo') + True + >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]') + True + >>> ep.matches(group='foo', name='other') + False + >>> ep.matches() + True + >>> ep.matches(extras=['extra1', 'extra2']) + True + >>> ep.matches(module='bing') + True + >>> ep.matches(attr='bong') + True + """ + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + + def _key(self): + return self.name, self.value, self.group + + def __lt__(self, other): + return self._key() < other._key() + + def __eq__(self, other): + return self._key() == other._key() + + def __setattr__(self, name, value): + raise AttributeError("EntryPoint objects are immutable.") + + def __repr__(self): + return ( + f'EntryPoint(name={self.name!r}, value={self.value!r}, ' + f'group={self.group!r})' + ) + + def __hash__(self): + return hash(self._key()) + + +class EntryPoints(tuple): + """ + An immutable collection of selectable EntryPoint objects. + """ + + __slots__ = () + + def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ + try: + return next(iter(self.select(name=name))) + except StopIteration: + raise KeyError(name) + + def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ + return EntryPoints(ep for ep in self if ep.matches(**params)) + + @property + def names(self): + """ + Return the set of all names of all entry points. + """ + return {ep.name for ep in self} + + @property + def groups(self): + """ + Return the set of all groups of all entry points. + """ + return {ep.group for ep in self} + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @staticmethod + def _from_text(text): + return ( + EntryPoint(name=item.value.name, value=item.value.value, group=item.name) + for item in Sectioned.section_pairs(text or '') + ) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return f'<FileHash mode: {self.mode} value: {self.value}>' + + +class DeprecatedNonAbstract: + def __new__(cls, *args, **kwargs): + all_names = { + name for subclass in inspect.getmro(cls) for name in vars(subclass) + } + abstract = { + name + for name in all_names + if getattr(getattr(cls, name), '__isabstractmethod__', False) + } + if abstract: + warnings.warn( + f"Unimplemented abstract methods {abstract}", + DeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls) + + +class Distribution(DeprecatedNonAbstract): + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename) -> Optional[str]: + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name: str): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + :raises ValueError: When an invalid value is supplied for name. + """ + if not name: + raise ValueError("A distribution name is required.") + try: + return next(cls.discover(name=name)) + except StopIteration: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) for finder in sys.meta_path + ) + return filter(None, declared) + + @property + def metadata(self) -> _meta.PackageMetadata: + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + opt_text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + text = cast(str, opt_text) + return _adapters.Message(email.message_from_string(text)) + + @property + def name(self): + """Return the 'Name' metadata for the distribution package.""" + return self.metadata['Name'] + + @property + def _normalized_name(self): + """Return a normalized version of the name.""" + return Prepared.normalize(self.name) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info, or installed-files.txt or + SOURCES.txt for egg-info) is missing. + Result may be empty if the metadata exists but is empty. + """ + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + @pass_none + def make_files(lines): + return starmap(make_file, csv.reader(lines)) + + @pass_none + def skip_missing_files(package_paths): + return list(filter(lambda path: path.locate().exists(), package_paths)) + + return skip_missing_files( + make_files( + self._read_files_distinfo() + or self._read_files_egginfo_installed() + or self._read_files_egginfo_sources() + ) + ) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo_installed(self): + """ + Read installed-files.txt and return lines in a similar + CSV-parsable format as RECORD: each file must be placed + relative to the site-packages directory and must also be + quoted (since file names can contain literal commas). + + This file is written when the package is installed by pip, + but it might not be written for other installation methods. + Assume the file is accurate if it exists. + """ + text = self.read_text('installed-files.txt') + # Prepend the .egg-info/ subdir to the lines in this file. + # But this subdir is only available from PathDistribution's + # self._path. + subdir = getattr(self, '_path', None) + if not text or not subdir: + return + + paths = ( + (subdir / name) + .resolve() + .relative_to(self.locate_file('').resolve()) + .as_posix() + for name in text.splitlines() + ) + return map('"{}"'.format, paths) + + def _read_files_egginfo_sources(self): + """ + Read SOURCES.txt and return lines in a similar CSV-parsable + format as RECORD: each file name must be quoted (since it + might contain literal commas). + + Note that SOURCES.txt is not a reliable source for what + files are installed by a package. This file is generated + for a source archive, and the files that are present + there (e.g. setup.py) may not correctly reflect the files + that are present after the package has been installed. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return pass_none(self._deps_from_requires_text)(source) + + @classmethod + def _deps_from_requires_text(cls, source): + return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + + def make_condition(name): + return name and f'extra == "{name}"' + + def quoted_marker(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = f'({markers})' + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + def url_req_space(req): + """ + PEP 508 requires a space between the url_spec and the quoted_marker. + Ref python/importlib_metadata#357. + """ + # '@' is uniquely indicative of a url_req. + return ' ' * ('@' in req) + + for section in sections: + space = url_req_space(section.value) + yield section.value + space + quoted_marker(section.name) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The sequence of directory path that a distribution finder + should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + + >>> FastPath('').children() + ['...'] + """ + + @functools.lru_cache() # type: ignore + def __new__(cls, root): + return super().__new__(cls) + + def __init__(self, root): + self.root = root + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '.') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipfile.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) + + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with suppress(OSError): + return os.stat(self.root).st_mtime + self.lookup.cache_clear() + + @method_cache + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) + + for child in path.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos[normalized].append(path.joinpath(child)) + elif base_is_egg and low == "egg-info": + name = base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs[legacy_normalized].append(path.joinpath(child)) + + self.infos.freeze() + self.eggs.freeze() + + def search(self, prepared): + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + + normalized = None + legacy_normalized = None + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = self.normalize(name) + self.legacy_normalized = self.legacy_normalize(name) + + @staticmethod + def normalize(name): + """ + PEP 503 normalization plus dashes as underscores. + """ + return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') + + @staticmethod + def legacy_normalize(name): + """ + Normalize the package name as found in the convention in + older packaging tools versions and specs. + """ + return name.lower().replace('-', '_') + + def __bool__(self): + return bool(self.name) + + +class MetadataPathFinder(DistributionFinder): + @classmethod + def find_distributions(cls, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = cls._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + prepared = Prepared(name) + return itertools.chain.from_iterable( + path.search(prepared) for path in map(FastPath, paths) + ) + + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + + +class PathDistribution(Distribution): + def __init__(self, path: SimplePath): + """Construct a distribution. + + :param path: SimplePath indicating the metadata directory. + """ + self._path = path + + def read_text(self, filename): + with suppress( + FileNotFoundError, + IsADirectoryError, + KeyError, + NotADirectoryError, + PermissionError, + ): + return self._path.joinpath(filename).read_text(encoding='utf-8') + + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + @property + def _normalized_name(self): + """ + Performance optimization: where possible, resolve the + normalized name from the file system path. + """ + stem = os.path.basename(str(self._path)) + return ( + pass_none(Prepared.normalize)(self._name_from_stem(stem)) + or super()._normalized_name + ) + + @staticmethod + def _name_from_stem(stem): + """ + >>> PathDistribution._name_from_stem('foo-3.0.egg-info') + 'foo' + >>> PathDistribution._name_from_stem('CherryPy-3.0.dist-info') + 'CherryPy' + >>> PathDistribution._name_from_stem('face.egg-info') + 'face' + >>> PathDistribution._name_from_stem('foo.bar') + """ + filename, ext = os.path.splitext(stem) + if ext not in ('.dist-info', '.egg-info'): + return + name, sep, rest = filename.partition('-') + return name + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name) -> _meta.PackageMetadata: + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: A PackageMetadata containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +_unique = functools.partial( + unique_everseen, + key=operator.attrgetter('_normalized_name'), +) +""" +Wrapper for ``distributions`` to return unique distributions by name. +""" + + +def entry_points(**params) -> EntryPoints: + """Return EntryPoint objects for all installed packages. + + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + :return: EntryPoints for all installed packages. + """ + eps = itertools.chain.from_iterable( + dist.entry_points for dist in _unique(distributions()) + ) + return EntryPoints(eps).select(**params) + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> import collections.abc + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in _top_level_declared(dist) or _top_level_inferred(dist): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) + + +def _top_level_declared(dist): + return (dist.read_text('top_level.txt') or '').split() + + +def _top_level_inferred(dist): + opt_names = { + f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) + for f in always_iterable(dist.files) + } + + @pass_none + def importable_name(name): + return '.' not in name + + return filter(importable_name, opt_names) diff --git a/contrib/tools/python3/Lib/importlib/metadata/_adapters.py b/contrib/tools/python3/Lib/importlib/metadata/_adapters.py new file mode 100644 index 0000000000..6aed69a308 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_adapters.py @@ -0,0 +1,89 @@ +import functools +import warnings +import re +import textwrap +import email.message + +from ._text import FoldedCase + + +# Do not remove prior to 2024-01-01 or Python 3.14 +_warn = functools.partial( + warnings.warn, + "Implicit None on return values is deprecated and will raise KeyErrors.", + DeprecationWarning, + stacklevel=2, +) + + +class Message(email.message.Message): + multiple_use_keys = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + 'Dynamic', + ], + ) + ) + """ + Keys that may be indicated multiple times per PEP 566. + """ + + def __new__(cls, orig: email.message.Message): + res = super().__new__(cls) + vars(res).update(vars(orig)) + return res + + def __init__(self, *args, **kwargs): + self._headers = self._repair_headers() + + # suppress spurious error from mypy + def __iter__(self): + return super().__iter__() + + def __getitem__(self, item): + """ + Warn users that a ``KeyError`` can be expected when a + mising key is supplied. Ref python/importlib_metadata#371. + """ + res = super().__getitem__(item) + if res is None: + _warn() + return res + + def _repair_headers(self): + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + headers = [(key, redent(value)) for key, value in vars(self)['_headers']] + if self._payload: + headers.append(('Description', self.get_payload())) + return headers + + @property + def json(self): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + + def transform(key): + value = self.get_all(key) if key in self.multiple_use_keys else self[key] + if key == 'Keywords': + value = re.split(r'\s+', value) + tk = key.lower().replace('-', '_') + return tk, value + + return dict(map(transform, map(FoldedCase, self))) diff --git a/contrib/tools/python3/Lib/importlib/metadata/_collections.py b/contrib/tools/python3/Lib/importlib/metadata/_collections.py new file mode 100644 index 0000000000..cf0954e1a3 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_collections.py @@ -0,0 +1,30 @@ +import collections + + +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): + """ + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. + + >>> dd = FreezableDefaultDict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() + + +class Pair(collections.namedtuple('Pair', 'name value')): + @classmethod + def parse(cls, text): + return cls(*map(str.strip, text.split("=", 1))) diff --git a/contrib/tools/python3/Lib/importlib/metadata/_functools.py b/contrib/tools/python3/Lib/importlib/metadata/_functools.py new file mode 100644 index 0000000000..71f66bd03c --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_functools.py @@ -0,0 +1,104 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper + + +# From jaraco.functools 3.3 +def pass_none(func): + """ + Wrap func so it's not called if its first param is None + + >>> print_text = pass_none(print) + >>> print_text('text') + text + >>> print_text(None) + """ + + @functools.wraps(func) + def wrapper(param, *args, **kwargs): + if param is not None: + return func(param, *args, **kwargs) + + return wrapper diff --git a/contrib/tools/python3/Lib/importlib/metadata/_itertools.py b/contrib/tools/python3/Lib/importlib/metadata/_itertools.py new file mode 100644 index 0000000000..d4ca9b9140 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_itertools.py @@ -0,0 +1,73 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element + + +# copied from more_itertools 8.8 +def always_iterable(obj, base_type=(str, bytes)): + """If *obj* is iterable, return an iterator over its items:: + + >>> obj = (1, 2, 3) + >>> list(always_iterable(obj)) + [1, 2, 3] + + If *obj* is not iterable, return a one-item iterable containing *obj*:: + + >>> obj = 1 + >>> list(always_iterable(obj)) + [1] + + If *obj* is ``None``, return an empty iterable: + + >>> obj = None + >>> list(always_iterable(None)) + [] + + By default, binary and text strings are not considered iterable:: + + >>> obj = 'foo' + >>> list(always_iterable(obj)) + ['foo'] + + If *base_type* is set, objects for which ``isinstance(obj, base_type)`` + returns ``True`` won't be considered iterable. + + >>> obj = {'a': 1} + >>> list(always_iterable(obj)) # Iterate over the dict's keys + ['a'] + >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit + [{'a': 1}] + + Set *base_type* to ``None`` to avoid any special handling and treat objects + Python considers iterable as iterable: + + >>> obj = 'foo' + >>> list(always_iterable(obj, base_type=None)) + ['f', 'o', 'o'] + """ + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) diff --git a/contrib/tools/python3/Lib/importlib/metadata/_meta.py b/contrib/tools/python3/Lib/importlib/metadata/_meta.py new file mode 100644 index 0000000000..c9a7ef906a --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_meta.py @@ -0,0 +1,63 @@ +from typing import Protocol +from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload + + +_T = TypeVar("_T") + + +class PackageMetadata(Protocol): + def __len__(self) -> int: + ... # pragma: no cover + + def __contains__(self, item: str) -> bool: + ... # pragma: no cover + + def __getitem__(self, key: str) -> str: + ... # pragma: no cover + + def __iter__(self) -> Iterator[str]: + ... # pragma: no cover + + @overload + def get(self, name: str, failobj: None = None) -> Optional[str]: + ... # pragma: no cover + + @overload + def get(self, name: str, failobj: _T) -> Union[str, _T]: + ... # pragma: no cover + + # overload per python/importlib_metadata#435 + @overload + def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: + ... # pragma: no cover + + @overload + def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: + """ + Return all values associated with a possibly multi-valued key. + """ + + @property + def json(self) -> Dict[str, Union[str, List[str]]]: + """ + A JSON-compatible form of the metadata. + """ + + +class SimplePath(Protocol[_T]): + """ + A minimal subset of pathlib.Path required by PathDistribution. + """ + + def joinpath(self) -> _T: + ... # pragma: no cover + + def __truediv__(self, other: Union[str, _T]) -> _T: + ... # pragma: no cover + + @property + def parent(self) -> _T: + ... # pragma: no cover + + def read_text(self) -> str: + ... # pragma: no cover diff --git a/contrib/tools/python3/Lib/importlib/metadata/_text.py b/contrib/tools/python3/Lib/importlib/metadata/_text.py new file mode 100644 index 0000000000..c88cfbb234 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/metadata/_text.py @@ -0,0 +1,99 @@ +import re + +from ._functools import method_cache + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super().lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super().lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) diff --git a/contrib/tools/python3/Lib/importlib/readers.py b/contrib/tools/python3/Lib/importlib/readers.py new file mode 100644 index 0000000000..df7fb92e5c --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/readers.py @@ -0,0 +1,12 @@ +""" +Compatibility shim for .resources.readers as found on Python 3.10. + +Consumers that can rely on Python 3.11 should use the other +module directly. +""" + +from .resources.readers import ( + FileReader, ZipReader, MultiplexedPath, NamespaceReader, +) + +__all__ = ['FileReader', 'ZipReader', 'MultiplexedPath', 'NamespaceReader'] diff --git a/contrib/tools/python3/Lib/importlib/resources/__init__.py b/contrib/tools/python3/Lib/importlib/resources/__init__.py new file mode 100644 index 0000000000..34e3a9950c --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/__init__.py @@ -0,0 +1,36 @@ +"""Read resources contained within a package.""" + +from ._common import ( + as_file, + files, + Package, +) + +from ._legacy import ( + contents, + open_binary, + read_binary, + open_text, + read_text, + is_resource, + path, + Resource, +) + +from .abc import ResourceReader + + +__all__ = [ + 'Package', + 'Resource', + 'ResourceReader', + 'as_file', + 'contents', + 'files', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', +] diff --git a/contrib/tools/python3/Lib/importlib/resources/_adapters.py b/contrib/tools/python3/Lib/importlib/resources/_adapters.py new file mode 100644 index 0000000000..50688fbb66 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/_adapters.py @@ -0,0 +1,168 @@ +from contextlib import suppress +from io import TextIOWrapper + +from . import abc + + +class SpecLoaderAdapter: + """ + Adapt a package spec to adapt the underlying loader. + """ + + def __init__(self, spec, adapter=lambda spec: spec.loader): + self.spec = spec + self.loader = adapter(spec) + + def __getattr__(self, name): + return getattr(self.spec, name) + + +class TraversableResourcesLoader: + """ + Adapt a loader to provide TraversableResources. + """ + + def __init__(self, spec): + self.spec = spec + + def get_resource_reader(self, name): + return CompatibilityFiles(self.spec)._native() + + +def _io_wrapper(file, mode='r', *args, **kwargs): + if mode == 'r': + return TextIOWrapper(file, *args, **kwargs) + elif mode == 'rb': + return file + raise ValueError(f"Invalid mode value '{mode}', only 'r' and 'rb' are supported") + + +class CompatibilityFiles: + """ + Adapter for an existing or non-existent resource reader + to provide a compatibility .files(). + """ + + class SpecPath(abc.Traversable): + """ + Path tied to a module spec. + Can be read and exposes the resource reader children. + """ + + def __init__(self, spec, reader): + self._spec = spec + self._reader = reader + + def iterdir(self): + if not self._reader: + return iter(()) + return iter( + CompatibilityFiles.ChildPath(self._reader, path) + for path in self._reader.contents() + ) + + def is_file(self): + return False + + is_dir = is_file + + def joinpath(self, other): + if not self._reader: + return CompatibilityFiles.OrphanPath(other) + return CompatibilityFiles.ChildPath(self._reader, other) + + @property + def name(self): + return self._spec.name + + def open(self, mode='r', *args, **kwargs): + return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) + + class ChildPath(abc.Traversable): + """ + Path tied to a resource reader child. + Can be read but doesn't expose any meaningful children. + """ + + def __init__(self, reader, name): + self._reader = reader + self._name = name + + def iterdir(self): + return iter(()) + + def is_file(self): + return self._reader.is_resource(self.name) + + def is_dir(self): + return not self.is_file() + + def joinpath(self, other): + return CompatibilityFiles.OrphanPath(self.name, other) + + @property + def name(self): + return self._name + + def open(self, mode='r', *args, **kwargs): + return _io_wrapper( + self._reader.open_resource(self.name), mode, *args, **kwargs + ) + + class OrphanPath(abc.Traversable): + """ + Orphan path, not tied to a module spec or resource reader. + Can't be read and doesn't expose any meaningful children. + """ + + def __init__(self, *path_parts): + if len(path_parts) < 1: + raise ValueError('Need at least one path part to construct a path') + self._path = path_parts + + def iterdir(self): + return iter(()) + + def is_file(self): + return False + + is_dir = is_file + + def joinpath(self, other): + return CompatibilityFiles.OrphanPath(*self._path, other) + + @property + def name(self): + return self._path[-1] + + def open(self, mode='r', *args, **kwargs): + raise FileNotFoundError("Can't open orphan path") + + def __init__(self, spec): + self.spec = spec + + @property + def _reader(self): + with suppress(AttributeError): + return self.spec.loader.get_resource_reader(self.spec.name) + + def _native(self): + """ + Return the native reader if it supports files(). + """ + reader = self._reader + return reader if hasattr(reader, 'files') else self + + def __getattr__(self, attr): + return getattr(self._reader, attr) + + def files(self): + return CompatibilityFiles.SpecPath(self.spec, self._reader) + + +def wrap_spec(package): + """ + Construct a package spec with traversable compatibility + on the spec/loader/reader. + """ + return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/tools/python3/Lib/importlib/resources/_common.py b/contrib/tools/python3/Lib/importlib/resources/_common.py new file mode 100644 index 0000000000..a390253534 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/_common.py @@ -0,0 +1,207 @@ +import os +import pathlib +import tempfile +import functools +import contextlib +import types +import importlib +import inspect +import warnings +import itertools + +from typing import Union, Optional, cast +from .abc import ResourceReader, Traversable + +from ._adapters import wrap_spec + +Package = Union[types.ModuleType, str] +Anchor = Package + + +def package_to_anchor(func): + """ + Replace 'package' parameter as 'anchor' and warn about the change. + + Other errors should fall through. + + >>> files('a', 'b') + Traceback (most recent call last): + TypeError: files() takes from 0 to 1 positional arguments but 2 were given + """ + undefined = object() + + @functools.wraps(func) + def wrapper(anchor=undefined, package=undefined): + if package is not undefined: + if anchor is not undefined: + return func(anchor, package) + warnings.warn( + "First parameter to files is renamed to 'anchor'", + DeprecationWarning, + stacklevel=2, + ) + return func(package) + elif anchor is undefined: + return func() + return func(anchor) + + return wrapper + + +@package_to_anchor +def files(anchor: Optional[Anchor] = None) -> Traversable: + """ + Get a Traversable resource for an anchor. + """ + return from_package(resolve(anchor)) + + +def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: + """ + Return the package's loader if it's a ResourceReader. + """ + # We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + if reader is None: + return None + return reader(spec.name) # type: ignore + + +@functools.singledispatch +def resolve(cand: Optional[Anchor]) -> types.ModuleType: + return cast(types.ModuleType, cand) + + +@resolve.register +def _(cand: str) -> types.ModuleType: + return importlib.import_module(cand) + + +@resolve.register +def _(cand: None) -> types.ModuleType: + return resolve(_infer_caller().f_globals['__name__']) + + +def _infer_caller(): + """ + Walk the stack and find the frame of the first caller not in this module. + """ + + def is_this_file(frame_info): + return frame_info.filename == __file__ + + def is_wrapper(frame_info): + return frame_info.function == 'wrapper' + + not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + # also exclude 'wrapper' due to singledispatch in the call stack + callers = itertools.filterfalse(is_wrapper, not_this_file) + return next(callers).frame + + +def from_package(package: types.ModuleType): + """ + Return a Traversable object for the given package. + + """ + spec = wrap_spec(package) + reader = spec.loader.get_resource_reader(spec.name) + return reader.files() + + +@contextlib.contextmanager +def _tempfile( + reader, + suffix='', + # gh-93353: Keep a reference to call os.remove() in late Python + # finalization. + *, + _os_remove=os.remove, +): + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on Windows + # properly. + fd, raw_path = tempfile.mkstemp(suffix=suffix) + try: + try: + os.write(fd, reader()) + finally: + os.close(fd) + del reader + yield pathlib.Path(raw_path) + finally: + try: + _os_remove(raw_path) + except FileNotFoundError: + pass + + +def _temp_file(path): + return _tempfile(path.read_bytes, suffix=path.name) + + +def _is_present_dir(path: Traversable) -> bool: + """ + Some Traversables implement ``is_dir()`` to raise an + exception (i.e. ``FileNotFoundError``) when the + directory doesn't exist. This function wraps that call + to always return a boolean and only return True + if there's a dir and it exists. + """ + with contextlib.suppress(FileNotFoundError): + return path.is_dir() + return False + + +@functools.singledispatch +def as_file(path): + """ + Given a Traversable object, return that object as a + path on the local file system in a context manager. + """ + return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) + + +@as_file.register(pathlib.Path) +@contextlib.contextmanager +def _(path): + """ + Degenerate behavior for pathlib.Path objects. + """ + yield path + + +@contextlib.contextmanager +def _temp_path(dir: tempfile.TemporaryDirectory): + """ + Wrap tempfile.TemporyDirectory to return a pathlib object. + """ + with dir as result: + yield pathlib.Path(result) + + +@contextlib.contextmanager +def _temp_dir(path): + """ + Given a traversable dir, recursively replicate the whole tree + to the file system in a context manager. + """ + assert path.is_dir() + with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: + yield _write_contents(temp_dir, path) + + +def _write_contents(target, source): + child = target.joinpath(source.name) + if source.is_dir(): + child.mkdir() + for item in source.iterdir(): + _write_contents(child, item) + else: + child.write_bytes(source.read_bytes()) + return child diff --git a/contrib/tools/python3/Lib/importlib/resources/_itertools.py b/contrib/tools/python3/Lib/importlib/resources/_itertools.py new file mode 100644 index 0000000000..7b775ef5ae --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/_itertools.py @@ -0,0 +1,38 @@ +# from more_itertools 9.0 +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) + + try: + second_value = next(it) + except StopIteration: + pass + else: + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value diff --git a/contrib/tools/python3/Lib/importlib/resources/_legacy.py b/contrib/tools/python3/Lib/importlib/resources/_legacy.py new file mode 100644 index 0000000000..b1ea8105da --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/_legacy.py @@ -0,0 +1,120 @@ +import functools +import os +import pathlib +import types +import warnings + +from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any + +from . import _common + +Package = Union[types.ModuleType, str] +Resource = str + + +def deprecated(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"{func.__name__} is deprecated. Use files() instead. " + "Refer to https://importlib-resources.readthedocs.io" + "/en/latest/using.html#migrating-from-legacy for migration advice.", + DeprecationWarning, + stacklevel=2, + ) + return func(*args, **kwargs) + + return wrapper + + +def normalize_path(path: Any) -> str: + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError(f'{path!r} must be only a file name') + return file_name + + +@deprecated +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + return (_common.files(package) / normalize_path(resource)).open('rb') + + +@deprecated +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + return (_common.files(package) / normalize_path(resource)).read_bytes() + + +@deprecated +def open_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + return (_common.files(package) / normalize_path(resource)).open( + 'r', encoding=encoding, errors=errors + ) + + +@deprecated +def read_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +@deprecated +def contents(package: Package) -> Iterable[str]: + """Return an iterable of entries in `package`. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + return [path.name for path in _common.files(package).iterdir()] + + +@deprecated +def is_resource(package: Package, name: str) -> bool: + """True if `name` is a resource inside `package`. + + Directories are *not* resources. + """ + resource = normalize_path(name) + return any( + traversable.name == resource and traversable.is_file() + for traversable in _common.files(package).iterdir() + ) + + +@deprecated +def path( + package: Package, + resource: Resource, +) -> ContextManager[pathlib.Path]: + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/contrib/tools/python3/Lib/importlib/resources/abc.py b/contrib/tools/python3/Lib/importlib/resources/abc.py new file mode 100644 index 0000000000..6750a7aaf1 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/abc.py @@ -0,0 +1,173 @@ +import abc +import io +import itertools +import os +import pathlib +from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional +from typing import runtime_checkable, Protocol +from typing import Union + + +StrPath = Union[str, os.PathLike[str]] + +__all__ = ["ResourceReader", "Traversable", "TraversableResources"] + + +class ResourceReader(metaclass=abc.ABCMeta): + """Abstract base class for loaders to provide resource reading support.""" + + @abc.abstractmethod + def open_resource(self, resource: Text) -> BinaryIO: + """Return an opened, file-like object for binary reading. + + The 'resource' argument is expected to represent only a file name. + If the resource cannot be found, FileNotFoundError is raised. + """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. + raise FileNotFoundError + + @abc.abstractmethod + def resource_path(self, resource: Text) -> Text: + """Return the file system path to the specified resource. + + The 'resource' argument is expected to represent only a file name. + If the resource does not exist on the file system, raise + FileNotFoundError. + """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. + raise FileNotFoundError + + @abc.abstractmethod + def is_resource(self, path: Text) -> bool: + """Return True if the named 'path' is a resource. + + Files are resources, directories are not. + """ + raise FileNotFoundError + + @abc.abstractmethod + def contents(self) -> Iterable[str]: + """Return an iterable of entries in `package`.""" + raise FileNotFoundError + + +class TraversalError(Exception): + pass + + +@runtime_checkable +class Traversable(Protocol): + """ + An object with a subset of pathlib.Path methods suitable for + traversing directories and opening files. + + Any exceptions that occur when accessing the backing resource + may propagate unaltered. + """ + + @abc.abstractmethod + def iterdir(self) -> Iterator["Traversable"]: + """ + Yield Traversable objects in self + """ + + def read_bytes(self) -> bytes: + """ + Read contents of self as bytes + """ + with self.open('rb') as strm: + return strm.read() + + def read_text(self, encoding: Optional[str] = None) -> str: + """ + Read contents of self as text + """ + with self.open(encoding=encoding) as strm: + return strm.read() + + @abc.abstractmethod + def is_dir(self) -> bool: + """ + Return True if self is a directory + """ + + @abc.abstractmethod + def is_file(self) -> bool: + """ + Return True if self is a file + """ + + def joinpath(self, *descendants: StrPath) -> "Traversable": + """ + Return Traversable resolved with any descendants applied. + + Each descendant should be a path segment relative to self + and each may contain multiple levels separated by + ``posixpath.sep`` (``/``). + """ + if not descendants: + return self + names = itertools.chain.from_iterable( + path.parts for path in map(pathlib.PurePosixPath, descendants) + ) + target = next(names) + matches = ( + traversable for traversable in self.iterdir() if traversable.name == target + ) + try: + match = next(matches) + except StopIteration: + raise TraversalError( + "Target not found during traversal.", target, list(names) + ) + return match.joinpath(*names) + + def __truediv__(self, child: StrPath) -> "Traversable": + """ + Return Traversable child in self + """ + return self.joinpath(child) + + @abc.abstractmethod + def open(self, mode='r', *args, **kwargs): + """ + mode may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as pathlib.Path.open). + + When opening as text, accepts encoding parameters such as those + accepted by io.TextIOWrapper. + """ + + @property + @abc.abstractmethod + def name(self) -> str: + """ + The base name of this object without any parent references. + """ + + +class TraversableResources(ResourceReader): + """ + The required interface for providing traversable + resources. + """ + + @abc.abstractmethod + def files(self) -> "Traversable": + """Return a Traversable object for the loaded package.""" + + def open_resource(self, resource: StrPath) -> io.BufferedReader: + return self.files().joinpath(resource).open('rb') + + def resource_path(self, resource: Any) -> NoReturn: + raise FileNotFoundError(resource) + + def is_resource(self, path: StrPath) -> bool: + return self.files().joinpath(path).is_file() + + def contents(self) -> Iterator[str]: + return (item.name for item in self.files().iterdir()) diff --git a/contrib/tools/python3/Lib/importlib/resources/readers.py b/contrib/tools/python3/Lib/importlib/resources/readers.py new file mode 100644 index 0000000000..c3cdf769cb --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/readers.py @@ -0,0 +1,144 @@ +import collections +import itertools +import pathlib +import operator +import zipfile + +from . import abc + +from ._itertools import only + + +def remove_duplicates(items): + return iter(collections.OrderedDict.fromkeys(items)) + + +class FileReader(abc.TraversableResources): + def __init__(self, loader): + self.path = pathlib.Path(loader.path).parent + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path + + +class ZipReader(abc.TraversableResources): + def __init__(self, loader, module): + _, _, name = module.rpartition('.') + self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.archive = loader.archive + + def open_resource(self, resource): + try: + return super().open_resource(resource) + except KeyError as exc: + raise FileNotFoundError(exc.args[0]) + + def is_resource(self, path): + """ + Workaround for `zipfile.Path.is_file` returning true + for non-existent paths. + """ + target = self.files().joinpath(path) + return target.is_file() and target.exists() + + def files(self): + return zipfile.Path(self.archive, self.prefix) + + +class MultiplexedPath(abc.Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + + def __init__(self, *paths): + self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + if not self._paths: + message = 'MultiplexedPath must contain at least one path' + raise FileNotFoundError(message) + if not all(path.is_dir() for path in self._paths): + raise NotADirectoryError('MultiplexedPath only supports directories') + + def iterdir(self): + children = (child for path in self._paths for child in path.iterdir()) + by_name = operator.attrgetter('name') + groups = itertools.groupby(sorted(children, key=by_name), key=by_name) + return map(self._follow, (locs for name, locs in groups)) + + def read_bytes(self): + raise FileNotFoundError(f'{self} is not a file') + + def read_text(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + def is_dir(self): + return True + + def is_file(self): + return False + + def joinpath(self, *descendants): + try: + return super().joinpath(*descendants) + except abc.TraversalError: + # One of the paths did not resolve (a directory does not exist). + # Just return something that will not exist. + return self._paths[0].joinpath(*descendants) + + @classmethod + def _follow(cls, children): + """ + Construct a MultiplexedPath if needed. + + If children contains a sole element, return it. + Otherwise, return a MultiplexedPath of the items. + Unless one of the items is not a Directory, then return the first. + """ + subdirs, one_dir, one_file = itertools.tee(children, 3) + + try: + return only(one_dir) + except ValueError: + try: + return cls(*subdirs) + except NotADirectoryError: + return next(one_file) + + def open(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + @property + def name(self): + return self._paths[0].name + + def __repr__(self): + paths = ', '.join(f"'{path}'" for path in self._paths) + return f'MultiplexedPath({paths})' + + +class NamespaceReader(abc.TraversableResources): + def __init__(self, namespace_path): + if 'NamespacePath' not in str(namespace_path): + raise ValueError('Invalid path') + self.path = MultiplexedPath(*list(namespace_path)) + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path diff --git a/contrib/tools/python3/Lib/importlib/resources/simple.py b/contrib/tools/python3/Lib/importlib/resources/simple.py new file mode 100644 index 0000000000..7770c922c8 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/resources/simple.py @@ -0,0 +1,106 @@ +""" +Interface adapters for low-level readers. +""" + +import abc +import io +import itertools +from typing import BinaryIO, List + +from .abc import Traversable, TraversableResources + + +class SimpleReader(abc.ABC): + """ + The minimum, low-level interface required from a resource + provider. + """ + + @property + @abc.abstractmethod + def package(self) -> str: + """ + The name of the package for which this reader loads resources. + """ + + @abc.abstractmethod + def children(self) -> List['SimpleReader']: + """ + Obtain an iterable of SimpleReader for available + child containers (e.g. directories). + """ + + @abc.abstractmethod + def resources(self) -> List[str]: + """ + Obtain available named resources for this virtual package. + """ + + @abc.abstractmethod + def open_binary(self, resource: str) -> BinaryIO: + """ + Obtain a File-like for a named resource. + """ + + @property + def name(self): + return self.package.split('.')[-1] + + +class ResourceContainer(Traversable): + """ + Traversable container for a package's resources via its reader. + """ + + def __init__(self, reader: SimpleReader): + self.reader = reader + + def is_dir(self): + return True + + def is_file(self): + return False + + def iterdir(self): + files = (ResourceHandle(self, name) for name in self.reader.resources) + dirs = map(ResourceContainer, self.reader.children()) + return itertools.chain(files, dirs) + + def open(self, *args, **kwargs): + raise IsADirectoryError() + + +class ResourceHandle(Traversable): + """ + Handle to a named resource in a ResourceReader. + """ + + def __init__(self, parent: ResourceContainer, name: str): + self.parent = parent + self.name = name # type: ignore + + def is_file(self): + return True + + def is_dir(self): + return False + + def open(self, mode='r', *args, **kwargs): + stream = self.parent.reader.open_binary(self.name) + if 'b' not in mode: + stream = io.TextIOWrapper(*args, **kwargs) + return stream + + def joinpath(self, name): + raise RuntimeError("Cannot traverse into a resource") + + +class TraversableReader(TraversableResources, SimpleReader): + """ + A TraversableResources based on SimpleReader. Resource providers + may derive from this class to provide the TraversableResources + interface by supplying the SimpleReader interface. + """ + + def files(self): + return ResourceContainer(self) diff --git a/contrib/tools/python3/Lib/importlib/simple.py b/contrib/tools/python3/Lib/importlib/simple.py new file mode 100644 index 0000000000..845bb90364 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/simple.py @@ -0,0 +1,14 @@ +""" +Compatibility shim for .resources.simple as found on Python 3.10. + +Consumers that can rely on Python 3.11 should use the other +module directly. +""" + +from .resources.simple import ( + SimpleReader, ResourceHandle, ResourceContainer, TraversableReader, +) + +__all__ = [ + 'SimpleReader', 'ResourceHandle', 'ResourceContainer', 'TraversableReader', +] diff --git a/contrib/tools/python3/Lib/importlib/util.py b/contrib/tools/python3/Lib/importlib/util.py new file mode 100644 index 0000000000..f4d6e82331 --- /dev/null +++ b/contrib/tools/python3/Lib/importlib/util.py @@ -0,0 +1,248 @@ +"""Utility code for constructing importers, etc.""" +from ._abc import Loader +from ._bootstrap import module_from_spec +from ._bootstrap import _resolve_name +from ._bootstrap import spec_from_loader +from ._bootstrap import _find_spec +from ._bootstrap_external import MAGIC_NUMBER +from ._bootstrap_external import _RAW_MAGIC_NUMBER +from ._bootstrap_external import cache_from_source +from ._bootstrap_external import decode_source +from ._bootstrap_external import source_from_cache +from ._bootstrap_external import spec_from_file_location + +import _imp +import sys +import types + + +def source_hash(source_bytes): + "Return the hash of *source_bytes* as used in hash-based pyc files." + return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + + +def resolve_name(name, package): + """Resolve a relative module name to an absolute one.""" + if not name.startswith('.'): + return name + elif not package: + raise ImportError(f'no package specified for {repr(name)} ' + '(required for relative module names)') + level = 0 + for character in name: + if character != '.': + break + level += 1 + return _resolve_name(name[level:], package, level) + + +def _find_spec_from_path(name, path=None): + """Return the spec for the specified module. + + First, sys.modules is checked to see if the module was already imported. If + so, then sys.modules[name].__spec__ is returned. If that happens to be + set to None, then ValueError is raised. If the module is not in + sys.modules, then sys.meta_path is searched for a suitable spec with the + value of 'path' given to the finders. None is returned if no spec could + be found. + + Dotted names do not have their parent packages implicitly imported. You will + most likely need to explicitly import all parent packages in the proper + order for a submodule to get the correct spec. + + """ + if name not in sys.modules: + return _find_spec(name, path) + else: + module = sys.modules[name] + if module is None: + return None + try: + spec = module.__spec__ + except AttributeError: + raise ValueError(f'{name}.__spec__ is not set') from None + else: + if spec is None: + raise ValueError(f'{name}.__spec__ is None') + return spec + + +def find_spec(name, package=None): + """Return the spec for the specified module. + + First, sys.modules is checked to see if the module was already imported. If + so, then sys.modules[name].__spec__ is returned. If that happens to be + set to None, then ValueError is raised. If the module is not in + sys.modules, then sys.meta_path is searched for a suitable spec with the + value of 'path' given to the finders. None is returned if no spec could + be found. + + If the name is for submodule (contains a dot), the parent module is + automatically imported. + + The name and package arguments work the same as importlib.import_module(). + In other words, relative module names (with leading dots) work. + + """ + fullname = resolve_name(name, package) if name.startswith('.') else name + if fullname not in sys.modules: + parent_name = fullname.rpartition('.')[0] + if parent_name: + parent = __import__(parent_name, fromlist=['__path__']) + try: + parent_path = parent.__path__ + except AttributeError as e: + raise ModuleNotFoundError( + f"__path__ attribute not found on {parent_name!r} " + f"while trying to find {fullname!r}", name=fullname) from e + else: + parent_path = None + return _find_spec(fullname, parent_path) + else: + module = sys.modules[fullname] + if module is None: + return None + try: + spec = module.__spec__ + except AttributeError: + raise ValueError(f'{name}.__spec__ is not set') from None + else: + if spec is None: + raise ValueError(f'{name}.__spec__ is None') + return spec + + +# Normally we would use contextlib.contextmanager. However, this module +# is imported by runpy, which means we want to avoid any unnecessary +# dependencies. Thus we use a class. + +class _incompatible_extension_module_restrictions: + """A context manager that can temporarily skip the compatibility check. + + NOTE: This function is meant to accommodate an unusual case; one + which is likely to eventually go away. There's is a pretty good + chance this is not what you were looking for. + + WARNING: Using this function to disable the check can lead to + unexpected behavior and even crashes. It should only be used during + extension module development. + + If "disable_check" is True then the compatibility check will not + happen while the context manager is active. Otherwise the check + *will* happen. + + Normally, extensions that do not support multiple interpreters + may not be imported in a subinterpreter. That implies modules + that do not implement multi-phase init or that explicitly of out. + + Likewise for modules import in a subinterpeter with its own GIL + when the extension does not support a per-interpreter GIL. This + implies the module does not have a Py_mod_multiple_interpreters slot + set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. + + In both cases, this context manager may be used to temporarily + disable the check for compatible extension modules. + + You can get the same effect as this function by implementing the + basic interface of multi-phase init (PEP 489) and lying about + support for mulitple interpreters (or per-interpreter GIL). + """ + + def __init__(self, *, disable_check): + self.disable_check = bool(disable_check) + + def __enter__(self): + self.old = _imp._override_multi_interp_extensions_check(self.override) + return self + + def __exit__(self, *args): + old = self.old + del self.old + _imp._override_multi_interp_extensions_check(old) + + @property + def override(self): + return -1 if self.disable_check else 1 + + +class _LazyModule(types.ModuleType): + + """A subclass of the module type which triggers loading upon attribute access.""" + + def __getattribute__(self, attr): + """Trigger the load of the module and return the attribute.""" + # All module metadata must be garnered from __spec__ in order to avoid + # using mutated values. + # Stop triggering this method. + self.__class__ = types.ModuleType + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = self.__spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = self.__spec__.loader_state['__dict__'] + attrs_now = self.__dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set the attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + self.__spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + self.__dict__.update(attrs_updated) + return getattr(self, attr) + + def __delattr__(self, attr): + """Trigger the load and then perform the deletion.""" + # To trigger the load and raise an exception if the attribute + # doesn't exist. + self.__getattribute__(attr) + delattr(self, attr) + + +class LazyLoader(Loader): + + """A loader that creates a module which defers loading until attribute access.""" + + @staticmethod + def __check_eager_loader(loader): + if not hasattr(loader, 'exec_module'): + raise TypeError('loader must define exec_module()') + + @classmethod + def factory(cls, loader): + """Construct a callable which returns the eager loader made lazy.""" + cls.__check_eager_loader(loader) + return lambda *args, **kwargs: cls(loader(*args, **kwargs)) + + def __init__(self, loader): + self.__check_eager_loader(loader) + self.loader = loader + + def create_module(self, spec): + return self.loader.create_module(spec) + + def exec_module(self, module): + """Make the module load lazily.""" + module.__spec__.loader = self.loader + module.__loader__ = self.loader + # Don't need to worry about deep-copying as trying to set an attribute + # on an object would have triggered the load, + # e.g. ``module.__spec__.loader = None`` would trigger a load from + # trying to access module.__spec__. + loader_state = {} + loader_state['__dict__'] = module.__dict__.copy() + loader_state['__class__'] = module.__class__ + module.__spec__.loader_state = loader_state + module.__class__ = _LazyModule |