diff options
| author | robot-contrib <robot-contrib@yandex-team.ru> | 2022-05-18 00:43:36 +0300 |
|---|---|---|
| committer | robot-contrib <robot-contrib@yandex-team.ru> | 2022-05-18 00:43:36 +0300 |
| commit | 9e5f436a8b2a27bcc7802e443ea3ef3e41a82a75 (patch) | |
| tree | 78b522cab9f76336e62064d4d8ff7c897659b20e /contrib/python/ipython/py3/IPython/core | |
| parent | 8113a823ffca6451bb5ff8f0334560885a939a24 (diff) | |
| download | ydb-9e5f436a8b2a27bcc7802e443ea3ef3e41a82a75.tar.gz | |
Update contrib/python/ipython/py3 to 8.3.0
ref:e84342d4d30476f9148137f37fd0c6405fd36f55
Diffstat (limited to 'contrib/python/ipython/py3/IPython/core')
48 files changed, 2841 insertions, 2874 deletions
diff --git a/contrib/python/ipython/py3/IPython/core/application.py b/contrib/python/ipython/py3/IPython/core/application.py index b319888b59..0cdea5c69b 100644 --- a/contrib/python/ipython/py3/IPython/core/application.py +++ b/contrib/python/ipython/py3/IPython/core/application.py @@ -20,6 +20,8 @@ import os import shutil import sys +from pathlib import Path + from traitlets.config.application import Application, catch_config_error from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader from IPython.core import release, crashhandler @@ -31,10 +33,10 @@ from traitlets import ( default, observe, ) -if os.name == 'nt': - programdata = os.environ.get('PROGRAMDATA', None) - if programdata: - SYSTEM_CONFIG_DIRS = [os.path.join(programdata, 'ipython')] +if os.name == "nt": + programdata = os.environ.get("PROGRAMDATA", None) + if programdata is not None: + SYSTEM_CONFIG_DIRS = [str(Path(programdata) / "ipython")] else: # PROGRAMDATA is not defined by default on XP. SYSTEM_CONFIG_DIRS = [] else: @@ -64,27 +66,49 @@ else: # aliases and flags -base_aliases = { - 'profile-dir' : 'ProfileDir.location', - 'profile' : 'BaseIPythonApplication.profile', - 'ipython-dir' : 'BaseIPythonApplication.ipython_dir', - 'log-level' : 'Application.log_level', - 'config' : 'BaseIPythonApplication.extra_config_file', -} - -base_flags = dict( - debug = ({'Application' : {'log_level' : logging.DEBUG}}, - "set log level to logging.DEBUG (maximize logging output)"), - quiet = ({'Application' : {'log_level' : logging.CRITICAL}}, - "set log level to logging.CRITICAL (minimize logging output)"), - init = ({'BaseIPythonApplication' : { - 'copy_config_files' : True, - 'auto_create' : True} - }, """Initialize profile with default config files. This is equivalent +base_aliases = {} +if isinstance(Application.aliases, dict): + # traitlets 5 + base_aliases.update(Application.aliases) +base_aliases.update( + { + "profile-dir": "ProfileDir.location", + "profile": "BaseIPythonApplication.profile", + "ipython-dir": "BaseIPythonApplication.ipython_dir", + "log-level": "Application.log_level", + "config": "BaseIPythonApplication.extra_config_file", + } +) + +base_flags = dict() +if isinstance(Application.flags, dict): + # traitlets 5 + base_flags.update(Application.flags) +base_flags.update( + dict( + debug=( + {"Application": {"log_level": logging.DEBUG}}, + "set log level to logging.DEBUG (maximize logging output)", + ), + quiet=( + {"Application": {"log_level": logging.CRITICAL}}, + "set log level to logging.CRITICAL (minimize logging output)", + ), + init=( + { + "BaseIPythonApplication": { + "copy_config_files": True, + "auto_create": True, + } + }, + """Initialize profile with default config files. This is equivalent to running `ipython profile create <profile>` prior to startup. - """) + """, + ), + ) ) + class ProfileAwareConfigLoader(PyFileConfigLoader): """A Python file config loader that is aware of IPython profiles.""" def load_subconfig(self, fname, path=None, profile=None): @@ -161,6 +185,17 @@ class BaseIPythonApplication(Application): get_ipython_package_dir(), u'config', u'profile', change['new'] ) + add_ipython_dir_to_sys_path = Bool( + False, + """Should the IPython profile directory be added to sys path ? + + This option was non-existing before IPython 8.0, and ipython_dir was added to + sys path to allow import of extensions present there. This was historical + baggage from when pip did not exist. This now default to false, + but can be set to true for legacy reasons. + """, + ).tag(config=True) + ipython_dir = Unicode( help=""" The name of the IPython directory. This directory is used for logging @@ -232,16 +267,6 @@ class BaseIPythonApplication(Application): # Various stages of Application creation #------------------------------------------------------------------------- - deprecated_subcommands = {} - - def initialize_subcommand(self, subc, argv=None): - if subc in self.deprecated_subcommands: - self.log.warning("Subcommand `ipython {sub}` is deprecated and will be removed " - "in future versions.".format(sub=subc)) - self.log.warning("You likely want to use `jupyter {sub}` in the " - "future".format(sub=subc)) - return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv) - def init_crash_handler(self): """Create a crash handler, typically setting sys.excepthook to it.""" self.crash_handler = self.crash_handler_class(self) @@ -252,7 +277,7 @@ class BaseIPythonApplication(Application): def excepthook(self, etype, evalue, tb): """this is sys.excepthook after init_crashhandler - + set self.verbose_crash=True to use our full crashhandler, instead of a regular traceback with a short message (crash_handler_lite) """ @@ -270,21 +295,24 @@ class BaseIPythonApplication(Application): str_old = os.path.abspath(old) if str_old in sys.path: sys.path.remove(str_old) - str_path = os.path.abspath(new) - sys.path.append(str_path) - ensure_dir_exists(new) - readme = os.path.join(new, 'README') - readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') - if not os.path.exists(readme) and os.path.exists(readme_src): - shutil.copy(readme_src, readme) - for d in ('extensions', 'nbextensions'): - path = os.path.join(new, d) - try: - ensure_dir_exists(path) - except OSError as e: - # this will not be EEXIST - self.log.error("couldn't create path %s: %s", path, e) - self.log.debug("IPYTHONDIR set to: %s" % new) + if self.add_ipython_dir_to_sys_path: + str_path = os.path.abspath(new) + sys.path.append(str_path) + ensure_dir_exists(new) + readme = os.path.join(new, "README") + readme_src = os.path.join( + get_ipython_package_dir(), "config", "profile", "README" + ) + if not os.path.exists(readme) and os.path.exists(readme_src): + shutil.copy(readme_src, readme) + for d in ("extensions", "nbextensions"): + path = os.path.join(new, d) + try: + ensure_dir_exists(path) + except OSError as e: + # this will not be EEXIST + self.log.error("couldn't create path %s: %s", path, e) + self.log.debug("IPYTHONDIR set to: %s" % new) def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): """Load the config file. @@ -409,14 +437,15 @@ class BaseIPythonApplication(Application): self.config_file_paths.extend(ENV_CONFIG_DIRS) self.config_file_paths.extend(SYSTEM_CONFIG_DIRS) # copy config files - path = self.builtin_profile_dir + path = Path(self.builtin_profile_dir) if self.copy_config_files: src = self.profile cfg = self.config_file_name - if path and os.path.exists(os.path.join(path, cfg)): - self.log.warning("Staging %r from %s into %r [overwrite=%s]"%( - cfg, src, self.profile_dir.location, self.overwrite) + if path and (path / cfg).exists(): + self.log.warning( + "Staging %r from %s into %r [overwrite=%s]" + % (cfg, src, self.profile_dir.location, self.overwrite) ) self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) else: @@ -425,9 +454,9 @@ class BaseIPythonApplication(Application): # Still stage *bundled* config files, but not generated ones # This is necessary for `ipython profile=sympy` to load the profile # on the first go - files = glob.glob(os.path.join(path, '*.py')) + files = path.glob("*.py") for fullpath in files: - cfg = os.path.basename(fullpath) + cfg = fullpath.name if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False): # file was copied self.log.warning("Staging bundled %s from %s into %r"%( @@ -438,11 +467,10 @@ class BaseIPythonApplication(Application): def stage_default_config_file(self): """auto generate default config file, and stage it into the profile.""" s = self.generate_config_file() - fname = os.path.join(self.profile_dir.location, self.config_file_name) - if self.overwrite or not os.path.exists(fname): - self.log.warning("Generating default config file: %r"%(fname)) - with open(fname, 'w') as f: - f.write(s) + config_file = Path(self.profile_dir.location) / self.config_file_name + if self.overwrite or not config_file.exists(): + self.log.warning("Generating default config file: %r" % (config_file)) + config_file.write_text(s, encoding="utf-8") @catch_config_error def initialize(self, argv=None): diff --git a/contrib/python/ipython/py3/IPython/core/async_helpers.py b/contrib/python/ipython/py3/IPython/core/async_helpers.py index fca78def85..0e7db0bb54 100644 --- a/contrib/python/ipython/py3/IPython/core/async_helpers.py +++ b/contrib/python/ipython/py3/IPython/core/async_helpers.py @@ -12,37 +12,89 @@ Python semantics. import ast -import sys import asyncio import inspect -from textwrap import dedent, indent +from functools import wraps +_asyncio_event_loop = None -class _AsyncIORunner: - def __init__(self): - self._loop = None - - @property - def loop(self): - """Always returns a non-closed event loop""" - if self._loop is None or self._loop.is_closed(): - policy = asyncio.get_event_loop_policy() - self._loop = policy.new_event_loop() - policy.set_event_loop(self._loop) - return self._loop +def get_asyncio_loop(): + """asyncio has deprecated get_event_loop + + Replicate it here, with our desired semantics: + + - always returns a valid, not-closed loop + - not thread-local like asyncio's, + because we only want one loop for IPython + - if called from inside a coroutine (e.g. in ipykernel), + return the running loop + + .. versionadded:: 8.0 + """ + try: + return asyncio.get_running_loop() + except RuntimeError: + # not inside a coroutine, + # track our own global + pass + + # not thread-local like asyncio's, + # because we only track one event loop to run for IPython itself, + # always in the main thread. + global _asyncio_event_loop + if _asyncio_event_loop is None or _asyncio_event_loop.is_closed(): + _asyncio_event_loop = asyncio.new_event_loop() + return _asyncio_event_loop + + +class _AsyncIORunner: def __call__(self, coro): """ Handler for asyncio autoawait """ - return self.loop.run_until_complete(coro) + return get_asyncio_loop().run_until_complete(coro) def __str__(self): - return 'asyncio' + return "asyncio" + _asyncio_runner = _AsyncIORunner() +class _AsyncIOProxy: + """Proxy-object for an asyncio + + Any coroutine methods will be wrapped in event_loop.run_ + """ + + def __init__(self, obj, event_loop): + self._obj = obj + self._event_loop = event_loop + + def __repr__(self): + return f"<_AsyncIOProxy({self._obj!r})>" + + def __getattr__(self, key): + attr = getattr(self._obj, key) + if inspect.iscoroutinefunction(attr): + # if it's a coroutine method, + # return a threadsafe wrapper onto the _current_ asyncio loop + @wraps(attr) + def _wrapped(*args, **kwargs): + concurrent_future = asyncio.run_coroutine_threadsafe( + attr(*args, **kwargs), self._event_loop + ) + return asyncio.wrap_future(concurrent_future) + + return _wrapped + else: + return attr + + def __dir__(self): + return dir(self._obj) + + def _curio_runner(coroutine): """ handler for curio autoawait @@ -72,7 +124,6 @@ def _pseudo_sync_runner(coro): See discussion in https://github.com/python-trio/trio/issues/608, Credit to Nathaniel Smith - """ try: coro.send(None) @@ -85,69 +136,6 @@ def _pseudo_sync_runner(coro): ) -def _asyncify(code: str) -> str: - """wrap code in async def definition. - - And setup a bit of context to run it later. - """ - res = dedent( - """ - async def __wrapper__(): - try: - {usercode} - finally: - locals() - """ - ).format(usercode=indent(code, " " * 8)) - return res - - -class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): - """ - Find syntax errors that would be an error in an async repl, but because - the implementation involves wrapping the repl in an async function, it - is erroneously allowed (e.g. yield or return at the top level) - """ - def __init__(self): - if sys.version_info >= (3,8): - raise ValueError('DEPRECATED in Python 3.8+') - self.depth = 0 - super().__init__() - - def generic_visit(self, node): - func_types = (ast.FunctionDef, ast.AsyncFunctionDef) - invalid_types_by_depth = { - 0: (ast.Return, ast.Yield, ast.YieldFrom), - 1: (ast.Nonlocal,) - } - - should_traverse = self.depth < max(invalid_types_by_depth.keys()) - if isinstance(node, func_types) and should_traverse: - self.depth += 1 - super().generic_visit(node) - self.depth -= 1 - elif isinstance(node, invalid_types_by_depth[self.depth]): - raise SyntaxError() - else: - super().generic_visit(node) - - -def _async_parse_cell(cell: str) -> ast.AST: - """ - This is a compatibility shim for pre-3.7 when async outside of a function - is a syntax error at the parse stage. - - It will return an abstract syntax tree parsed as if async and await outside - of a function were not a syntax error. - """ - if sys.version_info < (3, 7): - # Prior to 3.7 you need to asyncify before parse - wrapped_parse_tree = ast.parse(_asyncify(cell)) - return wrapped_parse_tree.body[0].body[0] - else: - return ast.parse(cell) - - def _should_be_async(cell: str) -> bool: """Detect if a block of code need to be wrapped in an `async def` @@ -159,25 +147,10 @@ def _should_be_async(cell: str) -> bool: Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ - if sys.version_info > (3, 8): - try: - code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) - return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except (SyntaxError, MemoryError): - return False try: - # we can't limit ourself to ast.parse, as it __accepts__ to parse on - # 3.7+, but just does not _compile_ - code = compile(cell, "<>", "exec") + code = compile( + cell, "<>", "exec", flags=getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0) + ) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE except (SyntaxError, MemoryError): - try: - parse_tree = _async_parse_cell(cell) - - # Raise a SyntaxError if there are top-level return or yields - v = _AsyncSyntaxErrorVisitor() - v.visit(parse_tree) - - except (SyntaxError, MemoryError): - return False - return True - return False + return False diff --git a/contrib/python/ipython/py3/IPython/core/autocall.py b/contrib/python/ipython/py3/IPython/core/autocall.py index bab7f859c9..54beec3f58 100644 --- a/contrib/python/ipython/py3/IPython/core/autocall.py +++ b/contrib/python/ipython/py3/IPython/core/autocall.py @@ -40,10 +40,10 @@ class IPyAutocall(object): self._ip = ip def set_ip(self, ip): - """ Will be used to set _ip point to current ipython instance b/f call - + """Will be used to set _ip point to current ipython instance b/f call + Override this method if you don't want this to happen. - + """ self._ip = ip diff --git a/contrib/python/ipython/py3/IPython/core/compilerop.py b/contrib/python/ipython/py3/IPython/core/compilerop.py index 50672a1954..b43e570b3a 100644 --- a/contrib/python/ipython/py3/IPython/core/compilerop.py +++ b/contrib/python/ipython/py3/IPython/core/compilerop.py @@ -92,6 +92,10 @@ class CachingCompiler(codeop.Compile): # (otherwise we'd lose our tracebacks). linecache.checkcache = check_linecache_ipython + # Caching a dictionary { filename: execution_count } for nicely + # rendered tracebacks. The filename corresponds to the filename + # argument used for the builtins.compile function. + self._filename_map = {} def ast_parse(self, source, filename='<unknown>', symbol='exec'): """Parse code to an AST with the current compiler flags active. @@ -118,12 +122,12 @@ class CachingCompiler(codeop.Compile): Parameters ---------- raw_code : str - The raw cell code. + The raw cell code. transformed_code : str - The executable Python source code to cache and compile. + The executable Python source code to cache and compile. number : int - A number which forms part of the code's name. Used for the execution - counter. + A number which forms part of the code's name. Used for the execution + counter. Returns ------- @@ -137,12 +141,12 @@ class CachingCompiler(codeop.Compile): Parameters ---------- transformed_code : str - The executable Python source code to cache and compile. + The executable Python source code to cache and compile. number : int - A number which forms part of the code's name. Used for the execution - counter. + A number which forms part of the code's name. Used for the execution + counter. raw_code : str - The raw code before transformation, if None, set to `transformed_code`. + The raw code before transformation, if None, set to `transformed_code`. Returns ------- @@ -153,6 +157,10 @@ class CachingCompiler(codeop.Compile): raw_code = transformed_code name = self.get_code_name(raw_code, transformed_code, number) + + # Save the execution count + self._filename_map[name] = number + entry = ( len(transformed_code), time.time(), diff --git a/contrib/python/ipython/py3/IPython/core/completer.py b/contrib/python/ipython/py3/IPython/core/completer.py index 6c6fa7e7e5..fcc9d20d59 100644 --- a/contrib/python/ipython/py3/IPython/core/completer.py +++ b/contrib/python/ipython/py3/IPython/core/completer.py @@ -35,7 +35,7 @@ or using unicode completion: .. code:: - \\greek small letter alpha<tab> + \\GREEK SMALL LETTER ALPHA<tab> α @@ -50,7 +50,7 @@ Backward latex completion It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press `<tab>` to expand it to its latex form. +and press ``<tab>`` to expand it to its latex form. .. code:: @@ -121,26 +121,29 @@ import string import sys import time import unicodedata +import uuid import warnings from contextlib import contextmanager from importlib import import_module from types import SimpleNamespace -from typing import Iterable, Iterator, List, Tuple +from typing import Iterable, Iterator, List, Tuple, Union, Any, Sequence, Dict, NamedTuple, Pattern, Optional from IPython.core.error import TryNext from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol from IPython.core.oinspect import InspectColors +from IPython.testing.skipdoctest import skip_doctest from IPython.utils import generics from IPython.utils.dir2 import dir2, get_real_method +from IPython.utils.path import ensure_dir_exists from IPython.utils.process import arg_split -from traitlets import Bool, Enum, Int, observe +from traitlets import Bool, Enum, Int, List as ListTrait, Unicode, default, observe from traitlets.config.configurable import Configurable import __main__ # skip module docstests -skip_doctest = True +__skip_doctest__ = True try: import jedi @@ -154,6 +157,14 @@ except ImportError: # Globals #----------------------------------------------------------------------------- +# ranges where we have most of the valid unicode names. We could be more finer +# grained but is it worth it for performance While unicode have character in the +# range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I +# write this). With below range we cover them all, with a density of ~67% +# biggest next gap we consider only adds up about 1% density and there are 600 +# gaps that would need hard coding. +_UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)] + # Public API __all__ = ['Completer','IPCompleter'] @@ -167,14 +178,6 @@ else: MATCHES_LIMIT = 500 -class Sentinel: - def __repr__(self): - return "<deprecated sentinel>" - - -_deprecation_readline_sentinel = Sentinel() - - class ProvisionalCompleterWarning(FutureWarning): """ Exception raise by an experimental feature in this module. @@ -186,11 +189,11 @@ class ProvisionalCompleterWarning(FutureWarning): warnings.filterwarnings('error', category=ProvisionalCompleterWarning) + +@skip_doctest @contextmanager def provisionalcompleter(action='ignore'): """ - - This context manager has to be used in any place where unstable completer behavior and API may be called. @@ -260,17 +263,17 @@ def expand_user(path:str) -> Tuple[str, bool, str]: Parameters ---------- path : str - String to be expanded. If no ~ is present, the output is the same as the - input. + String to be expanded. If no ~ is present, the output is the same as the + input. Returns ------- newpath : str - Result of ~ expansion in the input path. + Result of ~ expansion in the input path. tilde_expand : bool - Whether any expansion was performed or not. + Whether any expansion was performed or not. tilde_val : str - The value that ~ was replaced with. + The value that ~ was replaced with. """ # Default values tilde_expand = False @@ -429,20 +432,17 @@ def _deduplicate_completions(text: str, completions: _IC)-> _IC: Parameters ---------- - text: str + text : str text that should be completed. - completions: Iterator[Completion] + completions : Iterator[Completion] iterator over the completions to deduplicate Yields ------ `Completions` objects - - Completions coming from multiple sources, may be different but end up having the same effect when applied to ``text``. If this is the case, this will consider completions as equal and only emit the first encountered. - Not folded in `completions()` yet for debugging purpose, and to detect when the IPython completer does return things that Jedi does not, but should be at some point. @@ -462,7 +462,7 @@ def _deduplicate_completions(text: str, completions: _IC)-> _IC: seen.add(new_text) -def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: +def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) -> _IC: """ Rectify a set of completions to all have the same ``start`` and ``end`` @@ -475,12 +475,15 @@ def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: Parameters ---------- - text: str + text : str text that should be completed. - completions: Iterator[Completion] + completions : Iterator[Completion] iterator over the completions to rectify + _debug : bool + Log failed completion - + Notes + ----- :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though the Jupyter Protocol requires them to behave like so. This will readjust the completion to have the same ``start`` and ``end`` by padding both @@ -582,11 +585,11 @@ class Completer(Configurable): greedy = Bool(False, help="""Activate greedy completion - PENDING DEPRECTION. this is now mostly taken care of with Jedi. + PENDING DEPRECATION. this is now mostly taken care of with Jedi. This will enable completion on elements of lists, results of function calls, etc., but can be unsafe because the code is actually evaluated on TAB. - """ + """, ).tag(config=True) use_jedi = Bool(default_value=JEDI_INSTALLED, @@ -609,8 +612,6 @@ class Completer(Configurable): "Includes completion of latex commands, unicode names, and expanding " "unicode characters back to latex commands.").tag(config=True) - - def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -757,44 +758,77 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[str], prefix: str, delims: str): +def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str, + extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters - ========== - keys: + ---------- + keys list of keys in dictionary currently being completed. - prefix: - Part of the text already typed by the user. e.g. `mydict[b'fo` - delims: + prefix + Part of the text already typed by the user. E.g. `mydict[b'fo` + delims String of delimiters to consider when finding the current key. + extra_prefix : optional + Part of the text already typed in multi-key index cases. E.g. for + `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`. Returns - ======= - + ------- A tuple of three elements: ``quote``, ``token_start``, ``matched``, with ``quote`` being the quote that need to be used to close current string. ``token_start`` the position where the replacement should start occurring, ``matches`` a list of replacement/completion """ + prefix_tuple = extra_prefix if extra_prefix else () + Nprefix = len(prefix_tuple) + def filter_prefix_tuple(key): + # Reject too short keys + if len(key) <= Nprefix: + return False + # Reject keys with non str/bytes in it + for k in key: + if not isinstance(k, (str, bytes)): + return False + # Reject keys that do not match the prefix + for k, pt in zip(key, prefix_tuple): + if k != pt: + return False + # All checks passed! + return True + + filtered_keys:List[Union[str,bytes]] = [] + def _add_to_filtered_keys(key): + if isinstance(key, (str, bytes)): + filtered_keys.append(key) + + for k in keys: + if isinstance(k, tuple): + if filter_prefix_tuple(k): + _add_to_filtered_keys(k[Nprefix]) + else: + _add_to_filtered_keys(k) + if not prefix: - return None, 0, [repr(k) for k in keys - if isinstance(k, (str, bytes))] + return '', 0, [repr(k) for k in filtered_keys] quote_match = re.search('["\']', prefix) + assert quote_match is not None # silence mypy quote = quote_match.group() try: prefix_str = eval(prefix + quote, {}) except Exception: - return None, 0, [] + return '', 0, [] pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) + assert token_match is not None # silence mypy token_start = token_match.start() token_prefix = token_match.group() - matched = [] - for key in keys: + matched:List[str] = [] + for key in filtered_keys: try: if not key.startswith(prefix_str): continue @@ -806,14 +840,6 @@ def match_dict_keys(keys: List[str], prefix: str, delims: str): rem = key[len(prefix_str):] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') - if rem_repr.startswith('u') and prefix[0] not in 'uU': - # Found key is unicode, but prefix is Py2 string. - # Therefore attempt to interpret key as string. - try: - rem_repr = repr(rem.encode('ascii') + '"') - except UnicodeEncodeError: - continue - rem_repr = rem_repr[1 + rem_repr.index("'"):-2] if quote == '"': # The entered prefix is quoted with ", @@ -828,13 +854,11 @@ def match_dict_keys(keys: List[str], prefix: str, delims: str): def cursor_to_position(text:str, line:int, column:int)->int: """ - Convert the (line,column) position of the cursor in text to an offset in a string. Parameters ---------- - text : str The text in which to calculate the cursor offset line : int @@ -842,13 +866,13 @@ def cursor_to_position(text:str, line:int, column:int)->int: column : int Column of the cursor 0-indexed - Return - ------ - Position of the cursor in ``text``, 0-indexed. + Returns + ------- + Position of the cursor in ``text``, 0-indexed. See Also -------- - position_to_cursor: reciprocal of this function + position_to_cursor : reciprocal of this function """ lines = text.split('\n') @@ -865,23 +889,20 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: Parameters ---------- - text : str The text in which to calculate the cursor offset offset : int Position of the cursor in ``text``, 0-indexed. - Return - ------ + Returns + ------- (line, column) : (int, int) Line of the cursor; 0-indexed, column of the cursor 0-indexed - See Also -------- cursor_to_position : reciprocal of this function - """ assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text)) @@ -899,9 +920,8 @@ def _safe_isinstance(obj, module, class_name): return (module in sys.modules and isinstance(obj, getattr(import_module(module), class_name))) - -def back_unicode_name_matches(text): - u"""Match unicode characters back to unicode name +def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]: + """Match Unicode characters back to Unicode name This does ``☃`` -> ``\\snowman`` @@ -910,52 +930,60 @@ def back_unicode_name_matches(text): This will not either back-complete standard sequences like \\n, \\b ... - Used on Python 3 only. + Returns + ======= + + Return a tuple with two elements: + + - The Unicode character that was matched (preceded with a backslash), or + empty string, + - a sequence (of 1), name for the match Unicode character, preceded by + backslash, or empty if no match. + """ if len(text)<2: - return u'', () + return '', () maybe_slash = text[-2] if maybe_slash != '\\': - return u'', () + return '', () char = text[-1] # no expand on quote for completion in strings. # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () + if char in string.ascii_letters or char in ('"',"'"): + return '', () try : unic = unicodedata.name(char) - return '\\'+char,['\\'+unic] + return '\\'+char,('\\'+unic,) except KeyError: pass - return u'', () + return '', () -def back_latex_name_matches(text:str): +def back_latex_name_matches(text:str) -> Tuple[str, Sequence[str]] : """Match latex characters back to unicode name This does ``\\ℵ`` -> ``\\aleph`` - Used on Python 3 only. """ if len(text)<2: - return u'', () + return '', () maybe_slash = text[-2] if maybe_slash != '\\': - return u'', () + return '', () char = text[-1] # no expand on quote for completion in strings. # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () + if char in string.ascii_letters or char in ('"',"'"): + return '', () try : latex = reverse_latex_symbol[char] # '\\' replace the \ as well return '\\'+char,[latex] except KeyError: pass - return u'', () + return '', () def _formatparamchildren(parameter) -> str: @@ -964,18 +992,15 @@ def _formatparamchildren(parameter) -> str: Jedi does not expose a simple way to get `param=value` from its API. - Parameter - ========= - - parameter: + Parameters + ---------- + parameter Jedi's function `Param` Returns - ======= - + ------- A string like 'a', 'b=1', '*args', '**kwargs' - """ description = parameter.description if not description.startswith('param '): @@ -987,15 +1012,13 @@ def _make_signature(completion)-> str: """ Make the signature from a jedi completion - Parameter - ========= - - completion: jedi.Completion + Parameters + ---------- + completion : jedi.Completion object does not complete a function type Returns - ======= - + ------- a string consisting of the function signature, with the parenthesis but without the function name. example: `(a, *args, b=1, **kwargs)` @@ -1015,10 +1038,18 @@ def _make_signature(completion)-> str: return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures() for p in signature.defined_names()) if f]) + +class _CompleteResult(NamedTuple): + matched_text : str + matches: Sequence[str] + matches_origin: Sequence[str] + jedi_matches: Any + + class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - _names = None + __dict_key_regexps: Optional[Dict[bool,Pattern]] = None @observe('greedy') def _greedy_changed(self, change): @@ -1064,6 +1095,16 @@ class IPCompleter(Completer): """, ).tag(config=True) + profile_completions = Bool( + default_value=False, + help="If True, emit profiling data for completion subsystem using cProfile." + ).tag(config=True) + + profiler_output_dir = Unicode( + default_value=".completion_profiles", + help="Template for path at which to output profile data for completions." + ).tag(config=True) + @observe('limit_to__all__') def _limit_to_all_changed(self, change): warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration ' @@ -1071,42 +1112,41 @@ class IPCompleter(Completer): 'no effects and then removed in future version of IPython.', UserWarning) - def __init__(self, shell=None, namespace=None, global_namespace=None, - use_readline=_deprecation_readline_sentinel, config=None, **kwargs): + def __init__( + self, shell=None, namespace=None, global_namespace=None, config=None, **kwargs + ): """IPCompleter() -> completer Return a completer object. Parameters ---------- - shell a pointer to the ipython shell itself. This is needed because this completer knows about magic functions, and those can only be accessed via the ipython instance. - namespace : dict, optional an optional dict where completions are performed. - global_namespace : dict, optional secondary optional dict for completions, to handle cases (such as IPython embedded inside functions) where both Python scopes are visible. - - use_readline : bool, optional - DEPRECATED, ignored since IPython 6.0, will have no effects + config : Config + traitlet's config object + **kwargs + passed to super class unmodified. """ self.magic_escape = ESC_MAGIC self.splitter = CompletionSplitter() - if use_readline is not _deprecation_readline_sentinel: - warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.', - DeprecationWarning, stacklevel=2) - # _greedy_changed() depends on splitter and readline being defined: - Completer.__init__(self, namespace=namespace, global_namespace=global_namespace, - config=config, **kwargs) + super().__init__( + namespace=namespace, + global_namespace=global_namespace, + config=config, + **kwargs + ) # List where completion matches will be stored self.matches = [] @@ -1141,8 +1181,14 @@ class IPCompleter(Completer): # This is set externally by InteractiveShell self.custom_completers = None + # This is a list of names of unicode characters that can be completed + # into their corresponding unicode value. The list is large, so we + # lazily initialize it on first use. Consuming code should access this + # attribute through the `@unicode_names` property. + self._unicode_names = None + @property - def matchers(self): + def matchers(self) -> List[Any]: """All active matcher routines for completion""" if self.dict_keys_only: return [self.dict_key_matches] @@ -1164,7 +1210,7 @@ class IPCompleter(Completer): self.python_func_kw_matches, ] - def all_completions(self, text) -> List[str]: + def all_completions(self, text:str) -> List[str]: """ Wrapper around the completion methods for the benefit of emacs. """ @@ -1175,14 +1221,14 @@ class IPCompleter(Completer): return self.complete(text)[1] - def _clean_glob(self, text): + def _clean_glob(self, text:str): return self.glob("%s*" % text) - def _clean_glob_win32(self,text): + def _clean_glob_win32(self, text:str): return [f.replace("\\","/") for f in self.glob("%s*" % text)] - def file_matches(self, text): + def file_matches(self, text:str)->List[str]: """Match filenames, expanding ~USER type strings. Most of the seemingly convoluted logic in this completer is an @@ -1264,7 +1310,7 @@ class IPCompleter(Completer): # Mark directories in input list by appending '/' to their names. return [x+'/' if os.path.isdir(x) else x for x in matches] - def magic_matches(self, text): + def magic_matches(self, text:str): """Match magics""" # Get all shell magics now rather than statically, so magics loaded at # runtime show up too. @@ -1355,9 +1401,8 @@ class IPCompleter(Completer): if color.startswith(prefix) ] return [] - def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): + def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str) -> Iterable[Any]: """ - Return a list of :any:`jedi.api.Completions` object from a ``text`` and cursor position. @@ -1370,9 +1415,8 @@ class IPCompleter(Completer): text : str text to complete - Debugging - --------- - + Notes + ----- If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion` object containing a string with the Jedi debug information attached. """ @@ -1429,7 +1473,7 @@ class IPCompleter(Completer): else: return [] - def python_matches(self, text): + def python_matches(self, text:str)->List[str]: """Match attributes or global python names""" if "." in text: try: @@ -1511,7 +1555,7 @@ class IPCompleter(Completer): return list(set(ret)) - def python_func_kw_matches(self,text): + def python_func_kw_matches(self, text): """Match named parameters (kwargs) of the last open function""" if "." in text: # a parameter cannot be dotted @@ -1581,42 +1625,54 @@ class IPCompleter(Completer): # Remove used named arguments from the list, no need to show twice for namedArg in set(namedArgs) - usedNamedArgs: if namedArg.startswith(text): - argMatches.append(u"%s=" %namedArg) + argMatches.append("%s=" %namedArg) except: pass return argMatches - def dict_key_matches(self, text): + @staticmethod + def _get_keys(obj: Any) -> List[Any]: + # Objects can define their own completions by defining an + # _ipy_key_completions_() method. + method = get_real_method(obj, '_ipython_key_completions_') + if method is not None: + return method() + + # Special case some common in-memory dict-like types + if isinstance(obj, dict) or\ + _safe_isinstance(obj, 'pandas', 'DataFrame'): + try: + return list(obj.keys()) + except Exception: + return [] + elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ + _safe_isinstance(obj, 'numpy', 'void'): + return obj.dtype.names or [] + return [] + + def dict_key_matches(self, text:str) -> List[str]: "Match string keys in a dictionary, after e.g. 'foo[' " - def get_keys(obj): - # Objects can define their own completions by defining an - # _ipy_key_completions_() method. - method = get_real_method(obj, '_ipython_key_completions_') - if method is not None: - return method() - - # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): - try: - return list(obj.keys()) - except Exception: - return [] - elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ - _safe_isinstance(obj, 'numpy', 'void'): - return obj.dtype.names or [] - return [] - try: + + if self.__dict_key_regexps is not None: regexps = self.__dict_key_regexps - except AttributeError: + else: dict_key_re_fmt = r'''(?x) ( # match dict-referring expression wrt greedy setting %s ) \[ # open bracket \s* # and optional whitespace + # Capture any number of str-like objects (e.g. "a", "b", 'c') + ((?:[uUbB]? # string prefix (r not handled) + (?: + '(?:[^']|(?<!\\)\\')*' + | + "(?:[^"]|(?<!\\)\\")*" + ) + \s*,\s* + )*) ([uUbB]? # string prefix (r not handled) (?: # unclosed string '(?:[^']|(?<!\\)\\')* @@ -1638,10 +1694,11 @@ class IPCompleter(Completer): } match = regexps[self.greedy].search(self.text_until_cursor) + if match is None: return [] - expr, prefix = match.groups() + expr, prefix0, prefix = match.groups() try: obj = eval(expr, self.namespace) except Exception: @@ -1650,10 +1707,13 @@ class IPCompleter(Completer): except Exception: return [] - keys = get_keys(obj) + keys = self._get_keys(obj) if not keys: return keys - closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims) + + extra_prefix = eval(prefix0) if prefix0 != '' else None + + closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix) if not matches: return matches @@ -1663,7 +1723,7 @@ class IPCompleter(Completer): # - the start of the completion text_start = len(self.text_until_cursor) - len(text) if prefix: - key_start = match.start(2) + key_start = match.start(3) completion_start = key_start + token_offset else: key_start = completion_start = match.end() @@ -1695,16 +1755,15 @@ class IPCompleter(Completer): return [leading + k + suf for k in matches] - def unicode_name_matches(self, text): - u"""Match Latex-like syntax for unicode characters base + @staticmethod + def unicode_name_matches(text:str) -> Tuple[str, List[str]] : + """Match Latex-like syntax for unicode characters base on the name of the character. This does ``\\GREEK SMALL LETTER ETA`` -> ``η`` Works only on valid python 3 identifier, or on combining characters that will combine to form a valid identifier. - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1716,11 +1775,11 @@ class IPCompleter(Completer): return '\\'+s,[unic] except KeyError: pass - return u'', [] + return '', [] - def latex_matches(self, text): - u"""Match Latex syntax for unicode characters. + def latex_matches(self, text:str) -> Tuple[str, Sequence[str]]: + """Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` """ @@ -1737,7 +1796,7 @@ class IPCompleter(Completer): matches = [k for k in latex_symbols if k.startswith(s)] if matches: return s, matches - return u'', [] + return '', () def dispatch_custom_completer(self, text): if not self.custom_completers: @@ -1800,18 +1859,18 @@ class IPCompleter(Completer): Parameters ---------- - - text:str + text : str Full text of the current input, multi line string. - offset:int + offset : int Integer representing the position of the cursor in ``text``. Offset is 0-based indexed. Yields ------ - :any:`Completion` object - + Completion + Notes + ----- The cursor on a text can either be seen as being "in between" characters or "On" a character depending on the interface visible to the user. For consistency the cursor being on "in between" characters X @@ -1821,7 +1880,6 @@ class IPCompleter(Completer): Combining characters may span more that one position in the text. - .. note:: If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--`` @@ -1840,7 +1898,15 @@ class IPCompleter(Completer): category=ProvisionalCompleterWarning, stacklevel=2) seen = set() + profiler:Optional[cProfile.Profile] try: + if self.profile_completions: + import cProfile + profiler = cProfile.Profile() + profiler.enable() + else: + profiler = None + for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000): if c and (c in seen): continue @@ -1850,13 +1916,19 @@ class IPCompleter(Completer): """if completions take too long and users send keyboard interrupt, do not crash and return ASAP. """ pass - - def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]: + finally: + if profiler is not None: + profiler.disable() + ensure_dir_exists(self.profiler_output_dir) + output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4())) + print("Writing profiler output to", output_path) + profiler.dump_stats(output_path) + + def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]: """ Core completion module.Same signature as :any:`completions`, with the extra `timeout` parameter (in seconds). - Computing jedi's completion ``.type`` can be quite expensive (it is a lazy property) and can require some warm-up, more warm up than just computing the ``name`` of a completion. The warm-up can be : @@ -1936,7 +2008,7 @@ class IPCompleter(Completer): yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>') - def complete(self, text=None, line_buffer=None, cursor_pos=None): + def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]: """Find completions for the given text and line context. Note that both the text and the line_buffer are optional, but at least @@ -1944,35 +2016,31 @@ class IPCompleter(Completer): Parameters ---------- - text : string, optional + text : string, optional Text to perform the completion on. If not given, the line buffer is split using the instance's CompletionSplitter object. - - line_buffer : string, optional + line_buffer : string, optional If not given, the completer attempts to obtain the current line buffer via readline. This keyword allows clients which are requesting for text completions in non-readline contexts to inform the completer of the entire text. - - cursor_pos : int, optional + cursor_pos : int, optional Index of the cursor in the full line buffer. Should be provided by remote frontends where kernel has no access to frontend state. Returns ------- + Tuple of two items: text : str - Text that was actually used in the completion. - + Text that was actually used in the completion. matches : list - A list of completion matches. - - - .. note:: + A list of completion matches. + Notes + ----- This API is likely to be deprecated and replaced by :any:`IPCompleter.completions` in the future. - """ warnings.warn('`Completer.complete` is pending deprecation since ' 'IPython 6.0 and will be replaced by `Completer.completions`.', @@ -1982,21 +2050,46 @@ class IPCompleter(Completer): return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2] def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, - full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]: + full_text=None) -> _CompleteResult: """ - Like complete but can also returns raw jedi completions as well as the origin of the completion text. This could (and should) be made much cleaner but that will be simpler once we drop the old (and stateful) :any:`complete` API. - With current provisional API, cursor_pos act both (depending on the caller) as the offset in the ``text`` or ``line_buffer``, or as the ``column`` when passing multiline strings this could/should be renamed but would add extra noise. + + Parameters + ---------- + cursor_line + Index of the line the cursor is on. 0 indexed. + cursor_pos + Position of the cursor in the current line/line_buffer/text. 0 + indexed. + line_buffer : optional, str + The current line the cursor is in, this is mostly due to legacy + reason that readline could only give a us the single current line. + Prefer `full_text`. + text : str + The current "token" the cursor is in, mostly also for historical + reasons. as the completer would trigger only after the current line + was parsed. + full_text : str + Full text of the current cell. + + Returns + ------- + A tuple of N elements which are (likely): + matched_text: ? the text that the complete matched + matches: list of completions ? + matches_origin: ? list same length as matches, and where each completion came from + jedi_matches: list of Jedi matches, have it's own structure. """ + # if the cursor position isn't given, the only sane assumption we can # make is that it's at the end of the line (the common case) if cursor_pos is None: @@ -2014,17 +2107,16 @@ class IPCompleter(Completer): if self.backslash_combining_completions: # allow deactivation of these on windows. base_text = text if not line_buffer else line_buffer[:cursor_pos] - latex_text, latex_matches = self.latex_matches(base_text) - if latex_matches: - return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () - name_text = '' - name_matches = [] - # need to add self.fwd_unicode_match() function here when done - for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): + + for meth in (self.latex_matches, + self.unicode_name_matches, + back_latex_name_matches, + back_unicode_name_matches, + self.fwd_unicode_match): name_text, name_matches = meth(base_text) if name_text: - return name_text, name_matches[:MATCHES_LIMIT], \ - [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), () + return _CompleteResult(name_text, name_matches[:MATCHES_LIMIT], \ + [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ()) # If no line buffer is given, assume the input text is all there was @@ -2039,22 +2131,23 @@ class IPCompleter(Completer): matches = list(matcher(line_buffer))[:MATCHES_LIMIT] if matches: origins = [matcher.__qualname__] * len(matches) - return text, matches, origins, () + return _CompleteResult(text, matches, origins, ()) # Start with a clean slate of completions matches = [] - + # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have # richer completion semantics in other environments. - completions = () - if self.use_jedi: + is_magic_prefix = len(text) > 0 and text[0] == "%" + completions: Iterable[Any] = [] + if self.use_jedi and not is_magic_prefix: if not full_text: full_text = line_buffer completions = self._jedi_matches( cursor_pos, cursor_line, full_text) - + if self.merge_completions: matches = [] for matcher in self.matchers: @@ -2092,27 +2185,89 @@ class IPCompleter(Completer): self.matches = _matches - return text, _matches, origins, completions + return _CompleteResult(text, _matches, origins, completions) - def fwd_unicode_match(self, text:str) -> Tuple[str, list]: - if self._names is None: - self._names = [] - for c in range(0,0x10FFFF + 1): - try: - self._names.append(unicodedata.name(chr(c))) - except ValueError: - pass + def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]: + """ + Forward match a string starting with a backslash with a list of + potential Unicode completions. + + Will compute list list of Unicode character names on first call and cache it. + + Returns + ------- + At tuple with: + - matched text (empty if no matches) + - list of potential completions, empty tuple otherwise) + """ + # TODO: self.unicode_names is here a list we traverse each time with ~100k elements. + # We could do a faster match using a Trie. + + # Using pygtrie the following seem to work: + + # s = PrefixSet() + + # for c in range(0,0x10FFFF + 1): + # try: + # s.add(unicodedata.name(chr(c))) + # except ValueError: + # pass + # [''.join(k) for k in s.iter(prefix)] + + # But need to be timed and adds an extra dependency. slashpos = text.rfind('\\') # if text starts with slash if slashpos > -1: - s = text[slashpos+1:] - candidates = [x for x in self._names if x.startswith(s)] + # PERF: It's important that we don't access self._unicode_names + # until we're inside this if-block. _unicode_names is lazily + # initialized, and it takes a user-noticeable amount of time to + # initialize it, so we don't want to initialize it unless we're + # actually going to use it. + s = text[slashpos + 1 :] + sup = s.upper() + candidates = [x for x in self.unicode_names if x.startswith(sup)] + if candidates: + return s, candidates + candidates = [x for x in self.unicode_names if sup in x] + if candidates: + return s, candidates + splitsup = sup.split(" ") + candidates = [ + x for x in self.unicode_names if all(u in x for u in splitsup) + ] if candidates: return s, candidates - else: - return '', () + + return "", () # if text does not start with slash else: - return u'', () + return '', () + + @property + def unicode_names(self) -> List[str]: + """List of names of unicode code points that can be completed. + + The list is lazily initialized on first access. + """ + if self._unicode_names is None: + names = [] + for c in range(0,0x10FFFF + 1): + try: + names.append(unicodedata.name(chr(c))) + except ValueError: + pass + self._unicode_names = _unicode_name_compute(_UNICODE_RANGES) + + return self._unicode_names + +def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]: + names = [] + for start,stop in ranges: + for c in range(start, stop) : + try: + names.append(unicodedata.name(chr(c))) + except ValueError: + pass + return names diff --git a/contrib/python/ipython/py3/IPython/core/completerlib.py b/contrib/python/ipython/py3/IPython/core/completerlib.py index bda665d8a2..65efa42254 100644 --- a/contrib/python/ipython/py3/IPython/core/completerlib.py +++ b/contrib/python/ipython/py3/IPython/core/completerlib.py @@ -201,6 +201,17 @@ def is_importable(module, attr, only_modules): else: return not(attr[:2] == '__' and attr[-2:] == '__') +def is_possible_submodule(module, attr): + try: + obj = getattr(module, attr) + except AttributeError: + # Is possilby an unimported submodule + return True + except TypeError: + # https://github.com/ipython/ipython/issues/9678 + return False + return inspect.ismodule(obj) + def try_import(mod: str, only_modules=False) -> List[str]: """ @@ -220,7 +231,12 @@ def try_import(mod: str, only_modules=False) -> List[str]: completions.extend( [attr for attr in dir(m) if is_importable(m, attr, only_modules)]) - completions.extend(getattr(m, '__all__', [])) + m_all = getattr(m, "__all__", []) + if only_modules: + completions.extend(attr for attr in m_all if is_possible_submodule(m, attr)) + else: + completions.extend(m_all) + if m_is_init: completions.extend(arcadia_module_list(mod)) completions_set = {c for c in completions if isinstance(c, str)} diff --git a/contrib/python/ipython/py3/IPython/core/crashhandler.py b/contrib/python/ipython/py3/IPython/core/crashhandler.py index 1e0b429d09..4af39361e8 100644 --- a/contrib/python/ipython/py3/IPython/core/crashhandler.py +++ b/contrib/python/ipython/py3/IPython/core/crashhandler.py @@ -23,6 +23,7 @@ import os import sys import traceback from pprint import pformat +from pathlib import Path from IPython.core import ultratb from IPython.core.release import author_email @@ -31,6 +32,8 @@ from IPython.utils.py3compat import input from IPython.core.release import __version__ as version +from typing import Optional + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -94,34 +97,40 @@ class CrashHandler(object): message_template = _default_message_template section_sep = '\n\n'+'*'*75+'\n\n' - def __init__(self, app, contact_name=None, contact_email=None, - bug_tracker=None, show_crash_traceback=True, call_pdb=False): + def __init__( + self, + app, + contact_name: Optional[str] = None, + contact_email: Optional[str] = None, + bug_tracker: Optional[str] = None, + show_crash_traceback: bool = True, + call_pdb: bool = False, + ): """Create a new crash handler Parameters ---------- - app : Application + app : Application A running :class:`Application` instance, which will be queried at crash time for internal information. - contact_name : str A string with the name of the person to contact. - contact_email : str A string with the email address of the contact. - bug_tracker : str A string with the URL for your project's bug tracker. - show_crash_traceback : bool If false, don't print the crash traceback on stderr, only generate the on-disk report + call_pdb + Whether to call pdb on crash - Non-argument instance attributes: - + Attributes + ---------- These instances contain some non-argument attributes which allow for further customization of the crash handler's behavior. Please see the source for further details. + """ self.crash_report_fname = "Crash_report_%s.txt" % app.name self.app = app @@ -151,10 +160,10 @@ class CrashHandler(object): try: rptdir = self.app.ipython_dir except: - rptdir = os.getcwd() - if rptdir is None or not os.path.isdir(rptdir): - rptdir = os.getcwd() - report_name = os.path.join(rptdir,self.crash_report_fname) + rptdir = Path.cwd() + if rptdir is None or not Path.is_dir(rptdir): + rptdir = Path.cwd() + report_name = rptdir / self.crash_report_fname # write the report filename into the instance dict so it can get # properly expanded out in the user message template self.crash_report_fname = report_name @@ -176,7 +185,7 @@ class CrashHandler(object): # and generate a complete report on disk try: - report = open(report_name,'w') + report = open(report_name, "w", encoding="utf-8") except: print('Could not create crash report on disk.', file=sys.stderr) return diff --git a/contrib/python/ipython/py3/IPython/core/debugger.py b/contrib/python/ipython/py3/IPython/core/debugger.py index 1744bdb8a8..8e3dd9678c 100644 --- a/contrib/python/ipython/py3/IPython/core/debugger.py +++ b/contrib/python/ipython/py3/IPython/core/debugger.py @@ -69,8 +69,8 @@ or configure it in your ``.pdbrc`` -Licencse --------- +License +------- Modified from the standard pdb.Pdb class to avoid including readline, so that the command line completion of other programs which include this isn't @@ -102,7 +102,6 @@ All the changes since then are under the same license as IPython. #***************************************************************************** import bdb -import functools import inspect import linecache import sys @@ -114,12 +113,13 @@ from IPython import get_ipython from IPython.utils import PyColorize from IPython.utils import coloransi, py3compat from IPython.core.excolors import exception_colors -from IPython.testing.skipdoctest import skip_doctest +# skip module docstests +__skip_doctest__ = True prompt = 'ipdb> ' -#We have to check this directly from sys.argv, config struct not yet available +# We have to check this directly from sys.argv, config struct not yet available from pdb import Pdb as OldPdb # Allow the set_trace code to operate outside of an ipython instance, even if @@ -144,112 +144,15 @@ def BdbQuit_excepthook(et, ev, tb, excepthook=None): All other exceptions are processed using the `excepthook` parameter. """ - warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - if et==bdb.BdbQuit: - print('Exiting Debugger.') - elif excepthook is not None: - excepthook(et, ev, tb) - else: - # Backwards compatibility. Raise deprecation warning? - BdbQuit_excepthook.excepthook_ori(et,ev,tb) - - -def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None): - warnings.warn( - "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - print('Exiting Debugger.') - - -class Tracer(object): - """ - DEPRECATED - - Class for local debugging, similar to pdb.set_trace. - - Instances of this class, when called, behave like pdb.set_trace, but - providing IPython's enhanced capabilities. - - This is implemented as a class which must be initialized in your own code - and not as a standalone function because we need to detect at runtime - whether IPython is already active or not. That detection is done in the - constructor, ensuring that this code plays nicely with a running IPython, - while functioning acceptably (though with limitations) if outside of it. - """ - - @skip_doctest - def __init__(self, colors=None): - """ - DEPRECATED - - Create a local debugger instance. - - Parameters - ---------- + raise ValueError( + "`BdbQuit_excepthook` is deprecated since version 5.1", + ) - colors : str, optional - The name of the color scheme to use, it must be one of IPython's - valid color schemes. If not given, the function will default to - the current IPython scheme when running inside IPython, and to - 'NoColor' otherwise. - Examples - -------- - :: - - from IPython.core.debugger import Tracer; debug_here = Tracer() - - Later in your code:: - - debug_here() # -> will open up the debugger at that point. - - Once the debugger activates, you can use all of its regular commands to - step through code, set breakpoints, etc. See the pdb documentation - from the Python standard library for usage details. - """ - warnings.warn("`Tracer` is deprecated since version 5.1, directly use " - "`IPython.core.debugger.Pdb.set_trace()`", - DeprecationWarning, stacklevel=2) - - ip = get_ipython() - if ip is None: - # Outside of ipython, we set our own exception hook manually - sys.excepthook = functools.partial(BdbQuit_excepthook, - excepthook=sys.excepthook) - def_colors = 'NoColor' - else: - # In ipython, we use its custom exception handler mechanism - def_colors = ip.colors - ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook) - - if colors is None: - colors = def_colors - - # The stdlib debugger internally uses a modified repr from the `repr` - # module, that limits the length of printed strings to a hardcoded - # limit of 30 characters. That much trimming is too aggressive, let's - # at least raise that limit to 80 chars, which should be enough for - # most interactive uses. - try: - from reprlib import aRepr - aRepr.maxstring = 80 - except: - # This is only a user-facing convenience, so any error we encounter - # here can be warned about but can be otherwise ignored. These - # printouts will tell us about problems if this API changes - import traceback - traceback.print_exc() - - self.debugger = Pdb(colors) - - def __call__(self): - """Starts an interactive debugger at the point where called. - - This is similar to the pdb.set_trace() function from the std lib, but - using IPython's enhanced debugger.""" - - self.debugger.set_trace(sys._getframe().f_back) +def BdbQuit_IPython_excepthook(self, et, ev, tb, tb_offset=None): + raise ValueError( + "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", + DeprecationWarning, stacklevel=2) RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+') @@ -291,14 +194,11 @@ class Pdb(OldPdb): "debuggerskip": True, } - def __init__(self, color_scheme=None, completekey=None, - stdin=None, stdout=None, context=5, **kwargs): + def __init__(self, completekey=None, stdin=None, stdout=None, context=5, **kwargs): """Create a new IPython debugger. Parameters ---------- - color_scheme : default None - Deprecated, do not use. completekey : default None Passed to pdb.Pdb. stdin : default None @@ -322,8 +222,8 @@ class Pdb(OldPdb): self.context = int(context) if self.context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) @@ -339,14 +239,10 @@ class Pdb(OldPdb): self.shell = TerminalInteractiveShell.instance() # needed by any code which calls __import__("__main__") after # the debugger was entered. See also #9941. - sys.modules['__main__'] = save_main + sys.modules["__main__"] = save_main - if color_scheme is not None: - warnings.warn( - "The `color_scheme` argument is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - else: - color_scheme = self.shell.colors + + color_scheme = self.shell.colors self.aliases = {} @@ -374,7 +270,6 @@ class Pdb(OldPdb): cst['Neutral'].colors.breakpoint_enabled = C.LightRed cst['Neutral'].colors.breakpoint_disabled = C.Red - # Add a python parser so we can syntax highlight source while # debugging. self.parser = PyColorize.Parser(style=color_scheme) @@ -423,14 +318,14 @@ class Pdb(OldPdb): def hidden_frames(self, stack): """ - Given an index in the stack return wether it should be skipped. + Given an index in the stack return whether it should be skipped. This is used in up/down and where to skip frames. """ # The f_locals dictionary is updated from the actual frame # locals whenever the .f_locals accessor is called, so we # avoid calling it here to preserve self.curframe_locals. - # Futhermore, there is no good reason to hide the current frame. + # Furthermore, there is no good reason to hide the current frame. ip_hide = [self._hidden_predicate(s[0]) for s in stack] ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] if ip_start and self._predicates["ipython_internal"]: @@ -443,13 +338,25 @@ class Pdb(OldPdb): except KeyboardInterrupt: self.stdout.write("\n" + self.shell.get_exception_only()) + def precmd(self, line): + """Perform useful escapes on the command before it is executed.""" + + if line.endswith("??"): + line = "pinfo2 " + line[:-2] + elif line.endswith("?"): + line = "pinfo " + line[:-1] + + line = super().precmd(line) + + return line + def new_do_frame(self, arg): OldPdb.do_frame(self, arg) def new_do_quit(self, arg): if hasattr(self, 'old_all_completions'): - self.shell.Completer.all_completions=self.old_all_completions + self.shell.Completer.all_completions = self.old_all_completions return OldPdb.do_quit(self, arg) @@ -467,11 +374,11 @@ class Pdb(OldPdb): if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e try: skipped = 0 for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): @@ -496,11 +403,11 @@ class Pdb(OldPdb): if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) # vds: >> @@ -511,8 +418,8 @@ class Pdb(OldPdb): def _get_frame_locals(self, frame): """ " - Acessing f_local of current frame reset the namespace, so we want to avoid - that or the following can happend + Accessing f_local of current frame reset the namespace, so we want to avoid + that or the following can happen ipdb> foo "old" @@ -535,25 +442,22 @@ class Pdb(OldPdb): if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: print("Context must be a positive integer", file=self.stdout) except (TypeError, ValueError): print("Context must be a positive integer", file=self.stdout) - try: - import reprlib # Py 3 - except ImportError: - import repr as reprlib # Py 2 + + import reprlib ret = [] Colors = self.color_scheme_table.active_colors ColorsNormal = Colors.Normal - tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal) - tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal) - tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, - ColorsNormal) + tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal) + tpl_call = "%s%%s%s%%s%s" % (Colors.vName, Colors.valEm, ColorsNormal) + tpl_line = "%%s%s%%s %s%%s" % (Colors.lineno, ColorsNormal) + tpl_line_em = "%%s%s%%s %s%%s%s" % (Colors.linenoEm, Colors.line, ColorsNormal) frame, lineno = frame_lineno @@ -587,8 +491,8 @@ class Pdb(OldPdb): if frame is self.curframe: ret.append('> ') else: - ret.append(' ') - ret.append(u'%s(%s)%s\n' % (link,lineno,call)) + ret.append(" ") + ret.append("%s(%s)%s\n" % (link, lineno, call)) start = lineno - 1 - context//2 lines = linecache.getlines(filename) @@ -596,17 +500,17 @@ class Pdb(OldPdb): start = max(start, 0) lines = lines[start : start + context] - for i,line in enumerate(lines): - show_arrow = (start + 1 + i == lineno) - linetpl = (frame is self.curframe or show_arrow) \ - and tpl_line_em \ - or tpl_line - ret.append(self.__format_line(linetpl, filename, - start + 1 + i, line, - arrow = show_arrow) ) - return ''.join(ret) - - def __format_line(self, tpl_line, filename, lineno, line, arrow = False): + for i, line in enumerate(lines): + show_arrow = start + 1 + i == lineno + linetpl = (frame is self.curframe or show_arrow) and tpl_line_em or tpl_line + ret.append( + self.__format_line( + linetpl, filename, start + 1 + i, line, arrow=show_arrow + ) + ) + return "".join(ret) + + def __format_line(self, tpl_line, filename, lineno, line, arrow=False): bp_mark = "" bp_mark_color = "" @@ -636,7 +540,6 @@ class Pdb(OldPdb): return tpl_line % (bp_mark_color + bp_mark, num, line) - def print_list_lines(self, filename, first, last): """The printing (as opposed to the parsing part of a 'list' command.""" @@ -655,9 +558,13 @@ class Pdb(OldPdb): break if lineno == self.curframe.f_lineno: - line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True) + line = self.__format_line( + tpl_line_em, filename, lineno, line, arrow=True + ) else: - line = self.__format_line(tpl_line, filename, lineno, line, arrow = False) + line = self.__format_line( + tpl_line, filename, lineno, line, arrow=False + ) src.append(line) self.lineno = lineno @@ -891,7 +798,6 @@ class Pdb(OldPdb): def break_anywhere(self, frame): """ - _stop_in_decorator_internals is overly restrictive, as we may still want to trace function calls, so we need to also update break_anywhere so that is we don't `stop_here`, because of debugger skip, we may still @@ -909,13 +815,10 @@ class Pdb(OldPdb): return True return False - @skip_doctest def _is_in_decorator_internal_and_should_skip(self, frame): """ Utility to tell us whether we are in a decorator internal and should stop. - - """ # if we are disabled don't skip @@ -937,9 +840,6 @@ class Pdb(OldPdb): return False def stop_here(self, frame): - """Check if pdb should stop here""" - if not super().stop_here(frame): - return False if self._is_in_decorator_internal_and_should_skip(frame) is True: return False @@ -951,9 +851,10 @@ class Pdb(OldPdb): if self.report_skipped: Colors = self.color_scheme_table.active_colors ColorsNormal = Colors.Normal - print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") - return False - return True + print( + f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n" + ) + return super().stop_here(frame) def do_up(self, arg): """u(p) [count] @@ -976,11 +877,9 @@ class Pdb(OldPdb): if count < 0: _newframe = 0 else: - _newindex = self.curindex counter = 0 hidden_frames = self.hidden_frames(self.stack) for i in range(self.curindex - 1, -1, -1): - frame = self.stack[i][0] if hidden_frames[i] and self.skip_hidden: skipped += 1 continue @@ -988,8 +887,10 @@ class Pdb(OldPdb): if counter >= count: break else: - # if no break occured. - self.error("all frames above hidden") + # if no break occurred. + self.error( + "all frames above hidden, use `skip_hidden False` to get get into those." + ) return Colors = self.color_scheme_table.active_colors @@ -1019,12 +920,10 @@ class Pdb(OldPdb): if count < 0: _newframe = len(self.stack) - 1 else: - _newindex = self.curindex counter = 0 skipped = 0 hidden_frames = self.hidden_frames(self.stack) for i in range(self.curindex + 1, len(self.stack)): - frame = self.stack[i][0] if hidden_frames[i] and self.skip_hidden: skipped += 1 continue @@ -1032,7 +931,9 @@ class Pdb(OldPdb): if counter >= count: break else: - self.error("all frames bellow hidden") + self.error( + "all frames below hidden, use `skip_hidden False` to get get into those." + ) return Colors = self.color_scheme_table.active_colors diff --git a/contrib/python/ipython/py3/IPython/core/display.py b/contrib/python/ipython/py3/IPython/core/display.py index f45e7599c9..933295ad6c 100644 --- a/contrib/python/ipython/py3/IPython/core/display.py +++ b/contrib/python/ipython/py3/IPython/core/display.py @@ -5,12 +5,12 @@ # Distributed under the terms of the Modified BSD License. -from binascii import b2a_hex, b2a_base64, hexlify +from binascii import b2a_base64, hexlify +import html import json import mimetypes import os import struct -import sys import warnings from copy import deepcopy from os.path import splitext @@ -18,14 +18,37 @@ from pathlib import Path, PurePath from IPython.utils.py3compat import cast_unicode from IPython.testing.skipdoctest import skip_doctest +from . import display_functions + + +__all__ = ['display_pretty', 'display_html', 'display_markdown', + 'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', + 'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', + 'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', + 'GeoJSON', 'Javascript', 'Image', 'set_matplotlib_formats', + 'set_matplotlib_close', + 'Video'] + +_deprecated_names = ["display", "clear_output", "publish_display_data", "update_display", "DisplayHandle"] + +__all__ = __all__ + _deprecated_names + + +# ----- warn to import from IPython.display ----- + +from warnings import warn + + +def __getattr__(name): + if name in _deprecated_names: + warn(f"Importing {name} from IPython.core.display is deprecated since IPython 7.14, please import from IPython display", DeprecationWarning, stacklevel=2) + return getattr(display_functions, name) + + if name in globals().keys(): + return globals()[name] + else: + raise AttributeError(f"module {__name__} has no attribute {name}") -__all__ = ['display', 'display_pretty', 'display_html', 'display_markdown', -'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', -'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', -'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', -'GeoJSON', 'Javascript', 'Image', 'clear_output', 'set_matplotlib_formats', -'set_matplotlib_close', 'publish_display_data', 'update_display', 'DisplayHandle', -'Video'] #----------------------------------------------------------------------------- # utility functions @@ -38,17 +61,6 @@ def _safe_exists(path): except Exception: return False -def _merge(d1, d2): - """Like update, but merges sub-dicts instead of clobbering at the top level. - - Updates d1 in-place - """ - - if not isinstance(d2, dict) or not isinstance(d1, dict): - return d2 - for key, value in d2.items(): - d1[key] = _merge(d1.get(key), value) - return d1 def _display_mimetype(mimetype, objs, raw=False, metadata=None): """internal implementation of all display_foo methods @@ -71,334 +83,12 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): if raw: # turn list of pngdata into list of { 'image/png': pngdata } objs = [ {mimetype: obj} for obj in objs ] - display(*objs, raw=raw, metadata=metadata, include=[mimetype]) + display_functions.display(*objs, raw=raw, metadata=metadata, include=[mimetype]) #----------------------------------------------------------------------------- # Main functions #----------------------------------------------------------------------------- -# use * to indicate transient is keyword-only -def publish_display_data(data, metadata=None, source=None, *, transient=None, **kwargs): - """Publish data and metadata to all frontends. - - See the ``display_data`` message in the messaging documentation for - more details about this message type. - - Keys of data and metadata can be any mime-type. - - Parameters - ---------- - data : dict - A dictionary having keys that are valid MIME types (like - 'text/plain' or 'image/svg+xml') and values that are the data for - that MIME type. The data itself must be a JSON'able data - structure. Minimally all data should have the 'text/plain' data, - which can be displayed by all frontends. If more than the plain - text is given, it is up to the frontend to decide which - representation to use. - metadata : dict - A dictionary for metadata related to the data. This can contain - arbitrary key, value pairs that frontends can use to interpret - the data. mime-type keys matching those in data can be used - to specify metadata about particular representations. - source : str, deprecated - Unused. - transient : dict, keyword-only - A dictionary of transient data, such as display_id. - """ - from IPython.core.interactiveshell import InteractiveShell - - display_pub = InteractiveShell.instance().display_pub - - # only pass transient if supplied, - # to avoid errors with older ipykernel. - # TODO: We could check for ipykernel version and provide a detailed upgrade message. - if transient: - kwargs['transient'] = transient - - display_pub.publish( - data=data, - metadata=metadata, - **kwargs - ) - - -def _new_id(): - """Generate a new random text id with urandom""" - return b2a_hex(os.urandom(16)).decode('ascii') - - -def display(*objs, include=None, exclude=None, metadata=None, transient=None, display_id=None, **kwargs): - """Display a Python object in all frontends. - - By default all representations will be computed and sent to the frontends. - Frontends can decide which representation is used and how. - - In terminal IPython this will be similar to using :func:`print`, for use in richer - frontends see Jupyter notebook examples with rich display logic. - - Parameters - ---------- - *objs : object - The Python objects to display. - raw : bool, optional - Are the objects to be displayed already mimetype-keyed dicts of raw display data, - or Python objects that need to be formatted before display? [default: False] - include : list, tuple or set, optional - A list of format type strings (MIME types) to include in the - format data dict. If this is set *only* the format types included - in this list will be computed. - exclude : list, tuple or set, optional - A list of format type strings (MIME types) to exclude in the format - data dict. If this is set all format types will be computed, - except for those included in this argument. - metadata : dict, optional - A dictionary of metadata to associate with the output. - mime-type keys in this dictionary will be associated with the individual - representation formats, if they exist. - transient : dict, optional - A dictionary of transient data to associate with the output. - Data in this dict should not be persisted to files (e.g. notebooks). - display_id : str, bool optional - Set an id for the display. - This id can be used for updating this display area later via update_display. - If given as `True`, generate a new `display_id` - clear : bool, optional - Should the output area be cleared before displaying anything? If True, - this will wait for additional output before clearing. [default: False] - kwargs: additional keyword-args, optional - Additional keyword-arguments are passed through to the display publisher. - - Returns - ------- - - handle: DisplayHandle - Returns a handle on updatable displays for use with :func:`update_display`, - if `display_id` is given. Returns :any:`None` if no `display_id` is given - (default). - - Examples - -------- - - >>> class Json(object): - ... def __init__(self, json): - ... self.json = json - ... def _repr_pretty_(self, pp, cycle): - ... import json - ... pp.text(json.dumps(self.json, indent=2)) - ... def __repr__(self): - ... return str(self.json) - ... - - >>> d = Json({1:2, 3: {4:5}}) - - >>> print(d) - {1: 2, 3: {4: 5}} - - >>> display(d) - { - "1": 2, - "3": { - "4": 5 - } - } - - >>> def int_formatter(integer, pp, cycle): - ... pp.text('I'*integer) - - >>> plain = get_ipython().display_formatter.formatters['text/plain'] - >>> plain.for_type(int, int_formatter) - <function _repr_pprint at 0x...> - >>> display(7-5) - II - - >>> del plain.type_printers[int] - >>> display(7-5) - 2 - - See Also - -------- - - :func:`update_display` - - Notes - ----- - - In Python, objects can declare their textual representation using the - `__repr__` method. IPython expands on this idea and allows objects to declare - other, rich representations including: - - - HTML - - JSON - - PNG - - JPEG - - SVG - - LaTeX - - A single object can declare some or all of these representations; all are - handled by IPython's display system. - - The main idea of the first approach is that you have to implement special - display methods when you define your class, one for each representation you - want to use. Here is a list of the names of the special methods and the - values they must return: - - - `_repr_html_`: return raw HTML as a string, or a tuple (see below). - - `_repr_json_`: return a JSONable dict, or a tuple (see below). - - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). - - `_repr_png_`: return raw PNG data, or a tuple (see below). - - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). - - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", - or a tuple (see below). - - `_repr_mimebundle_`: return a full mimebundle containing the mapping - from all mimetypes to data. - Use this for any mime-type not listed above. - - The above functions may also return the object's metadata alonside the - data. If the metadata is available, the functions will return a tuple - containing the data and metadata, in that order. If there is no metadata - available, then the functions will return the data only. - - When you are directly writing your own classes, you can adapt them for - display in IPython by following the above approach. But in practice, you - often need to work with existing classes that you can't easily modify. - - You can refer to the documentation on integrating with the display system in - order to register custom formatters for already existing types - (:ref:`integrating_rich_display`). - - .. versionadded:: 5.4 display available without import - .. versionadded:: 6.1 display available without import - - Since IPython 5.4 and 6.1 :func:`display` is automatically made available to - the user without import. If you are using display in a document that might - be used in a pure python context or with older version of IPython, use the - following import at the top of your file:: - - from IPython.display import display - - """ - from IPython.core.interactiveshell import InteractiveShell - - if not InteractiveShell.initialized(): - # Directly print objects. - print(*objs) - return - - raw = kwargs.pop("raw", False) - clear = kwargs.pop("clear", False) - if transient is None: - transient = {} - if metadata is None: - metadata={} - if display_id: - if display_id is True: - display_id = _new_id() - transient['display_id'] = display_id - if kwargs.get('update') and 'display_id' not in transient: - raise TypeError('display_id required for update_display') - if transient: - kwargs['transient'] = transient - - if not objs and display_id: - # if given no objects, but still a request for a display_id, - # we assume the user wants to insert an empty output that - # can be updated later - objs = [{}] - raw = True - - if not raw: - format = InteractiveShell.instance().display_formatter.format - - if clear: - clear_output(wait=True) - - for obj in objs: - if raw: - publish_display_data(data=obj, metadata=metadata, **kwargs) - else: - format_dict, md_dict = format(obj, include=include, exclude=exclude) - if not format_dict: - # nothing to display (e.g. _ipython_display_ took over) - continue - if metadata: - # kwarg-specified metadata gets precedence - _merge(md_dict, metadata) - publish_display_data(data=format_dict, metadata=md_dict, **kwargs) - if display_id: - return DisplayHandle(display_id) - - -# use * for keyword-only display_id arg -def update_display(obj, *, display_id, **kwargs): - """Update an existing display by id - - Parameters - ---------- - - obj: - The object with which to update the display - display_id: keyword-only - The id of the display to update - - See Also - -------- - - :func:`display` - """ - kwargs['update'] = True - display(obj, display_id=display_id, **kwargs) - - -class DisplayHandle(object): - """A handle on an updatable display - - Call `.update(obj)` to display a new object. - - Call `.display(obj`) to add a new instance of this display, - and update existing instances. - - See Also - -------- - - :func:`display`, :func:`update_display` - - """ - - def __init__(self, display_id=None): - if display_id is None: - display_id = _new_id() - self.display_id = display_id - - def __repr__(self): - return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) - - def display(self, obj, **kwargs): - """Make a new display with my id, updating existing instances. - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to display - """ - display(obj, display_id=self.display_id, **kwargs) - - def update(self, obj, **kwargs): - """Update existing displays with my id - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to update_display - """ - update_display(obj, display_id=self.display_id, **kwargs) - def display_pretty(*objs, **kwargs): """Display the pretty (default) representation of an object. @@ -659,7 +349,8 @@ class DisplayObject(object): def reload(self): """Reload the raw data from file or URL.""" if self.filename is not None: - with open(self.filename, self._read_flags) as f: + encoding = None if "b" in self._read_flags else "utf-8" + with open(self.filename, self._read_flags, encoding=encoding) as f: self.data = f.read() elif self.url is not None: # Deferred import @@ -679,7 +370,11 @@ class DisplayObject(object): if 'gzip' in response.headers['content-encoding']: import gzip from io import BytesIO - with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + + # assume utf-8 if encoding is not specified + with gzip.open( + BytesIO(data), "rt", encoding=encoding or "utf-8" + ) as fp: encoding = None data = fp.read() @@ -792,16 +487,16 @@ class SVG(DisplayObject): pass svg = cast_unicode(svg) self._data = svg - + def _repr_svg_(self): return self._data_and_metadata() class ProgressBar(DisplayObject): - """Progressbar supports displaying a progressbar like element + """Progressbar supports displaying a progressbar like element """ def __init__(self, total): """Creates a new progressbar - + Parameters ---------- total : int @@ -827,10 +522,10 @@ class ProgressBar(DisplayObject): self.html_width, self.total, self.progress) def display(self): - display(self, display_id=self._display_id) + display_functions.display(self, display_id=self._display_id) def update(self): - display(self, display_id=self._display_id, update=True) + display_functions.display(self, display_id=self._display_id, update=True) @property def progress(self): @@ -878,10 +573,10 @@ class JSON(DisplayObject): Path to a local file to load the data from. expanded : boolean Metadata to control whether a JSON display component is expanded. - metadata: dict + metadata : dict Specify extra metadata to attach to the json display object. root : str - The name of the root element of the JSON tree + The name of the root element of the JSON tree """ self.metadata = { 'expanded': expanded, @@ -944,7 +639,7 @@ class GeoJSON(JSON): Scalar types (None, number, string) are not allowed, only dict containers. """ - + def __init__(self, *args, **kwargs): """Create a GeoJSON display object given raw data. @@ -962,12 +657,11 @@ class GeoJSON(JSON): A URL to download the data from. filename : unicode Path to a local file to load the data from. - metadata: dict + metadata : dict Specify extra metadata to attach to the json display object. Examples -------- - The following will display an interactive map of Mars with a point of interest on frontend that do support GeoJSON display. @@ -993,7 +687,7 @@ class GeoJSON(JSON): the GeoJSON object. """ - + super(GeoJSON, self).__init__(*args, **kwargs) @@ -1005,7 +699,7 @@ class GeoJSON(JSON): metadata = { 'application/geo+json': self.metadata } - display(bundle, metadata=metadata, raw=True) + display_functions.display(bundle, metadata=metadata, raw=True) class Javascript(TextDisplayObject): @@ -1034,7 +728,7 @@ class Javascript(TextDisplayObject): running the source code. The full URLs of the libraries should be given. A single Javascript library URL can also be given as a string. - css: : list or str + css : list or str A sequence of css files to load before running the source code. The full URLs of the css files should be given. A single css URL can also be given as a string. @@ -1112,9 +806,20 @@ class Image(DisplayObject): _FMT_GIF: 'image/gif', } - def __init__(self, data=None, url=None, filename=None, format=None, - embed=None, width=None, height=None, retina=False, - unconfined=False, metadata=None): + def __init__( + self, + data=None, + url=None, + filename=None, + format=None, + embed=None, + width=None, + height=None, + retina=False, + unconfined=False, + metadata=None, + alt=None, + ): """Create a PNG/JPEG/GIF image object given raw data. When this object is returned by an input cell or passed to the @@ -1126,15 +831,19 @@ class Image(DisplayObject): data : unicode, str or bytes The raw image data or a URL or filename to load the data from. This always results in embedded image data. + url : unicode A URL to download the data from. If you specify `url=`, the image data will not be embedded unless you also specify `embed=True`. + filename : unicode Path to a local file to load the data from. Images from a file are always embedded. + format : unicode The format of the image data (png/jpeg/jpg/gif). If a filename or URL is given for format will be inferred from the filename extension. + embed : bool Should the image data be embedded using a data URI (True) or be loaded using an <img> tag. Set this to True if you want the image @@ -1144,10 +853,13 @@ class Image(DisplayObject): default value is `False`. Note that QtConsole is not able to display images if `embed` is set to `False` + width : int Width in pixels to which to constrain the image in html + height : int Height in pixels to which to constrain the image in html + retina : bool Automatically set the width and height to half of the measured width and height. @@ -1155,25 +867,38 @@ class Image(DisplayObject): from image data. For non-embedded images, you can just set the desired display width and height directly. - unconfined: bool + + unconfined : bool Set unconfined=True to disable max-width confinement of the image. - metadata: dict + + metadata : dict Specify extra metadata to attach to the image. + alt : unicode + Alternative text for the image, for use by screen readers. + Examples -------- - # embedded image data, works in qtconsole and notebook - # when passed positionally, the first arg can be any of raw image data, - # a URL, or a filename from which to load image data. - # The result is always embedding image data for inline images. - Image('http://www.google.fr/images/srpr/logo3w.png') - Image('/path/to/image.jpg') - Image(b'RAW_PNG_DATA...') - - # Specifying Image(url=...) does not embed the image data, - # it only generates `<img>` tag with a link to the source. - # This will not work in the qtconsole or offline. - Image(url='http://www.google.fr/images/srpr/logo3w.png') + embedded image data, works in qtconsole and notebook + when passed positionally, the first arg can be any of raw image data, + a URL, or a filename from which to load image data. + The result is always embedding image data for inline images. + + >>> Image('https://www.google.fr/images/srpr/logo3w.png') # doctest: +SKIP + <IPython.core.display.Image object> + + >>> Image('/path/to/image.jpg') + <IPython.core.display.Image object> + + >>> Image(b'RAW_PNG_DATA...') + <IPython.core.display.Image object> + + Specifying Image(url=...) does not embed the image data, + it only generates ``<img>`` tag with a link to the source. + This will not work in the qtconsole or offline. + + >>> Image(url='https://www.google.fr/images/srpr/logo3w.png') + <IPython.core.display.Image object> """ if isinstance(data, (Path, PurePath)): @@ -1228,7 +953,8 @@ class Image(DisplayObject): self.height = height self.retina = retina self.unconfined = unconfined - super(Image, self).__init__(data=data, url=url, filename=filename, + self.alt = alt + super(Image, self).__init__(data=data, url=url, filename=filename, metadata=metadata) if self.width is None and self.metadata.get('width', {}): @@ -1237,6 +963,9 @@ class Image(DisplayObject): if self.height is None and self.metadata.get('height', {}): self.height = metadata['height'] + if self.alt is None and self.metadata.get("alt", {}): + self.alt = metadata["alt"] + if retina: self._retina_shape() @@ -1266,18 +995,21 @@ class Image(DisplayObject): def _repr_html_(self): if not self.embed: - width = height = klass = '' + width = height = klass = alt = "" if self.width: width = ' width="%d"' % self.width if self.height: height = ' height="%d"' % self.height if self.unconfined: klass = ' class="unconfined"' - return u'<img src="{url}"{width}{height}{klass}/>'.format( + if self.alt: + alt = ' alt="%s"' % html.escape(self.alt) + return '<img src="{url}"{width}{height}{klass}{alt}/>'.format( url=self.url, width=width, height=height, klass=klass, + alt=alt, ) def _repr_mimebundle_(self, include=None, exclude=None): @@ -1298,9 +1030,9 @@ class Image(DisplayObject): """shortcut for returning metadata with shape information, if defined""" try: b64_data = b2a_base64(self.data).decode('ascii') - except TypeError: + except TypeError as e: raise FileNotFoundError( - "No such file or directory: '%s'" % (self.data)) + "No such file or directory: '%s'" % (self.data)) from e md = {} if self.metadata: md.update(self.metadata) @@ -1310,6 +1042,8 @@ class Image(DisplayObject): md['height'] = self.height if self.unconfined: md['unconfined'] = self.unconfined + if self.alt: + md["alt"] = self.alt if md or always_both: return b64_data, md else: @@ -1348,12 +1082,15 @@ class Video(DisplayObject): data : unicode, str or bytes The raw video data or a URL or filename to load the data from. Raw data will require passing ``embed=True``. + url : unicode A URL for the video. If you specify ``url=``, the image data will not be embedded. + filename : unicode Path to a local file containing the video. Will be interpreted as a local URL unless ``embed=True``. + embed : bool Should the video be embedded using a data URI (True) or be loaded using a <video> tag (False). @@ -1365,15 +1102,18 @@ class Video(DisplayObject): Video('./video.mp4') - mimetype: unicode + mimetype : unicode Specify the mimetype for embedded videos. Default will be guessed from file extension, if available. + width : int Width in pixels to which to constrain the video in HTML. If not supplied, defaults to the width of the video. + height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str Attributes for the HTML ``<video>`` block. Default: ``"controls"`` to get video controls. @@ -1382,7 +1122,6 @@ class Video(DisplayObject): Examples -------- - :: Video('https://archive.org/download/Sita_Sings_the_Blues/Sita_Sings_the_Blues_small.mp4') @@ -1397,7 +1136,7 @@ class Video(DisplayObject): if url is None and isinstance(data, str) and data.startswith(('http:', 'https:')): url = data data = None - elif os.path.exists(data): + elif data is not None and os.path.exists(data): filename = data data = None @@ -1459,23 +1198,6 @@ class Video(DisplayObject): pass -def clear_output(wait=False): - """Clear the output of the current cell receiving output. - - Parameters - ---------- - wait : bool [default: false] - Wait to clear the output until new output is available to replace it.""" - from IPython.core.interactiveshell import InteractiveShell - if InteractiveShell.initialized(): - InteractiveShell.instance().display_pub.clear_output(wait) - else: - print('\033[2K\r', end='') - sys.stdout.flush() - print('\033[2K\r', end='') - sys.stderr.flush() - - @skip_doctest def set_matplotlib_formats(*formats, **kwargs): """ @@ -1498,7 +1220,7 @@ def set_matplotlib_formats(*formats, **kwargs): ---------- *formats : strs One or more figure formats to enable: 'png', 'retina', 'jpeg', 'svg', 'pdf'. - **kwargs : + **kwargs Keyword args will be relayed to ``figure.canvas.print_figure``. """ warnings.warn( @@ -1521,7 +1243,6 @@ def set_matplotlib_close(close=True): use `matplotlib_inline.backend_inline.set_matplotlib_close()` - Set whether the inline backend closes all figures automatically or not. By default, the inline backend used in the IPython Notebook will close all diff --git a/contrib/python/ipython/py3/IPython/core/display_functions.py b/contrib/python/ipython/py3/IPython/core/display_functions.py new file mode 100644 index 0000000000..567cf3fa60 --- /dev/null +++ b/contrib/python/ipython/py3/IPython/core/display_functions.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +"""Top-level display functions for displaying object in different formats.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + + +from binascii import b2a_hex +import os +import sys +import warnings + +__all__ = ['display', 'clear_output', 'publish_display_data', 'update_display', 'DisplayHandle'] + +#----------------------------------------------------------------------------- +# utility functions +#----------------------------------------------------------------------------- + + +def _merge(d1, d2): + """Like update, but merges sub-dicts instead of clobbering at the top level. + + Updates d1 in-place + """ + + if not isinstance(d2, dict) or not isinstance(d1, dict): + return d2 + for key, value in d2.items(): + d1[key] = _merge(d1.get(key), value) + return d1 + + +#----------------------------------------------------------------------------- +# Main functions +#----------------------------------------------------------------------------- + +class _Sentinel: + def __repr__(self): + return "<deprecated>" + + +_sentinel = _Sentinel() + +# use * to indicate transient is keyword-only +def publish_display_data( + data, metadata=None, source=_sentinel, *, transient=None, **kwargs +): + """Publish data and metadata to all frontends. + + See the ``display_data`` message in the messaging documentation for + more details about this message type. + + Keys of data and metadata can be any mime-type. + + Parameters + ---------- + data : dict + A dictionary having keys that are valid MIME types (like + 'text/plain' or 'image/svg+xml') and values that are the data for + that MIME type. The data itself must be a JSON'able data + structure. Minimally all data should have the 'text/plain' data, + which can be displayed by all frontends. If more than the plain + text is given, it is up to the frontend to decide which + representation to use. + metadata : dict + A dictionary for metadata related to the data. This can contain + arbitrary key, value pairs that frontends can use to interpret + the data. mime-type keys matching those in data can be used + to specify metadata about particular representations. + source : str, deprecated + Unused. + transient : dict, keyword-only + A dictionary of transient data, such as display_id. + """ + from IPython.core.interactiveshell import InteractiveShell + + if source is not _sentinel: + warnings.warn( + "The `source` parameter emit a deprecation warning since" + " IPython 8.0, it had no effects for a long time and will " + " be removed in future versions.", + DeprecationWarning, + stacklevel=2, + ) + display_pub = InteractiveShell.instance().display_pub + + # only pass transient if supplied, + # to avoid errors with older ipykernel. + # TODO: We could check for ipykernel version and provide a detailed upgrade message. + if transient: + kwargs['transient'] = transient + + display_pub.publish( + data=data, + metadata=metadata, + **kwargs + ) + + +def _new_id(): + """Generate a new random text id with urandom""" + return b2a_hex(os.urandom(16)).decode('ascii') + + +def display( + *objs, + include=None, + exclude=None, + metadata=None, + transient=None, + display_id=None, + raw=False, + clear=False, + **kwargs +): + """Display a Python object in all frontends. + + By default all representations will be computed and sent to the frontends. + Frontends can decide which representation is used and how. + + In terminal IPython this will be similar to using :func:`print`, for use in richer + frontends see Jupyter notebook examples with rich display logic. + + Parameters + ---------- + *objs : object + The Python objects to display. + raw : bool, optional + Are the objects to be displayed already mimetype-keyed dicts of raw display data, + or Python objects that need to be formatted before display? [default: False] + include : list, tuple or set, optional + A list of format type strings (MIME types) to include in the + format data dict. If this is set *only* the format types included + in this list will be computed. + exclude : list, tuple or set, optional + A list of format type strings (MIME types) to exclude in the format + data dict. If this is set all format types will be computed, + except for those included in this argument. + metadata : dict, optional + A dictionary of metadata to associate with the output. + mime-type keys in this dictionary will be associated with the individual + representation formats, if they exist. + transient : dict, optional + A dictionary of transient data to associate with the output. + Data in this dict should not be persisted to files (e.g. notebooks). + display_id : str, bool optional + Set an id for the display. + This id can be used for updating this display area later via update_display. + If given as `True`, generate a new `display_id` + clear : bool, optional + Should the output area be cleared before displaying anything? If True, + this will wait for additional output before clearing. [default: False] + **kwargs : additional keyword-args, optional + Additional keyword-arguments are passed through to the display publisher. + + Returns + ------- + handle: DisplayHandle + Returns a handle on updatable displays for use with :func:`update_display`, + if `display_id` is given. Returns :any:`None` if no `display_id` is given + (default). + + Examples + -------- + >>> class Json(object): + ... def __init__(self, json): + ... self.json = json + ... def _repr_pretty_(self, pp, cycle): + ... import json + ... pp.text(json.dumps(self.json, indent=2)) + ... def __repr__(self): + ... return str(self.json) + ... + + >>> d = Json({1:2, 3: {4:5}}) + + >>> print(d) + {1: 2, 3: {4: 5}} + + >>> display(d) + { + "1": 2, + "3": { + "4": 5 + } + } + + >>> def int_formatter(integer, pp, cycle): + ... pp.text('I'*integer) + + >>> plain = get_ipython().display_formatter.formatters['text/plain'] + >>> plain.for_type(int, int_formatter) + <function _repr_pprint at 0x...> + >>> display(7-5) + II + + >>> del plain.type_printers[int] + >>> display(7-5) + 2 + + See Also + -------- + :func:`update_display` + + Notes + ----- + In Python, objects can declare their textual representation using the + `__repr__` method. IPython expands on this idea and allows objects to declare + other, rich representations including: + + - HTML + - JSON + - PNG + - JPEG + - SVG + - LaTeX + + A single object can declare some or all of these representations; all are + handled by IPython's display system. + + The main idea of the first approach is that you have to implement special + display methods when you define your class, one for each representation you + want to use. Here is a list of the names of the special methods and the + values they must return: + + - `_repr_html_`: return raw HTML as a string, or a tuple (see below). + - `_repr_json_`: return a JSONable dict, or a tuple (see below). + - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). + - `_repr_png_`: return raw PNG data, or a tuple (see below). + - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). + - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", + or a tuple (see below). + - `_repr_mimebundle_`: return a full mimebundle containing the mapping + from all mimetypes to data. + Use this for any mime-type not listed above. + + The above functions may also return the object's metadata alonside the + data. If the metadata is available, the functions will return a tuple + containing the data and metadata, in that order. If there is no metadata + available, then the functions will return the data only. + + When you are directly writing your own classes, you can adapt them for + display in IPython by following the above approach. But in practice, you + often need to work with existing classes that you can't easily modify. + + You can refer to the documentation on integrating with the display system in + order to register custom formatters for already existing types + (:ref:`integrating_rich_display`). + + .. versionadded:: 5.4 display available without import + .. versionadded:: 6.1 display available without import + + Since IPython 5.4 and 6.1 :func:`display` is automatically made available to + the user without import. If you are using display in a document that might + be used in a pure python context or with older version of IPython, use the + following import at the top of your file:: + + from IPython.display import display + + """ + from IPython.core.interactiveshell import InteractiveShell + + if not InteractiveShell.initialized(): + # Directly print objects. + print(*objs) + return + + if transient is None: + transient = {} + if metadata is None: + metadata={} + if display_id: + if display_id is True: + display_id = _new_id() + transient['display_id'] = display_id + if kwargs.get('update') and 'display_id' not in transient: + raise TypeError('display_id required for update_display') + if transient: + kwargs['transient'] = transient + + if not objs and display_id: + # if given no objects, but still a request for a display_id, + # we assume the user wants to insert an empty output that + # can be updated later + objs = [{}] + raw = True + + if not raw: + format = InteractiveShell.instance().display_formatter.format + + if clear: + clear_output(wait=True) + + for obj in objs: + if raw: + publish_display_data(data=obj, metadata=metadata, **kwargs) + else: + format_dict, md_dict = format(obj, include=include, exclude=exclude) + if not format_dict: + # nothing to display (e.g. _ipython_display_ took over) + continue + if metadata: + # kwarg-specified metadata gets precedence + _merge(md_dict, metadata) + publish_display_data(data=format_dict, metadata=md_dict, **kwargs) + if display_id: + return DisplayHandle(display_id) + + +# use * for keyword-only display_id arg +def update_display(obj, *, display_id, **kwargs): + """Update an existing display by id + + Parameters + ---------- + obj + The object with which to update the display + display_id : keyword-only + The id of the display to update + + See Also + -------- + :func:`display` + """ + kwargs['update'] = True + display(obj, display_id=display_id, **kwargs) + + +class DisplayHandle(object): + """A handle on an updatable display + + Call `.update(obj)` to display a new object. + + Call `.display(obj`) to add a new instance of this display, + and update existing instances. + + See Also + -------- + + :func:`display`, :func:`update_display` + + """ + + def __init__(self, display_id=None): + if display_id is None: + display_id = _new_id() + self.display_id = display_id + + def __repr__(self): + return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) + + def display(self, obj, **kwargs): + """Make a new display with my id, updating existing instances. + + Parameters + ---------- + obj + object to display + **kwargs + additional keyword arguments passed to display + """ + display(obj, display_id=self.display_id, **kwargs) + + def update(self, obj, **kwargs): + """Update existing displays with my id + + Parameters + ---------- + obj + object to display + **kwargs + additional keyword arguments passed to update_display + """ + update_display(obj, display_id=self.display_id, **kwargs) + + +def clear_output(wait=False): + """Clear the output of the current cell receiving output. + + Parameters + ---------- + wait : bool [default: false] + Wait to clear the output until new output is available to replace it.""" + from IPython.core.interactiveshell import InteractiveShell + if InteractiveShell.initialized(): + InteractiveShell.instance().display_pub.clear_output(wait) + else: + print('\033[2K\r', end='') + sys.stdout.flush() + print('\033[2K\r', end='') + sys.stderr.flush() diff --git a/contrib/python/ipython/py3/IPython/core/displayhook.py b/contrib/python/ipython/py3/IPython/core/displayhook.py index 3c06675e86..578e783ab8 100644 --- a/contrib/python/ipython/py3/IPython/core/displayhook.py +++ b/contrib/python/ipython/py3/IPython/core/displayhook.py @@ -146,7 +146,7 @@ class DisplayHook(Configurable): MIME type representation of the object. md_dict is a :class:`dict` with the same MIME type keys of metadata associated with each output. - + """ return self.shell.display_formatter.format(result) diff --git a/contrib/python/ipython/py3/IPython/core/displaypub.py b/contrib/python/ipython/py3/IPython/core/displaypub.py index 1da0458cf0..74028ec79e 100644 --- a/contrib/python/ipython/py3/IPython/core/displaypub.py +++ b/contrib/python/ipython/py3/IPython/core/displaypub.py @@ -22,7 +22,7 @@ from traitlets.config.configurable import Configurable from traitlets import List # This used to be defined here - it is imported for backwards compatibility -from .display import publish_display_data +from .display_functions import publish_display_data #----------------------------------------------------------------------------- # Main payload class @@ -94,11 +94,11 @@ class DisplayPublisher(Configurable): the data itself. source : str, deprecated Unused. - transient: dict, keyword-only + transient : dict, keyword-only A dictionary for transient data. Data in this dictionary should not be persisted as part of saving this output. Examples include 'display_id'. - update: bool, keyword-only, default: False + update : bool, keyword-only, default: False If True, only update existing outputs with the same display_id, rather than creating a new output. """ diff --git a/contrib/python/ipython/py3/IPython/core/events.py b/contrib/python/ipython/py3/IPython/core/events.py index 1af13ca406..73fc181ae5 100644 --- a/contrib/python/ipython/py3/IPython/core/events.py +++ b/contrib/python/ipython/py3/IPython/core/events.py @@ -28,34 +28,34 @@ class EventManager(object): """ def __init__(self, shell, available_events): """Initialise the :class:`CallbackManager`. - + Parameters ---------- shell - The :class:`~IPython.core.interactiveshell.InteractiveShell` instance - available_callbacks - An iterable of names for callback events. + The :class:`~IPython.core.interactiveshell.InteractiveShell` instance + available_events + An iterable of names for callback events. """ self.shell = shell self.callbacks = {n:[] for n in available_events} def register(self, event, function): """Register a new event callback. - + Parameters ---------- event : str - The event for which to register this callback. + The event for which to register this callback. function : callable - A function to be called on the given event. It should take the same - parameters as the appropriate callback prototype. - + A function to be called on the given event. It should take the same + parameters as the appropriate callback prototype. + Raises ------ TypeError - If ``function`` is not callable. + If ``function`` is not callable. KeyError - If ``event`` is not one of the known events. + If ``event`` is not one of the known events. """ if not callable(function): raise TypeError('Need a callable, got %r' % function) @@ -80,7 +80,7 @@ class EventManager(object): def trigger(self, event, *args, **kwargs): """Call callbacks for ``event``. - + Any additional arguments are passed to all callbacks registered for this event. Exceptions raised by callbacks are caught, and a message printed. """ @@ -109,7 +109,7 @@ def _define_event(callback_function): @_define_event def pre_execute(): """Fires before code is executed in response to user/frontend action. - + This includes comm and widget messages and silent execution, as well as user code cells. """ @@ -122,14 +122,14 @@ def pre_run_cell(info): Parameters ---------- info : :class:`~IPython.core.interactiveshell.ExecutionInfo` - An object containing information used for the code execution. + An object containing information used for the code execution. """ pass @_define_event def post_execute(): """Fires after code is executed in response to user/frontend action. - + This includes comm and widget messages and silent execution, as well as user code cells. """ @@ -142,20 +142,20 @@ def post_run_cell(result): Parameters ---------- result : :class:`~IPython.core.interactiveshell.ExecutionResult` - The object which will be returned as the execution result. + The object which will be returned as the execution result. """ pass @_define_event def shell_initialized(ip): """Fires after initialisation of :class:`~IPython.core.interactiveshell.InteractiveShell`. - + This is before extensions and startup scripts are loaded, so it can only be set by subclassing. - + Parameters ---------- ip : :class:`~IPython.core.interactiveshell.InteractiveShell` - The newly initialised shell. + The newly initialised shell. """ pass diff --git a/contrib/python/ipython/py3/IPython/core/excolors.py b/contrib/python/ipython/py3/IPython/core/excolors.py index 487bde18c8..c47ce922c4 100644 --- a/contrib/python/ipython/py3/IPython/core/excolors.py +++ b/contrib/python/ipython/py3/IPython/core/excolors.py @@ -164,21 +164,3 @@ def exception_colors(): ex_colors.add_scheme(ex_colors['Linux'].copy('Neutral')) return ex_colors - -class Deprec(object): - - def __init__(self, wrapped_obj): - self.wrapped=wrapped_obj - - def __getattr__(self, name): - val = getattr(self.wrapped, name) - warnings.warn("Using ExceptionColors global is deprecated and will be removed in IPython 6.0", - DeprecationWarning, stacklevel=2) - # using getattr after warnings break ipydoctest in weird way for 3.5 - return val - -# For backwards compatibility, keep around a single global object. Note that -# this should NOT be used, the factory function should be used instead, since -# these objects are stateful and it's very easy to get strange bugs if any code -# modifies the module-level object's state. -ExceptionColors = Deprec(exception_colors()) diff --git a/contrib/python/ipython/py3/IPython/core/extensions.py b/contrib/python/ipython/py3/IPython/core/extensions.py index bf5e0ad06c..4ca266869c 100644 --- a/contrib/python/ipython/py3/IPython/core/extensions.py +++ b/contrib/python/ipython/py3/IPython/core/extensions.py @@ -19,6 +19,9 @@ from traitlets import Instance # Main class #----------------------------------------------------------------------------- +BUILTINS_EXTS = {"storemagic": False, "autoreload": False} + + class ExtensionManager(Configurable): """A class to manage IPython extensions. @@ -62,13 +65,22 @@ class ExtensionManager(Configurable): def _on_ipython_dir_changed(self, change): ensure_dir_exists(self.ipython_extension_dir) - def load_extension(self, module_str): + def load_extension(self, module_str: str): """Load an IPython extension by its module name. Returns the string "already loaded" if the extension is already loaded, "no load function" if the module doesn't have a load_ipython_extension function, or None if it succeeded. """ + try: + return self._load_extension(module_str) + except ModuleNotFoundError: + if module_str in BUILTINS_EXTS: + BUILTINS_EXTS[module_str] = True + return self._load_extension("IPython.extensions." + module_str) + raise + + def _load_extension(self, module_str: str): if module_str in self.loaded: return "already loaded" @@ -89,19 +101,21 @@ class ExtensionManager(Configurable): else: return "no load function" - def unload_extension(self, module_str): + def unload_extension(self, module_str: str): """Unload an IPython extension by its module name. This function looks up the extension's name in ``sys.modules`` and simply calls ``mod.unload_ipython_extension(self)``. - + Returns the string "no unload function" if the extension doesn't define a function to unload itself, "not loaded" if the extension isn't loaded, otherwise None. """ + if BUILTINS_EXTS.get(module_str, False) is True: + module_str = "IPython.extensions." + module_str if module_str not in self.loaded: return "not loaded" - + if module_str in sys.modules: mod = sys.modules[module_str] if self._call_unload_ipython_extension(mod): @@ -109,7 +123,7 @@ class ExtensionManager(Configurable): else: return "no unload function" - def reload_extension(self, module_str): + def reload_extension(self, module_str: str): """Reload an IPython extension by calling reload. If the module has not been loaded before, @@ -119,6 +133,9 @@ class ExtensionManager(Configurable): """ from IPython.utils.syspathcontext import prepended_to_syspath + if BUILTINS_EXTS.get(module_str, False) is True: + module_str = "IPython.extensions." + module_str + if (module_str in self.loaded) and (module_str in sys.modules): self.unload_extension(module_str) mod = sys.modules[module_str] diff --git a/contrib/python/ipython/py3/IPython/core/formatters.py b/contrib/python/ipython/py3/IPython/core/formatters.py index c13caab91a..4e0b9e455a 100644 --- a/contrib/python/ipython/py3/IPython/core/formatters.py +++ b/contrib/python/ipython/py3/IPython/core/formatters.py @@ -121,19 +121,17 @@ class DisplayFormatter(Configurable): Returns ------- (format_dict, metadata_dict) : tuple of two dicts - format_dict is a dictionary of key/value pairs, one of each format that was generated for the object. The keys are the format types, which will usually be MIME type strings and the values and JSON'able data structure containing the raw data for the representation in that format. - + metadata_dict is a dictionary of metadata about each mime-type output. Its keys will be a strict subset of the keys in format_dict. Notes ----- - If an object implement `_repr_mimebundle_` as well as various `_repr_*_`, the data returned by `_repr_mimebundle_` will take precedence and the corresponding `_repr_*_` for this mimetype will @@ -263,7 +261,7 @@ class FormatterABC(metaclass=abc.ABCMeta): def _mod_name_key(typ): """Return a (__module__, __name__) tuple for a type. - + Used as key in Formatter.deferred_printers. """ module = getattr(typ, '__module__', None) @@ -358,7 +356,7 @@ class BaseFormatter(Configurable): def _check_return(self, r, obj): """Check that a return value is appropriate - + Return the value if so, None otherwise, warning if invalid. """ if r is None or isinstance(r, self._return_type) or \ @@ -373,10 +371,10 @@ class BaseFormatter(Configurable): def lookup(self, obj): """Look up the formatter for a given instance. - + Parameters ---------- - obj : object instance + obj : object instance Returns ------- @@ -399,7 +397,7 @@ class BaseFormatter(Configurable): Parameters ---------- - typ : type or '__module__.__name__' string for a type + typ : type or '__module__.__name__' string for a type Returns ------- @@ -430,21 +428,22 @@ class BaseFormatter(Configurable): def for_type(self, typ, func=None): """Add a format function for a given type. - + Parameters ---------- typ : type or '__module__.__name__' string for a type The class of the object that will be formatted using `func`. + func : callable A callable for computing the format data. `func` will be called with the object to be formatted, and will return the raw data in this formatter's format. Subclasses may use a different call signature for the `func` argument. - + If `func` is None or not specified, there will be no change, only returning the current value. - + Returns ------- oldfunc : callable @@ -476,18 +475,20 @@ class BaseFormatter(Configurable): type_module : str The full dotted name of the module the type is defined in, like ``numpy``. + type_name : str The name of the type (the class name), like ``dtype`` + func : callable A callable for computing the format data. `func` will be called with the object to be formatted, and will return the raw data in this formatter's format. Subclasses may use a different call signature for the `func` argument. - + If `func` is None or unspecified, there will be no change, only returning the current value. - + Returns ------- oldfunc : callable @@ -636,24 +637,23 @@ class PlainTextFormatter(BaseFormatter): This parameter can be set via the '%precision' magic. """ - new = change['new'] if '%' in new: # got explicit format string fmt = new try: fmt%3.14159 - except Exception: - raise ValueError("Precision must be int or format string, not %r"%new) + except Exception as e: + raise ValueError("Precision must be int or format string, not %r"%new) from e elif new: # otherwise, should be an int try: i = int(new) assert i >= 0 - except ValueError: - raise ValueError("Precision must be int or format string, not %r"%new) - except AssertionError: - raise ValueError("int precision must be non-negative, not %r"%i) + except ValueError as e: + raise ValueError("Precision must be int or format string, not %r"%new) from e + except AssertionError as e: + raise ValueError("int precision must be non-negative, not %r"%i) from e fmt = '%%.%if'%i if 'numpy' in sys.modules: @@ -678,6 +678,11 @@ class PlainTextFormatter(BaseFormatter): def _type_printers_default(self): d = pretty._type_pprinters.copy() d[float] = lambda obj,p,cycle: p.text(self.float_format%obj) + # if NumPy is used, set precision for its float64 type + if "numpy" in sys.modules: + import numpy + + d[numpy.float64] = lambda obj, p, cycle: p.text(self.float_format % obj) return d @default('deferred_printers') @@ -823,7 +828,7 @@ class JSONFormatter(BaseFormatter): def _check_return(self, r, obj): """Check that a return value is appropriate - + Return the value if so, None otherwise, warning if invalid. """ if r is None: @@ -832,13 +837,11 @@ class JSONFormatter(BaseFormatter): if isinstance(r, tuple): # unpack data, metadata tuple for type checking on first element r, md = r - - # handle deprecated JSON-as-string form from IPython < 3 - if isinstance(r, str): - warnings.warn("JSON expects JSONable list/dict containers, not JSON strings", - FormatterWarning) - r = json.loads(r) - + + assert not isinstance( + r, str + ), "JSON-as-string has been deprecated since IPython < 3" + if md is not None: # put the tuple back together r = (r, md) diff --git a/contrib/python/ipython/py3/IPython/core/getipython.py b/contrib/python/ipython/py3/IPython/core/getipython.py index e6d8a4c91d..5e9b13cf3c 100644 --- a/contrib/python/ipython/py3/IPython/core/getipython.py +++ b/contrib/python/ipython/py3/IPython/core/getipython.py @@ -16,7 +16,7 @@ def get_ipython(): """Get the global InteractiveShell instance. - + Returns None if no InteractiveShell instance is registered. """ from IPython.core.interactiveshell import InteractiveShell diff --git a/contrib/python/ipython/py3/IPython/core/history.py b/contrib/python/ipython/py3/IPython/core/history.py index 98373f279c..9b0b2cbd04 100644 --- a/contrib/python/ipython/py3/IPython/core/history.py +++ b/contrib/python/ipython/py3/IPython/core/history.py @@ -6,15 +6,9 @@ import atexit import datetime -import os +from pathlib import Path import re -try: - import sqlite3 -except ImportError: - try: - from pysqlite2 import dbapi2 as sqlite3 - except ImportError: - sqlite3 = None +import sqlite3 import threading from traitlets.config.configurable import LoggingConfigurable @@ -22,10 +16,18 @@ from decorator import decorator from IPython.utils.decorators import undoc from IPython.paths import locate_profile from traitlets import ( - Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError, - default, observe, + Any, + Bool, + Dict, + Instance, + Integer, + List, + Unicode, + Union, + TraitError, + default, + observe, ) -from warnings import warn #----------------------------------------------------------------------------- # Classes and functions @@ -34,42 +36,30 @@ from warnings import warn @undoc class DummyDB(object): """Dummy DB that will act as a black hole for history. - + Only used in the absence of sqlite""" def execute(*args, **kwargs): return [] - + def commit(self, *args, **kwargs): pass - + def __enter__(self, *args, **kwargs): pass - + def __exit__(self, *args, **kwargs): pass @decorator -def needs_sqlite(f, self, *a, **kw): +def only_when_enabled(f, self, *a, **kw): """Decorator: return an empty list in the absence of sqlite.""" - if sqlite3 is None or not self.enabled: + if not self.enabled: return [] else: return f(self, *a, **kw) -if sqlite3 is not None: - DatabaseError = sqlite3.DatabaseError - OperationalError = sqlite3.OperationalError -else: - @undoc - class DatabaseError(Exception): - "Dummy exception when sqlite could not be imported. Should never occur." - - @undoc - class OperationalError(Exception): - "Dummy exception when sqlite could not be imported. Should never occur." - # use 16kB as threshold for whether a corrupt history db should be saved # that should be at least 100 entries or so _SAVE_DB_SIZE = 16384 @@ -85,24 +75,25 @@ def catch_corrupt_db(f, self, *a, **kw): """ try: return f(self, *a, **kw) - except (DatabaseError, OperationalError) as e: + except (sqlite3.DatabaseError, sqlite3.OperationalError) as e: self._corrupt_db_counter += 1 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e) if self.hist_file != ':memory:': if self._corrupt_db_counter > self._corrupt_db_limit: self.hist_file = ':memory:' self.log.error("Failed to load history too many times, history will not be saved.") - elif os.path.isfile(self.hist_file): + elif self.hist_file.is_file(): # move the file out of the way - base, ext = os.path.splitext(self.hist_file) - size = os.stat(self.hist_file).st_size + base = str(self.hist_file.parent / self.hist_file.stem) + ext = self.hist_file.suffix + size = self.hist_file.stat().st_size if size >= _SAVE_DB_SIZE: # if there's significant content, avoid clobbering now = datetime.datetime.now().isoformat().replace(':', '.') newpath = base + '-corrupt-' + now + ext # don't clobber previous corrupt backups for i in range(100): - if not os.path.isfile(newpath): + if not Path(newpath).exists(): break else: newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext @@ -110,14 +101,15 @@ def catch_corrupt_db(f, self, *a, **kw): # not much content, possibly empty; don't worry about clobbering # maybe we should just delete it? newpath = base + '-corrupt' + ext - os.rename(self.hist_file, newpath) + self.hist_file.rename(newpath) self.log.error("History file was moved to %s and a new file created.", newpath) self.init_db() return [] else: # Failed with :memory:, something serious is wrong raise - + + class HistoryAccessorBase(LoggingConfigurable): """An abstract class for History Accessors """ @@ -137,7 +129,7 @@ class HistoryAccessorBase(LoggingConfigurable): class HistoryAccessor(HistoryAccessorBase): """Access the history database without adding to it. - + This is intended for use by standalone history tools. IPython shells use HistoryManager, below, which is a subclass of this.""" @@ -147,37 +139,39 @@ class HistoryAccessor(HistoryAccessorBase): _corrupt_db_limit = 2 # String holding the path to the history file - hist_file = Unicode( + hist_file = Union( + [Instance(Path), Unicode()], help="""Path to file to use for SQLite history database. - + By default, IPython will put the history database in the IPython profile directory. If you would rather share one history among profiles, you can set this value in each, so that they are consistent. - + Due to an issue with fcntl, SQLite is known to misbehave on some NFS mounts. If you see IPython hanging, try setting this to something on a local disk, e.g:: - + ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite you can also use the specific value `:memory:` (including the colon at both end but not the back ticks), to avoid creating an history file. - - """).tag(config=True) - + + """, + ).tag(config=True) + enabled = Bool(True, help="""enable the SQLite history - + set enabled=False to disable the SQLite history, in which case there will be no stored history, no SQLite connection, and no background saving thread. This may be necessary in some threaded environments where IPython is embedded. - """ + """, ).tag(config=True) - + connection_options = Dict( help="""Options for configuring the SQLite connection - + These options are passed as keyword args to sqlite3.connect when establishing database connections. """ @@ -189,26 +183,24 @@ class HistoryAccessor(HistoryAccessorBase): def _db_changed(self, change): """validate the db, since it can be an Instance of two different types""" new = change['new'] - connection_types = (DummyDB,) - if sqlite3 is not None: - connection_types = (DummyDB, sqlite3.Connection) + connection_types = (DummyDB, sqlite3.Connection) if not isinstance(new, connection_types): msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \ (self.__class__.__name__, new) raise TraitError(msg) - - def __init__(self, profile='default', hist_file=u'', **traits): + + def __init__(self, profile="default", hist_file="", **traits): """Create a new history accessor. - + Parameters ---------- profile : str - The name of the profile from which to open history. + The name of the profile from which to open history. hist_file : str - Path to an SQLite history database stored by IPython. If specified, - hist_file overrides profile. + Path to an SQLite history database stored by IPython. If specified, + hist_file overrides profile. config : :class:`~traitlets.config.loader.Config` - Config object. hist_file can also be set through this. + Config object. hist_file can also be set through this. """ # We need a pointer back to the shell for various tasks. super(HistoryAccessor, self).__init__(**traits) @@ -217,53 +209,57 @@ class HistoryAccessor(HistoryAccessorBase): # set by config if hist_file: self.hist_file = hist_file - - if self.hist_file == u'': + + try: + self.hist_file + except TraitError: # No one has set the hist_file, yet. self.hist_file = self._get_hist_file_name(profile) - if sqlite3 is None and self.enabled: - warn("IPython History requires SQLite, your history will not be saved") - self.enabled = False - self.init_db() - + def _get_hist_file_name(self, profile='default'): """Find the history file for the given profile name. - + This is overridden by the HistoryManager subclass, to use the shell's active profile. - + Parameters ---------- profile : str - The name of a profile which has a history file. + The name of a profile which has a history file. """ - return os.path.join(locate_profile(profile), 'history.sqlite') - + return Path(locate_profile(profile)) / "history.sqlite" + @catch_corrupt_db def init_db(self): """Connect to the database, and create tables if necessary.""" if not self.enabled: self.db = DummyDB() return - + # use detect_types so that timestamps return datetime objects kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) kwargs.update(self.connection_options) - self.db = sqlite3.connect(self.hist_file, **kwargs) - self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer - primary key autoincrement, start timestamp, - end timestamp, num_cmds integer, remark text)""") - self.db.execute("""CREATE TABLE IF NOT EXISTS history - (session integer, line integer, source text, source_raw text, - PRIMARY KEY (session, line))""") - # Output history is optional, but ensure the table's there so it can be - # enabled later. - self.db.execute("""CREATE TABLE IF NOT EXISTS output_history - (session integer, line integer, output text, - PRIMARY KEY (session, line))""") - self.db.commit() + self.db = sqlite3.connect(str(self.hist_file), **kwargs) + with self.db: + self.db.execute( + """CREATE TABLE IF NOT EXISTS sessions (session integer + primary key autoincrement, start timestamp, + end timestamp, num_cmds integer, remark text)""" + ) + self.db.execute( + """CREATE TABLE IF NOT EXISTS history + (session integer, line integer, source text, source_raw text, + PRIMARY KEY (session, line))""" + ) + # Output history is optional, but ensure the table's there so it can be + # enabled later. + self.db.execute( + """CREATE TABLE IF NOT EXISTS output_history + (session integer, line integer, output text, + PRIMARY KEY (session, line))""" + ) # success! reset corrupt db count self._corrupt_db_counter = 0 @@ -275,17 +271,19 @@ class HistoryAccessor(HistoryAccessorBase): ## ------------------------------- ## Methods for retrieving history: ## ------------------------------- - def _run_sql(self, sql, params, raw=True, output=False): + def _run_sql(self, sql, params, raw=True, output=False, latest=False): """Prepares and runs an SQL query for the history database. Parameters ---------- sql : str - Any filtering expressions to go after SELECT ... FROM ... + Any filtering expressions to go after SELECT ... FROM ... params : tuple - Parameters passed to the SQL query (to replace "?") + Parameters passed to the SQL query (to replace "?") raw, output : bool - See :meth:`get_range` + See :meth:`get_range` + latest : bool + Select rows with max (session, line) Returns ------- @@ -296,36 +294,38 @@ class HistoryAccessor(HistoryAccessorBase): if output: sqlfrom = "history LEFT JOIN output_history USING (session, line)" toget = "history.%s, output_history.output" % toget - cur = self.db.execute("SELECT session, line, %s FROM %s " %\ - (toget, sqlfrom) + sql, params) + if latest: + toget += ", MAX(session * 128 * 1024 + line)" + this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql + cur = self.db.execute(this_querry, params) + if latest: + cur = (row[:-1] for row in cur) if output: # Regroup into 3-tuples, and parse JSON return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) return cur - @needs_sqlite + @only_when_enabled @catch_corrupt_db def get_session_info(self, session): """Get info about a session. Parameters ---------- - session : int Session number to retrieve. Returns ------- - session_id : int - Session ID number + Session ID number start : datetime - Timestamp for the start of the session. + Timestamp for the start of the session. end : datetime - Timestamp for the end of the session, or None if IPython crashed. + Timestamp for the end of the session, or None if IPython crashed. num_cmds : int - Number of commands run, or None if IPython crashed. + Number of commands run, or None if IPython crashed. remark : unicode - A manually set description. + A manually set description. """ query = "SELECT * from sessions where session == ?" return self.db.execute(query, (session,)).fetchone() @@ -333,7 +333,7 @@ class HistoryAccessor(HistoryAccessorBase): @catch_corrupt_db def get_last_session_id(self): """Get the last session ID currently in the database. - + Within IPython, this should be the same as the value stored in :attr:`HistoryManager.session_number`. """ @@ -344,16 +344,21 @@ class HistoryAccessor(HistoryAccessorBase): def get_tail(self, n=10, raw=True, output=False, include_latest=False): """Get the last n lines from the history database. + Most recent entry last. + + Completion will be reordered so that that the last ones are when + possible from current session. + Parameters ---------- n : int - The number of lines to get + The number of lines to get raw, output : bool - See :meth:`get_range` + See :meth:`get_range` include_latest : bool - If False (default), n+1 lines are fetched, and the latest one - is discarded. This is intended to be used where the function - is called by a user command, which it should not return. + If False (default), n+1 lines are fetched, and the latest one + is discarded. This is intended to be used where the function + is called by a user command, which it should not return. Returns ------- @@ -362,11 +367,31 @@ class HistoryAccessor(HistoryAccessorBase): self.writeout_cache() if not include_latest: n += 1 - cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?", - (n,), raw=raw, output=output) + # cursor/line/entry + this_cur = list( + self._run_sql( + "WHERE session == ? ORDER BY line DESC LIMIT ? ", + (self.session_number, n), + raw=raw, + output=output, + ) + ) + other_cur = list( + self._run_sql( + "WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?", + (self.session_number, n), + raw=raw, + output=output, + ) + ) + + everything = this_cur + other_cur + + everything = everything[:n] + if not include_latest: - return reversed(list(cur)[1:]) - return reversed(list(cur)) + return list(everything)[:0:-1] + return list(everything)[::-1] @catch_corrupt_db def search(self, pattern="*", raw=True, search_raw=True, @@ -377,16 +402,16 @@ class HistoryAccessor(HistoryAccessorBase): Parameters ---------- pattern : str - The wildcarded pattern to match when searching + The wildcarded pattern to match when searching search_raw : bool - If True, search the raw input, otherwise, the parsed input + If True, search the raw input, otherwise, the parsed input raw, output : bool - See :meth:`get_range` + See :meth:`get_range` n : None or int - If an integer is given, it defines the limit of - returned entries. + If an integer is given, it defines the limit of + returned entries. unique : bool - When it is true, return only unique entries. + When it is true, return only unique entries. Returns ------- @@ -405,11 +430,11 @@ class HistoryAccessor(HistoryAccessorBase): params += (n,) elif unique: sqlform += " ORDER BY session, line" - cur = self._run_sql(sqlform, params, raw=raw, output=output) + cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique) if n is not None: return reversed(list(cur)) return cur - + @catch_corrupt_db def get_range(self, session, start=1, stop=None, raw=True,output=False): """Retrieve input by session. @@ -434,9 +459,9 @@ class HistoryAccessor(HistoryAccessorBase): Returns ------- entries - An iterator over the desired lines. Each line is a 3-tuple, either - (session, line, input) if output is False, or - (session, line, (input, output)) if output is True. + An iterator over the desired lines. Each line is a 3-tuple, either + (session, line, input) if output is False, or + (session, line, (input, output)) if output is True. """ if stop: lineclause = "line >= ? AND line < ?" @@ -455,10 +480,13 @@ class HistoryAccessor(HistoryAccessorBase): Parameters ---------- rangestr : str - A string specifying ranges, e.g. "5 ~2/1-4". See - :func:`magic_history` for full details. + A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used, + this will return everything from current session's history. + + See the documentation of :func:`%history` for the full details. + raw, output : bool - As :meth:`get_range` + As :meth:`get_range` Returns ------- @@ -486,7 +514,7 @@ class HistoryManager(HistoryAccessor): @default('dir_hist') def _dir_hist_default(self): try: - return [os.getcwd()] + return [Path.cwd()] except OSError: return [] @@ -498,7 +526,7 @@ class HistoryManager(HistoryAccessor): # The number of the current session in the history database session_number = Integer() - + db_log_output = Bool(False, help="Should the history database include output? (default: no)" ).tag(config=True) @@ -509,12 +537,12 @@ class HistoryManager(HistoryAccessor): # The input and output caches db_input_cache = List() db_output_cache = List() - + # History saving in separate thread save_thread = Instance('IPython.core.history.HistorySavingThread', allow_none=True) save_flag = Instance(threading.Event, allow_none=True) - + # Private interface # Variables used to store the three last inputs from the user. On each new # history update, we populate the user's namespace with these, shifted as @@ -538,37 +566,37 @@ class HistoryManager(HistoryAccessor): self.save_flag = threading.Event() self.db_input_cache_lock = threading.Lock() self.db_output_cache_lock = threading.Lock() - + try: self.new_session() - except OperationalError: + except sqlite3.OperationalError: self.log.error("Failed to create history session in %s. History will not be saved.", self.hist_file, exc_info=True) self.hist_file = ':memory:' - + if self.enabled and self.hist_file != ':memory:': self.save_thread = HistorySavingThread(self) self.save_thread.start() def _get_hist_file_name(self, profile=None): """Get default history file name based on the Shell's profile. - + The profile parameter is ignored, but must exist for compatibility with the parent class.""" profile_dir = self.shell.profile_dir.location - return os.path.join(profile_dir, 'history.sqlite') - - @needs_sqlite + return Path(profile_dir) / "history.sqlite" + + @only_when_enabled def new_session(self, conn=None): """Get a new session number.""" if conn is None: conn = self.db - + with conn: cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL, NULL, "") """, (datetime.datetime.now(),)) self.session_number = cur.lastrowid - + def end_session(self): """Close the database session, filling in the end time and line count.""" self.writeout_cache() @@ -577,20 +605,20 @@ class HistoryManager(HistoryAccessor): session==?""", (datetime.datetime.now(), len(self.input_hist_parsed)-1, self.session_number)) self.session_number = 0 - + def name_session(self, name): """Give the current session a name in the history database.""" with self.db: self.db.execute("UPDATE sessions SET remark=? WHERE session==?", (name, self.session_number)) - + def reset(self, new_session=True): """Clear the session history, releasing all object references, and optionally open a new session.""" self.output_hist.clear() # The directory history can't be completely empty - self.dir_hist[:] = [os.getcwd()] - + self.dir_hist[:] = [Path.cwd()] + if new_session: if self.session_number: self.end_session() @@ -606,24 +634,22 @@ class HistoryManager(HistoryAccessor): Parameters ---------- - session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is the previous session. Returns ------- - session_id : int - Session ID number + Session ID number start : datetime - Timestamp for the start of the session. + Timestamp for the start of the session. end : datetime - Timestamp for the end of the session, or None if IPython crashed. + Timestamp for the end of the session, or None if IPython crashed. num_cmds : int - Number of commands run, or None if IPython crashed. + Number of commands run, or None if IPython crashed. remark : unicode - A manually set description. + A manually set description. """ if session <= 0: session += self.session_number @@ -634,7 +660,7 @@ class HistoryManager(HistoryAccessor): """Get input and output history from the current session. Called by get_range, and takes similar parameters.""" input_hist = self.input_hist_raw if raw else self.input_hist_parsed - + n = len(input_hist) if start < 0: start += n @@ -642,17 +668,17 @@ class HistoryManager(HistoryAccessor): stop = n elif stop < 0: stop += n - + for i in range(start, stop): if output: line = (input_hist[i], self.output_hist_reprs.get(i)) else: line = input_hist[i] yield (0, i, line) - + def get_range(self, session=0, start=1, stop=None, raw=True,output=False): """Retrieve input by session. - + Parameters ---------- session : int @@ -670,13 +696,13 @@ class HistoryManager(HistoryAccessor): objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. - + Returns ------- entries - An iterator over the desired lines. Each line is a 3-tuple, either - (session, line, input) if output is False, or - (session, line, (input, output)) if output is True. + An iterator over the desired lines. Each line is a 3-tuple, either + (session, line, input) if output is False, or + (session, line, (input, output)) if output is True. """ if session <= 0: session += self.session_number @@ -695,14 +721,12 @@ class HistoryManager(HistoryAccessor): Parameters ---------- line_num : int - The prompt number of this input. - + The prompt number of this input. source : str - Python input. - + Python input. source_raw : str, optional - If given, this is the raw input without any IPython transformations - applied to it. If not given, ``source`` is used. + If given, this is the raw input without any IPython transformations + applied to it. If not given, ``source`` is used. """ if source_raw is None: source_raw = source @@ -734,7 +758,7 @@ class HistoryManager(HistoryAccessor): '_ii': self._ii, '_iii': self._iii, new_i : self._i00 } - + if self.shell is not None: self.shell.push(to_main, interactive=False) @@ -746,7 +770,7 @@ class HistoryManager(HistoryAccessor): Parameters ---------- line_num : int - The line number from which to save outputs + The line number from which to save outputs """ if (not self.db_log_output) or (line_num not in self.output_hist_reprs): return @@ -769,7 +793,7 @@ class HistoryManager(HistoryAccessor): conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", (self.session_number,)+line) - @needs_sqlite + @only_when_enabled def writeout_cache(self, conn=None): """Write any entries in the cache to the database.""" if conn is None: @@ -818,12 +842,13 @@ class HistorySavingThread(threading.Thread): self.enabled = history_manager.enabled atexit.register(self.stop) - @needs_sqlite + @only_when_enabled def run(self): # We need a separate db connection per thread: try: - self.db = sqlite3.connect(self.history_manager.hist_file, - **self.history_manager.connection_options + self.db = sqlite3.connect( + str(self.history_manager.hist_file), + **self.history_manager.connection_options, ) while True: self.history_manager.save_flag.wait() @@ -860,11 +885,18 @@ $""", re.VERBOSE) def extract_hist_ranges(ranges_str): """Turn a string of history ranges into 3-tuples of (session, start, stop). + Empty string results in a `[(0, 1, None)]`, i.e. "everything from current + session". + Examples -------- >>> list(extract_hist_ranges("~8/5-~7/4 2")) [(-8, 5, None), (-7, 1, 5), (0, 2, 3)] """ + if ranges_str == "": + yield (0, 1, None) # Everything from current session + return + for range_str in ranges_str.split(): rmatch = range_re.match(range_str) if not rmatch: diff --git a/contrib/python/ipython/py3/IPython/core/historyapp.py b/contrib/python/ipython/py3/IPython/core/historyapp.py index a6437eff26..01a55343f8 100644 --- a/contrib/python/ipython/py3/IPython/core/historyapp.py +++ b/contrib/python/ipython/py3/IPython/core/historyapp.py @@ -5,8 +5,8 @@ An application for managing IPython history. To be invoked as the `ipython history` subcommand. """ -import os import sqlite3 +from pathlib import Path from traitlets.config.application import Application from .application import BaseIPythonApplication @@ -52,8 +52,8 @@ class HistoryTrim(BaseIPythonApplication): )) def start(self): - profile_dir = self.profile_dir.location - hist_file = os.path.join(profile_dir, 'history.sqlite') + profile_dir = Path(self.profile_dir.location) + hist_file = profile_dir / "history.sqlite" con = sqlite3.connect(hist_file) # Grab the recent history from the current database. @@ -77,12 +77,12 @@ class HistoryTrim(BaseIPythonApplication): con.close() # Create the new history database. - new_hist_file = os.path.join(profile_dir, 'history.sqlite.new') + new_hist_file = profile_dir / "history.sqlite.new" i = 0 - while os.path.exists(new_hist_file): + while new_hist_file.exists(): # Make sure we don't interfere with an existing file. i += 1 - new_hist_file = os.path.join(profile_dir, 'history.sqlite.new'+str(i)) + new_hist_file = profile_dir / ("history.sqlite.new" + str(i)) new_db = sqlite3.connect(new_hist_file) new_db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer primary key autoincrement, start timestamp, @@ -106,16 +106,16 @@ class HistoryTrim(BaseIPythonApplication): if self.backup: i = 1 - backup_hist_file = os.path.join(profile_dir, 'history.sqlite.old.%d' % i) - while os.path.exists(backup_hist_file): + backup_hist_file = profile_dir / ("history.sqlite.old.%d" % i) + while backup_hist_file.exists(): i += 1 - backup_hist_file = os.path.join(profile_dir, 'history.sqlite.old.%d' % i) - os.rename(hist_file, backup_hist_file) + backup_hist_file = profile_dir / ("history.sqlite.old.%d" % i) + hist_file.rename(backup_hist_file) print("Backed up longer history file to", backup_hist_file) else: - os.remove(hist_file) + hist_file.unlink() - os.rename(new_hist_file, hist_file) + new_hist_file.rename(hist_file) class HistoryClear(HistoryTrim): description = clear_hist_help diff --git a/contrib/python/ipython/py3/IPython/core/hooks.py b/contrib/python/ipython/py3/IPython/core/hooks.py index fa732f7ba8..09b08d942e 100644 --- a/contrib/python/ipython/py3/IPython/core/hooks.py +++ b/contrib/python/ipython/py3/IPython/core/hooks.py @@ -44,10 +44,13 @@ from .error import TryNext # List here all the default hooks. For now it's just the editor functions # but over time we'll move here all the public API for user-accessible things. -__all__ = ['editor', 'synchronize_with_editor', - 'shutdown_hook', 'late_startup_hook', - 'show_in_pager','pre_prompt_hook', - 'pre_run_code_hook', 'clipboard_get'] +__all__ = [ + "editor", + "synchronize_with_editor", + "show_in_pager", + "pre_prompt_hook", + "clipboard_get", +] deprecated = {'pre_run_code_hook': "a callback for the 'pre_execute' or 'pre_run_cell' event", 'late_startup_hook': "a callback for the 'shell_initialized' event", @@ -132,23 +135,6 @@ class CommandChainDispatcher: return iter(self.chain) -def shutdown_hook(self): - """ default shutdown hook - - Typically, shutdown hooks should raise TryNext so all shutdown ops are done - """ - - #print "default shutdown hook ok" # dbg - return - - -def late_startup_hook(self): - """ Executed after ipython has been constructed and configured - - """ - #print "default startup hook ok" # dbg - - def show_in_pager(self, data, start, screen_lines): """ Run a string through pager """ # raising TryNext here will use the default paging functionality @@ -165,11 +151,6 @@ def pre_prompt_hook(self): return None -def pre_run_code_hook(self): - """ Executed before running the (prefiltered) code in IPython """ - return None - - def clipboard_get(self): """ Get text from the clipboard. """ diff --git a/contrib/python/ipython/py3/IPython/core/inputsplitter.py b/contrib/python/ipython/py3/IPython/core/inputsplitter.py index e7bc6e7f5a..01b9c8a5e6 100644 --- a/contrib/python/ipython/py3/IPython/core/inputsplitter.py +++ b/contrib/python/ipython/py3/IPython/core/inputsplitter.py @@ -210,7 +210,7 @@ def last_blank(src): Parameters ---------- src : string - A single or multiline string. + A single or multiline string. """ if not src: return False ll = src.splitlines()[-1] @@ -228,7 +228,7 @@ def last_two_blanks(src): Parameters ---------- src : string - A single or multiline string. + A single or multiline string. """ if not src: return False # The logic here is tricky: I couldn't get a regexp to work and pass all @@ -251,7 +251,7 @@ def remove_comments(src): Parameters ---------- src : string - A single or multiline input string. + A single or multiline input string. Returns ------- @@ -351,22 +351,22 @@ class InputSplitter(object): def check_complete(self, source): """Return whether a block of code is ready to execute, or should be continued - + This is a non-stateful API, and will reset the state of this InputSplitter. - + Parameters ---------- source : string - Python input code, which can be multiline. - + Python input code, which can be multiline. + Returns ------- status : str - One of 'complete', 'incomplete', or 'invalid' if source is not a - prefix of valid code. + One of 'complete', 'incomplete', or 'invalid' if source is not a + prefix of valid code. indent_spaces : int or None - The number of spaces by which to indent the next line of code. If - status is not 'incomplete', this is None. + The number of spaces by which to indent the next line of code. If + status is not 'incomplete', this is None. """ self.reset() try: @@ -397,15 +397,15 @@ class InputSplitter(object): Parameters ---------- lines : string - One or more lines of Python input. + One or more lines of Python input. Returns ------- is_complete : boolean - True if the current input source (the result of the current input - plus prior inputs) forms a complete Python execution block. Note that - this value is also stored as a private attribute (``_is_complete``), so it - can be queried at any time. + True if the current input source (the result of the current input + plus prior inputs) forms a complete Python execution block. Note that + this value is also stored as a private attribute (``_is_complete``), so it + can be queried at any time. """ assert isinstance(lines, str) self._store(lines) @@ -448,7 +448,7 @@ class InputSplitter(object): guess whether a block is complete or not based solely on prior and current input lines. The InputSplitter considers it has a complete interactive block and will not accept more input when either: - + * A SyntaxError is raised * The code is complete and consists of a single line or a single @@ -618,9 +618,9 @@ class IPythonInputSplitter(InputSplitter): def flush_transformers(self): def _flush(transform, outs): """yield transformed lines - + always strings, never None - + transform: the current transform outs: an iterable of previously transformed inputs. Each may be multiline, which will be passed @@ -690,15 +690,15 @@ class IPythonInputSplitter(InputSplitter): Parameters ---------- lines : string - One or more lines of Python input. + One or more lines of Python input. Returns ------- is_complete : boolean - True if the current input source (the result of the current input - plus prior inputs) forms a complete Python execution block. Note that - this value is also stored as a private attribute (_is_complete), so it - can be queried at any time. + True if the current input source (the result of the current input + plus prior inputs) forms a complete Python execution block. Note that + this value is also stored as a private attribute (_is_complete), so it + can be queried at any time. """ assert isinstance(lines, str) # We must ensure all input is pure unicode @@ -728,10 +728,10 @@ class IPythonInputSplitter(InputSplitter): def _transform_line(self, line): """Push a line of input code through the various transformers. - + Returns any output from the transformers, or None if a transformer is accumulating lines. - + Sets self.transformer_accumulating as a side effect. """ def _accumulating(dbg): diff --git a/contrib/python/ipython/py3/IPython/core/inputtransformer.py b/contrib/python/ipython/py3/IPython/core/inputtransformer.py index 14a351d40a..77f69f388f 100644 --- a/contrib/python/ipython/py3/IPython/core/inputtransformer.py +++ b/contrib/python/ipython/py3/IPython/core/inputtransformer.py @@ -46,7 +46,7 @@ class InputTransformer(metaclass=abc.ABCMeta): def push(self, line): """Send a line of input to the transformer, returning the transformed input or None if the transformer is waiting for more input. - + Must be overridden by subclasses. Implementations may raise ``SyntaxError`` if the input is invalid. No @@ -58,7 +58,7 @@ class InputTransformer(metaclass=abc.ABCMeta): def reset(self): """Return, transformed any lines that the transformer has accumulated, and reset its internal state. - + Must be overridden by subclasses. """ pass @@ -313,7 +313,7 @@ def has_comment(src): Parameters ---------- src : string - A single line input string. + A single line input string. Returns ------- @@ -325,11 +325,11 @@ def has_comment(src): def ends_in_comment_or_string(src): """Indicates whether or not an input line ends in a comment or within a multiline string. - + Parameters ---------- src : string - A single line input string. + A single line input string. Returns ------- @@ -356,7 +356,7 @@ def help_end(line): @CoroutineInputTransformer.wrap def cellmagic(end_on_blank_line=False): """Captures & transforms cell magics. - + After a cell magic is started, this stores up any lines it gets until it is reset (sent None). """ @@ -395,7 +395,7 @@ def cellmagic(end_on_blank_line=False): def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None): """Remove matching input prompts from a block of input. - + Parameters ---------- prompt_re : regular expression @@ -405,9 +405,11 @@ def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None): If no initial expression is given, prompt_re will be used everywhere. Used mainly for plain Python prompts, where the continuation prompt ``...`` is a valid Python expression in Python 3, so shouldn't be stripped. - - If initial_re and prompt_re differ, - only initial_re will be tested against the first line. + + Notes + ----- + If `initial_re` and `prompt_re differ`, + only `initial_re` will be tested against the first line. If any prompt is found on the first two lines, prompts will be stripped from the rest of the block. """ @@ -473,7 +475,7 @@ def ipy_prompt(): @CoroutineInputTransformer.wrap def leading_indent(): """Remove leading indentation. - + If the first line starts with a spaces or tabs, the same whitespace will be removed from each following line until it is reset. """ diff --git a/contrib/python/ipython/py3/IPython/core/inputtransformer2.py b/contrib/python/ipython/py3/IPython/core/inputtransformer2.py index c0bb39979d..a8f676f495 100644 --- a/contrib/python/ipython/py3/IPython/core/inputtransformer2.py +++ b/contrib/python/ipython/py3/IPython/core/inputtransformer2.py @@ -15,7 +15,7 @@ import sys from codeop import CommandCompiler, Compile import re import tokenize -from typing import List, Tuple, Union +from typing import List, Tuple, Optional, Any import warnings _indent_re = re.compile(r'^[ \t]+') @@ -91,7 +91,30 @@ classic_prompt = PromptStripper( initial_re=re.compile(r'^>>>( |$)') ) -ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')) +ipython_prompt = PromptStripper( + re.compile( + r""" + ^( # Match from the beginning of a line, either: + + # 1. First-line prompt: + ((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there + In\ # The 'In' of the prompt, with a space + \[\d+\]: # Command index, as displayed in the prompt + \ # With a mandatory trailing space + + | # ... or ... + + # 2. The three dots of the multiline prompt + \s* # All leading whitespace characters + \.{3,}: # The three (or more) dots + \ ? # With an optional trailing space + + ) + """, + re.VERBOSE, + ) +) + def cell_magic(lines): if not lines or not lines[0].startswith('%%'): @@ -105,7 +128,7 @@ def cell_magic(lines): % (magic_name, first_line, body)] -def _find_assign_op(token_line) -> Union[int, None]: +def _find_assign_op(token_line) -> Optional[int]: """Get the index of the first assignment in the line ('=' not inside brackets) Note: We don't try to support multiple special assignment (a = b = %foo) @@ -120,6 +143,7 @@ def _find_assign_op(token_line) -> Union[int, None]: elif s in {')', ']', '}'}: if paren_level > 0: paren_level -= 1 + return None def find_end_of_continued_line(lines, start_line: int): """Find the last line of a line explicitly extended using backslashes. @@ -472,10 +496,15 @@ def make_tokens_by_line(lines:List[str]): # lines or comments. This is intentional - see https://bugs.python.org/issue17061 # We want to group the former case together but split the latter, so we # track parentheses level, similar to the internals of tokenize. - NEWLINE, NL = tokenize.NEWLINE, tokenize.NL - tokens_by_line = [[]] - if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')): - warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified") + + # reexported from token on 3.7+ + NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore + tokens_by_line: List[List[Any]] = [[]] + if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")): + warnings.warn( + "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified", + stacklevel=2, + ) parenlev = 0 try: for token in tokenize.generate_tokens(iter(lines).__next__): @@ -499,6 +528,20 @@ def make_tokens_by_line(lines:List[str]): return tokens_by_line + +def has_sunken_brackets(tokens: List[tokenize.TokenInfo]): + """Check if the depth of brackets in the list of tokens drops below 0""" + parenlev = 0 + for token in tokens: + if token.string in {"(", "[", "{"}: + parenlev += 1 + elif token.string in {")", "]", "}"}: + parenlev -= 1 + if parenlev < 0: + return True + return False + + def show_linewise_tokens(s: str): """For investigation and debugging""" if not s.endswith('\n'): @@ -592,17 +635,17 @@ class TransformerManager: Parameters ---------- - source : string - Python input code, which can be multiline. + cell : string + Python input code, which can be multiline. Returns ------- status : str - One of 'complete', 'incomplete', or 'invalid' if source is not a - prefix of valid code. + One of 'complete', 'incomplete', or 'invalid' if source is not a + prefix of valid code. indent_spaces : int or None - The number of spaces by which to indent the next line of code. If - status is not 'incomplete', this is None. + The number of spaces by which to indent the next line of code. If + status is not 'incomplete', this is None. """ # Remember if the lines ends in a new line. ends_with_newline = False @@ -653,6 +696,15 @@ class TransformerManager: tokens_by_line = make_tokens_by_line(lines) + # Bail if we got one line and there are more closing parentheses than + # the opening ones + if ( + len(lines) == 1 + and tokens_by_line + and has_sunken_brackets(tokens_by_line[0]) + ): + return "invalid", None + if not tokens_by_line: return 'incomplete', find_last_indent(lines) @@ -660,7 +712,7 @@ class TransformerManager: # We're in a multiline string or expression return 'incomplete', find_last_indent(lines) - newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} + newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore # Pop the last line which only contains DEDENTs and ENDMARKER last_token_line = None @@ -726,19 +778,11 @@ class MaybeAsyncCompile(Compile): self.flags |= extra_flags - if sys.version_info < (3,8): - def __call__(self, *args, **kwds): - return compile(*args, **kwds) - - class MaybeAsyncCommandCompiler(CommandCompiler): def __init__(self, extra_flags=0): self.compiler = MaybeAsyncCompile(extra_flags=extra_flags) -if (sys.version_info.major, sys.version_info.minor) >= (3, 8): - _extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT -else: - _extra_flags = ast.PyCF_ONLY_AST +_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags) diff --git a/contrib/python/ipython/py3/IPython/core/interactiveshell.py b/contrib/python/ipython/py3/IPython/core/interactiveshell.py index 8fd546b847..ea9f6310ba 100644 --- a/contrib/python/ipython/py3/IPython/core/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/core/interactiveshell.py @@ -21,34 +21,53 @@ import inspect import os import re import runpy +import subprocess import sys import tempfile import traceback import types -import subprocess import warnings +from ast import stmt from io import open as io_open - +from logging import error from pathlib import Path -from pickleshare import PickleShareDB +from typing import Callable +from typing import List as ListType +from typing import Optional, Tuple +from warnings import warn +from pickleshare import PickleShareDB +from tempfile import TemporaryDirectory +from traitlets import ( + Any, + Bool, + CaselessStrEnum, + Dict, + Enum, + Instance, + Integer, + List, + Type, + Unicode, + default, + observe, + validate, +) from traitlets.config.configurable import SingletonConfigurable from traitlets.utils.importstring import import_item -from IPython.core import oinspect -from IPython.core import magic -from IPython.core import page -from IPython.core import prefilter -from IPython.core import ultratb + +import IPython.core.hooks +from IPython.core import magic, oinspect, page, prefilter, ultratb from IPython.core.alias import Alias, AliasManager from IPython.core.autocall import ExitAutocall from IPython.core.builtin_trap import BuiltinTrap -from IPython.core.events import EventManager, available_events from IPython.core.compilerop import CachingCompiler, check_linecache_ipython from IPython.core.debugger import InterruptiblePdb from IPython.core.display_trap import DisplayTrap from IPython.core.displayhook import DisplayHook from IPython.core.displaypub import DisplayPublisher from IPython.core.error import InputRejected, UsageError +from IPython.core.events import EventManager, available_events from IPython.core.extensions import ExtensionManager from IPython.core.formatters import DisplayFormatter from IPython.core.history import HistoryManager @@ -60,36 +79,19 @@ from IPython.core.prefilter import PrefilterManager from IPython.core.profiledir import ProfileDir from IPython.core.usage import default_banner from IPython.display import display +from IPython.paths import get_ipython_dir from IPython.testing.skipdoctest import skip_doctest -from IPython.utils import PyColorize -from IPython.utils import io -from IPython.utils import py3compat -from IPython.utils import openpy +from IPython.utils import PyColorize, io, openpy, py3compat from IPython.utils.decorators import undoc from IPython.utils.io import ask_yes_no from IPython.utils.ipstruct import Struct -from IPython.paths import get_ipython_dir -from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists -from IPython.utils.process import system, getoutput +from IPython.utils.path import ensure_dir_exists, get_home_dir, get_py_filename +from IPython.utils.process import getoutput, system from IPython.utils.strdispatch import StrDispatch from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.text import format_screen, LSString, SList, DollarFormatter -from IPython.utils.tempdir import TemporaryDirectory -from traitlets import ( - Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type, - observe, default, validate, Any -) -from warnings import warn -from logging import error -import IPython.core.hooks +from IPython.utils.text import DollarFormatter, LSString, SList, format_screen -from typing import List as ListType, Tuple, Optional -from ast import AST - -# NoOpContext is deprecated, but ipykernel imports it from here. -# See https://github.com/ipython/ipykernel/issues/157 -# (2016, let's try to remove than in IPython 8.0) -from IPython.utils.contexts import NoOpContext +sphinxify: Optional[Callable] try: import docrepr.sphinxify as sphx @@ -115,118 +117,25 @@ class ProvisionalWarning(DeprecationWarning): """ pass -if sys.version_info > (3,8): - from ast import Module -else : - # mock the new API, ignore second argument - # see https://github.com/ipython/ipython/issues/11590 - from ast import Module as OriginalModule - Module = lambda nodelist, type_ignores: OriginalModule(nodelist) - -if sys.version_info > (3,6): - _assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign) - _single_targets_nodes = (ast.AugAssign, ast.AnnAssign) -else: - _assign_nodes = (ast.AugAssign, ast.Assign ) - _single_targets_nodes = (ast.AugAssign, ) +from ast import Module + +_assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign) +_single_targets_nodes = (ast.AugAssign, ast.AnnAssign) #----------------------------------------------------------------------------- # Await Helpers #----------------------------------------------------------------------------- -def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType: - """Return a function that do not create a new local scope. - - Given a function, create a clone of this function where the co_newlocal flag - has been removed, making this function code actually run in the sourounding - scope. - - We need this in order to run asynchronous code in user level namespace. - """ - from types import CodeType, FunctionType - CO_NEWLOCALS = 0x0002 - code = function.__code__ - new_co_flags = code.co_flags & ~CO_NEWLOCALS - if sys.version_info > (3, 8, 0, 'alpha', 3): - new_code = code.replace(co_flags=new_co_flags) - else: - new_code = CodeType( - code.co_argcount, - code.co_kwonlyargcount, - code.co_nlocals, - code.co_stacksize, - new_co_flags, - code.co_code, - code.co_consts, - code.co_names, - code.co_varnames, - code.co_filename, - code.co_name, - code.co_firstlineno, - code.co_lnotab, - code.co_freevars, - code.co_cellvars - ) - return FunctionType(new_code, globals(), function.__name__, function.__defaults__) - - # we still need to run things using the asyncio eventloop, but there is no # async integration -from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner) -from .async_helpers import _curio_runner, _trio_runner, _should_be_async - - -def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module: - """ - Parse a cell with top-level await and modify the AST to be able to run it later. - - Parameter - --------- - - cell: str - The code cell to asyncronify - wrapper_name: str - The name of the function to be used to wrap the passed `cell`. It is - advised to **not** use a python identifier in order to not pollute the - global namespace in which the function will be ran. - - Return - ------ - - A module object AST containing **one** function named `wrapper_name`. - - The given code is wrapped in a async-def function, parsed into an AST, and - the resulting function definition AST is modified to return the last - expression. - - The last expression or await node is moved into a return statement at the - end of the function, and removed from its original location. If the last - node is not Expr or Await nothing is done. - - The function `__code__` will need to be later modified (by - ``removed_co_newlocals``) in a subsequent step to not create new `locals()` - meaning that the local and global scope are the same, ie as if the body of - the function was at module level. - - Lastly a call to `locals()` is made just before the last expression of the - function, or just after the last assignment or statement to make sure the - global dict is updated as python function work with a local fast cache which - is updated only on `local()` calls. - """ +from .async_helpers import ( + _asyncio_runner, + _curio_runner, + _pseudo_sync_runner, + _should_be_async, + _trio_runner, +) - from ast import Expr, Await, Return - if sys.version_info >= (3,8): - return ast.parse(cell) - tree = ast.parse(_asyncify(cell)) - - function_def = tree.body[0] - function_def.name = wrapper_name - try_block = function_def.body[0] - lastexpr = try_block.body[-1] - if isinstance(lastexpr, (Expr, Await)): - try_block.body[-1] = Return(lastexpr.value) - ast.fix_missing_locations(tree) - return tree #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- @@ -262,13 +171,6 @@ def no_op(*a, **kw): class SpaceInInput(Exception): pass -def get_default_colors(): - "DEPRECATED" - warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.', - DeprecationWarning, stacklevel=2) - return 'Neutral' - - class SeparateUnicode(Unicode): r"""A Unicode subclass to validate separate_in, separate_out, etc. @@ -329,7 +231,7 @@ class ExecutionResult(object): """ execution_count = None error_before_exec = None - error_in_exec = None + error_in_exec: Optional[BaseException] = None info = None result = None @@ -551,29 +453,6 @@ class InteractiveShell(SingletonConfigurable): will be displayed as regular output instead.""" ).tag(config=True) - # deprecated prompt traits: - - prompt_in1 = Unicode('In [\\#]: ', - help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly." - ).tag(config=True) - prompt_in2 = Unicode(' .\\D.: ', - help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly." - ).tag(config=True) - prompt_out = Unicode('Out[\\#]: ', - help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly." - ).tag(config=True) - prompts_pad_left = Bool(True, - help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly." - ).tag(config=True) - - @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left') - def _prompt_trait_changed(self, change): - name = change['name'] - warn("InteractiveShell.{name} is deprecated since IPython 4.0" - " and ignored since 5.0, set TerminalInteractiveShell.prompts" - " object directly.".format(name=name)) - - # protect against weird cases where self.config may not exist: show_rewritten_input = Bool(True, help="Show rewritten input, e.g. for autocall." @@ -707,8 +586,6 @@ class InteractiveShell(SingletonConfigurable): self.init_pdb() self.init_extension_manager() self.init_payload() - self.init_deprecation_warnings() - self.hooks.late_startup_hook() self.events.trigger('shell_initialized', self) atexit.register(self.atexit_operations) @@ -775,6 +652,7 @@ class InteractiveShell(SingletonConfigurable): self.meta = Struct() # Temporary files used for various purposes. Deleted at exit. + # The files here are stored with Path from Pathlib self.tempfiles = [] self.tempdirs = [] @@ -832,16 +710,6 @@ class InteractiveShell(SingletonConfigurable): elif self.logstart: self.magic('logstart') - def init_deprecation_warnings(self): - """ - register default filter for deprecation warning. - - This will allow deprecation warning of function used interactively to show - warning to users, and still hide deprecation warning from libraries import. - """ - if sys.version_info < (3,7): - warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__")) - def init_builtins(self): # A single, static flag that we set to True. Its presence indicates @@ -862,16 +730,9 @@ class InteractiveShell(SingletonConfigurable): self.object_info_string_level) def init_io(self): - # This will just use sys.stdout and sys.stderr. If you want to - # override sys.stdout and sys.stderr themselves, you need to do that - # *before* instantiating this class, because io holds onto - # references to the underlying streams. - # io.std* are deprecated, but don't show our own deprecation warnings - # during initialization of the deprecated API. - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - io.stdout = io.IOStream(sys.stdout) - io.stderr = io.IOStream(sys.stderr) + # implemented in subclasses, TerminalInteractiveShell does call + # colorama.init(). + pass def init_prompts(self): # Set system prompts, so that scripts can decide if they are running @@ -907,13 +768,42 @@ class InteractiveShell(SingletonConfigurable): # the appropriate time. self.display_trap = DisplayTrap(hook=self.displayhook) + @staticmethod + def get_path_links(p: Path): + """Gets path links including all symlinks + + Examples + -------- + In [1]: from IPython.core.interactiveshell import InteractiveShell + + In [2]: import sys, pathlib + + In [3]: paths = InteractiveShell.get_path_links(pathlib.Path(sys.executable)) + + In [4]: len(paths) == len(set(paths)) + Out[4]: True + + In [5]: bool(paths) + Out[5]: True + """ + paths = [p] + while p.is_symlink(): + new_path = Path(os.readlink(p)) + if not new_path.is_absolute(): + new_path = p.parent / new_path + p = new_path + paths.append(p) + return paths + def init_virtualenv(self): """Add the current virtualenv to sys.path so the user can import modules from it. This isn't perfect: it doesn't use the Python interpreter with which the virtualenv was built, and it ignores the --no-site-packages option. A warning will appear suggesting the user installs IPython in the virtualenv, but for many cases, it probably works well enough. + Adapted from code snippets online. + http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv """ if 'VIRTUAL_ENV' not in os.environ: @@ -930,10 +820,7 @@ class InteractiveShell(SingletonConfigurable): # stdlib venv may symlink sys.executable, so we can't use realpath. # but others can symlink *to* the venv Python, so we can't just use sys.executable. # So we just check every item in the symlink tree (generally <= 3) - paths = [p] - while p.is_symlink(): - p = Path(os.readlink(p)) - paths.append(p.resolve()) + paths = self.get_path_links(p) # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible if p_venv.parts[1] == "cygdrive": @@ -1029,13 +916,12 @@ class InteractiveShell(SingletonConfigurable): for hook_name in hooks.__all__: # default hooks have priority 100, i.e. low; user hooks should have # 0-100 priority - self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False) + self.set_hook(hook_name, getattr(hooks, hook_name), 100) if self.display_page: self.set_hook('show_in_pager', page.as_hook(page.display_page), 90) - def set_hook(self,name,hook, priority=50, str_key=None, re_key=None, - _warn_deprecated=True): + def set_hook(self, name, hook, priority=50, str_key=None, re_key=None): """set_hook(name,hook) -> sets an internal IPython hook. IPython exposes some of its internal API as user-modifiable hooks. By @@ -1065,9 +951,13 @@ class InteractiveShell(SingletonConfigurable): print("Warning! Hook '%s' is not one of %s" % \ (name, IPython.core.hooks.__all__ )) - if _warn_deprecated and (name in IPython.core.hooks.deprecated): + if name in IPython.core.hooks.deprecated: alternative = IPython.core.hooks.deprecated[name] - warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2) + raise ValueError( + "Hook {} has been deprecated since IPython 5.0. Use {} instead.".format( + name, alternative + ) + ) if not dp: dp = IPython.core.hooks.CommandChainDispatcher() @@ -1091,12 +981,13 @@ class InteractiveShell(SingletonConfigurable): def register_post_execute(self, func): """DEPRECATED: Use ip.events.register('post_run_cell', func) - + Register a function for calling after code execution. """ - warn("ip.register_post_execute is deprecated, use " - "ip.events.register('post_run_cell', func) instead.", stacklevel=2) - self.events.register('post_run_cell', func) + raise ValueError( + "ip.register_post_execute is deprecated since IPython 1.0, use " + "ip.events.register('post_run_cell', func) instead." + ) def _clear_warning_registry(self): # clear the warning registry, so that different code blocks with @@ -1111,14 +1002,14 @@ class InteractiveShell(SingletonConfigurable): def new_main_mod(self, filename, modname): """Return a new 'main' module object for user code execution. - + ``filename`` should be the path of the script which will be run in the module. Requests with the same filename will get the same module, with its namespace cleared. - + ``modname`` should be the module name - normally either '__main__' or the basename of the file without the extension. - + When scripts are executed via %run, we must keep a reference to their __main__ module around so that Python doesn't clear it, rendering references to module globals useless. @@ -1154,7 +1045,6 @@ class InteractiveShell(SingletonConfigurable): Examples -------- - In [15]: import IPython In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython') @@ -1304,10 +1194,10 @@ class InteractiveShell(SingletonConfigurable): def prepare_user_module(self, user_module=None, user_ns=None): """Prepare the module and namespace in which user code will be run. - + When IPython is started normally, both parameters are None: a new module is created automatically, and its __dict__ used as the namespace. - + If only user_module is provided, its __dict__ is used as the namespace. If only user_ns is provided, a dummy module is created, and user_ns becomes the global namespace. If both are provided (as they may be @@ -1428,7 +1318,7 @@ class InteractiveShell(SingletonConfigurable): def all_ns_refs(self): """Get a list of references to all the namespace dictionaries in which IPython might store a user-created object. - + Note that this does not include the displayhook, which also caches objects from the output.""" return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \ @@ -1524,8 +1414,8 @@ class InteractiveShell(SingletonConfigurable): else: # Delete by object try: obj = self.user_ns[varname] - except KeyError: - raise NameError("name '%s' is not defined" % varname) + except KeyError as e: + raise NameError("name '%s' is not defined" % varname) from e # Also check in output history ns_refs.append(self.history_manager.output_hist) for ns in ns_refs: @@ -1555,8 +1445,8 @@ class InteractiveShell(SingletonConfigurable): if regex is not None: try: m = re.compile(regex) - except TypeError: - raise TypeError('regex must be a string or compiled pattern') + except TypeError as e: + raise TypeError('regex must be a string or compiled pattern') from e # Search for keys in each namespace that match the given regex # If a match is found, delete the key/value pair. for ns in self.all_ns_refs: @@ -1615,15 +1505,15 @@ class InteractiveShell(SingletonConfigurable): def drop_by_id(self, variables): """Remove a dict of variables from the user namespace, if they are the same as the values in the dictionary. - + This is intended for use by extensions: variables that they've added can be taken back out if they are unloaded, without removing any that the user has overwritten. - + Parameters ---------- variables : dict - A dictionary mapping object names (as strings) to the objects. + A dictionary mapping object names (as strings) to the objects. """ for name, obj in variables.items(): if name in self.user_ns and self.user_ns[name] is obj: @@ -1820,7 +1710,7 @@ class InteractiveShell(SingletonConfigurable): """Get object info as formatted text""" return self.object_inspect_mime(oname, detail_level)['text/plain'] - def object_inspect_mime(self, oname, detail_level=0): + def object_inspect_mime(self, oname, detail_level=0, omit_sections=()): """Get object info as a mimebundle of formatted representations. A mimebundle is a dictionary, keyed by mime-type. @@ -1840,6 +1730,7 @@ class InteractiveShell(SingletonConfigurable): info=info, detail_level=detail_level, formatter=docformat, + omit_sections=omit_sections, ) else: raise KeyError(oname) @@ -1892,7 +1783,6 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - exc_tuple : tuple of exception classes A *tuple* of exception classes, for which to call the defined handler. It is very important that you use a tuple, and NOT A @@ -1922,10 +1812,11 @@ class InteractiveShell(SingletonConfigurable): Notes ----- - WARNING: by putting in your own exception handler into IPython's main execution loop, you run a very good chance of nasty crashes. This - facility should only be used if you really know what you are doing.""" + facility should only be used if you really know what you are doing. + """ + if not isinstance(exc_tuple, tuple): raise TypeError("The custom exceptions must be given as a tuple.") @@ -1937,10 +1828,10 @@ class InteractiveShell(SingletonConfigurable): def validate_stb(stb): """validate structured traceback return type - + return type of CustomTB *should* be a list of strings, but allow single strings or None, which are harmless. - + This function will *always* return a list of strings, and will raise a TypeError if stb is inappropriate. """ @@ -2095,10 +1986,19 @@ class InteractiveShell(SingletonConfigurable): # Exception classes can customise their traceback - we # use this in IPython.parallel for exceptions occurring # in the engines. This should return a list of strings. - stb = value._render_traceback_() + if hasattr(value, "_render_traceback_"): + stb = value._render_traceback_() + else: + stb = self.InteractiveTB.structured_traceback( + etype, value, tb, tb_offset=tb_offset + ) + except Exception: - stb = self.InteractiveTB.structured_traceback(etype, - value, tb, tb_offset=tb_offset) + print( + "Unexpected exception formatting exception. Falling back to standard exception" + ) + traceback.print_exc() + return None self._showtraceback(etype, value, stb) if self.call_pdb: @@ -2135,7 +2035,7 @@ class InteractiveShell(SingletonConfigurable): If the syntax error occurred when running a compiled code (i.e. running_compile_code=True), longer stack trace will be displayed. - """ + """ etype, value, last_traceback = self._get_exc_info() if filename and issubclass(etype, SyntaxError): @@ -2160,19 +2060,6 @@ class InteractiveShell(SingletonConfigurable): the %paste magic.""" self.showsyntaxerror() - #------------------------------------------------------------------------- - # Things related to readline - #------------------------------------------------------------------------- - - def init_readline(self): - """DEPRECATED - - Moved to terminal subclass, here only to simplify the init logic.""" - # Set a number of methods that depend on readline to be no-op - warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated', - DeprecationWarning, stacklevel=2) - self.set_custom_completer = no_op - @skip_doctest def set_next_input(self, s, replace=False): """ Sets the 'default' input string for the next command line. @@ -2201,8 +2088,12 @@ class InteractiveShell(SingletonConfigurable): (typically over the network by remote frontends). """ from IPython.core.completer import IPCompleter - from IPython.core.completerlib import (module_completer, - magic_run_completer, cd_completer, reset_completer) + from IPython.core.completerlib import ( + cd_completer, + magic_run_completer, + module_completer, + reset_completer, + ) self.Completer = IPCompleter(shell=self, namespace=self.user_ns, @@ -2229,26 +2120,24 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - - text : string - A string of text to be completed on. It can be given as empty and - instead a line/position pair are given. In this case, the - completer itself will split the line like readline does. - - line : string, optional - The complete line that text is part of. - - cursor_pos : int, optional - The position of the cursor on the input line. + text : string + A string of text to be completed on. It can be given as empty and + instead a line/position pair are given. In this case, the + completer itself will split the line like readline does. + line : string, optional + The complete line that text is part of. + cursor_pos : int, optional + The position of the cursor on the input line. Returns ------- - text : string + text : string The actual text that was completed. - - matches : list + matches : list A sorted list with all possible completions. + Notes + ----- The optional arguments allow the completion to take more context into account, and are part of the low-level completion API. @@ -2257,8 +2146,8 @@ class InteractiveShell(SingletonConfigurable): exposing it as a method, it can be used by other non-readline environments (such as GUIs) for text completion. - Simple usage example: - + Examples + -------- In [1]: x = 'hello' In [2]: _ip.complete('x.l') @@ -2341,7 +2230,7 @@ class InteractiveShell(SingletonConfigurable): func, magic_kind=magic_kind, magic_name=magic_name ) - def _find_with_lazy_load(self, type_, magic_name: str): + def _find_with_lazy_load(self, /, type_, magic_name: str): """ Try to find a magic potentially lazy-loading it. @@ -2374,14 +2263,12 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- magic_name : str - Name of the desired magic function, without '%' prefix. - + Name of the desired magic function, without '%' prefix. line : str - The rest of the input line as a single string. - + The rest of the input line as a single string. _stack_depth : int - If run_line_magic() is called from magic() then _stack_depth=2. - This is added to ensure backward compatibility for use of 'get_ipython().magic()' + If run_line_magic() is called from magic() then _stack_depth=2. + This is added to ensure backward compatibility for use of 'get_ipython().magic()' """ fn = self._find_with_lazy_load("line", magic_name) if fn is None: @@ -2423,7 +2310,7 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- stack_depth : int - Depth relative to calling frame + Depth relative to calling frame """ return sys._getframe(stack_depth + 1).f_locals @@ -2433,13 +2320,11 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- magic_name : str - Name of the desired magic function, without '%' prefix. - + Name of the desired magic function, without '%' prefix. line : str - The rest of the first input line as a single string. - + The rest of the first input line as a single string. cell : str - The body of the cell as a (possibly multiline) string. + The body of the cell as a (possibly multiline) string. """ fn = self._find_with_lazy_load("cell", magic_name) if fn is None: @@ -2491,7 +2376,11 @@ class InteractiveShell(SingletonConfigurable): return self.magics_manager.magics[magic_kind].get(magic_name) def magic(self, arg_s): - """DEPRECATED. Use run_line_magic() instead. + """ + DEPRECATED + + Deprecated since IPython 0.13 (warning added in + 8.1), use run_line_magic(magic_name, parameter_s). Call a magic function by name. @@ -2509,6 +2398,12 @@ class InteractiveShell(SingletonConfigurable): valid Python code you can type at the interpreter, including loops and compound statements. """ + warnings.warn( + "`magic(...)` is deprecated since IPython 0.13 (warning added in " + "8.1), use run_line_magic(magic_name, parameter_s).", + DeprecationWarning, + stacklevel=2, + ) # TODO: should we issue a loud deprecation warning here? magic_name, _, magic_arg_s = arg_s.partition(' ') magic_name = magic_name.lstrip(prefilter.ESC_MAGIC) @@ -2548,9 +2443,9 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- cmd : str - Command to execute (can not end in '&', as background processes are - not supported. Should not be a command that expects input - other than simple text. + Command to execute (can not end in '&', as background processes are + not supported. Should not be a command that expects input + other than simple text. """ if cmd.rstrip().endswith('&'): # this is *far* from a rigorous test @@ -2572,9 +2467,21 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- cmd : str - Command to execute. + Command to execute. """ cmd = self.var_expand(cmd, depth=1) + # warn if there is an IPython magic alternative. + main_cmd = cmd.split()[0] + has_magic_alternatives = ("pip", "conda", "cd") + + if main_cmd in has_magic_alternatives: + warnings.warn( + ( + "You executed the system command !{0} which may not work " + "as expected. Try the IPython magic %{0} instead." + ).format(main_cmd) + ) + # protect os.system from UNC paths on Windows, which it can't handle: if sys.platform == 'win32': from IPython.utils._process_win32 import AvoidUNCPath @@ -2623,18 +2530,18 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- cmd : str - Command to execute (can not end in '&', as background processes are - not supported. + Command to execute (can not end in '&', as background processes are + not supported. split : bool, optional - If True, split the output into an IPython SList. Otherwise, an - IPython LSString is returned. These are objects similar to normal - lists and strings, with a few convenience attributes for easier - manipulation of line-based output. You can use '?' on them for - details. + If True, split the output into an IPython SList. Otherwise, an + IPython LSString is returned. These are objects similar to normal + lists and strings, with a few convenience attributes for easier + manipulation of line-based output. You can use '?' on them for + details. depth : int, optional - How many frames above the caller are the local variables which should - be expanded in the command string? The default (0) assumes that the - expansion variables are in the stack frame calling this function. + How many frames above the caller are the local variables which should + be expanded in the command string? The default (0) assumes that the + expansion variables are in the stack frame calling this function. """ if cmd.rstrip().endswith('&'): # this is *far* from a rigorous test @@ -2717,10 +2624,10 @@ class InteractiveShell(SingletonConfigurable): stb = self.InteractiveTB.get_exception_only(etype, evalue) exc_info = { - u'status' : 'error', - u'traceback' : stb, - u'ename' : etype.__name__, - u'evalue' : py3compat.safe_unicode(evalue), + "status": "error", + "traceback": stb, + "ename": etype.__name__, + "evalue": py3compat.safe_unicode(evalue), } return exc_info @@ -2745,9 +2652,9 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- expressions : dict - A dict with string keys and string values. The expression values - should be valid Python expressions, each of which will be evaluated - in the user namespace. + A dict with string keys and string values. The expression values + should be valid Python expressions, each of which will be evaluated + in the user namespace. Returns ------- @@ -2794,7 +2701,7 @@ class InteractiveShell(SingletonConfigurable): ---------- fname : string The name of the file to be executed. - where : tuple + *where : tuple One or two namespaces, passed to execfile() as (globals,locals). If only one is given, it is passed as both. exit_ignore : bool (False) @@ -2809,11 +2716,11 @@ class InteractiveShell(SingletonConfigurable): __future__ imports are not shared in either direction. """ - fname = os.path.abspath(os.path.expanduser(fname)) + fname = Path(fname).expanduser().resolve() # Make sure we can open the file try: - with open(fname): + with fname.open("rb"): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2822,7 +2729,7 @@ class InteractiveShell(SingletonConfigurable): # Find things also in current directory. This is needed to mimic the # behavior of running a script from the system command line, where # Python inserts the script's directory into sys.path - dname = os.path.dirname(fname) + dname = str(fname.parent) with prepended_to_syspath(dname), self.builtin_trap: try: @@ -2867,11 +2774,11 @@ class InteractiveShell(SingletonConfigurable): raise_exceptions : bool (False) If True raise exceptions everywhere. Meant for testing. """ - fname = os.path.abspath(os.path.expanduser(fname)) + fname = Path(fname).expanduser().resolve() # Make sure we can open the file try: - with open(fname): + with fname.open("rb"): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2880,11 +2787,11 @@ class InteractiveShell(SingletonConfigurable): # Find things also in current directory. This is needed to mimic the # behavior of running a script from the system command line, where # Python inserts the script's directory into sys.path - dname = os.path.dirname(fname) - + dname = str(fname.parent) + def get_cells(): """generator for sequence of code blocks to run""" - if fname.endswith('.ipynb'): + if fname.suffix == ".ipynb": from nbformat import read nb = read(fname, as_version=4) if not nb.cells: @@ -2893,8 +2800,7 @@ class InteractiveShell(SingletonConfigurable): if cell.cell_type == 'code': yield cell.source else: - with open(fname) as f: - yield f.read() + yield fname.read_text(encoding="utf-8") with prepended_to_syspath(dname): try: @@ -2951,19 +2857,19 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- raw_cell : str - The code (including IPython code such as %magic functions) to run. + The code (including IPython code such as %magic functions) to run. store_history : bool - If True, the raw and translated cell will be stored in IPython's - history. For user code calling back into IPython's machinery, this - should be set to False. + If True, the raw and translated cell will be stored in IPython's + history. For user code calling back into IPython's machinery, this + should be set to False. silent : bool - If True, avoid side-effects, such as implicit displayhooks and - and logging. silent=True forces store_history=False. + If True, avoid side-effects, such as implicit displayhooks and + and logging. silent=True forces store_history=False. shell_futures : bool - If True, the code will share future statements with the interactive - shell. It will both be affected by previous __future__ imports, and - any __future__ imports in the code will affect the shell. If False, - __future__ imports are not shared in either direction. + If True, the code will share future statements with the interactive + shell. It will both be affected by previous __future__ imports, and + any __future__ imports in the code will affect the shell. If False, + __future__ imports are not shared in either direction. Returns ------- @@ -3035,7 +2941,6 @@ class InteractiveShell(SingletonConfigurable): result.error_in_exec = e self.showtraceback(running_compiled_code=True) return result - return def should_run_async( self, raw_cell: str, *, transformed_cell=None, preprocessing_exc_tuple=None @@ -3044,14 +2949,13 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - raw_cell: str + raw_cell : str The code to be executed Returns ------- result: bool Whether the code needs to be run with a coroutine runner or not - .. versionadded:: 7.0 """ if not self.autoawait: @@ -3175,9 +3079,8 @@ class InteractiveShell(SingletonConfigurable): cell = raw_cell # Store raw and processed history - if store_history: - self.history_manager.store_inputs(self.execution_count, - cell, raw_cell) + if store_history and raw_cell.strip(" %") != "paste": + self.history_manager.store_inputs(self.execution_count, cell, raw_cell) if not silent: self.logger.log(cell, raw_cell) @@ -3196,35 +3099,12 @@ class InteractiveShell(SingletonConfigurable): _run_async = False with self.builtin_trap: - cell_name = self.compile.cache( - cell, self.execution_count, raw_code=raw_cell - ) + cell_name = compiler.cache(cell, self.execution_count, raw_code=raw_cell) with self.display_trap: # Compile to bytecode try: - if sys.version_info < (3,8) and self.autoawait: - if _should_be_async(cell): - # the code AST below will not be user code: we wrap it - # in an `async def`. This will likely make some AST - # transformer below miss some transform opportunity and - # introduce a small coupling to run_code (in which we - # bake some assumptions of what _ast_asyncify returns. - # they are ways around (like grafting part of the ast - # later: - # - Here, return code_ast.body[0].body[1:-1], as well - # as last expression in return statement which is - # the user code part. - # - Let it go through the AST transformers, and graft - # - it back after the AST transform - # But that seem unreasonable, at least while we - # do not need it. - code_ast = _ast_asyncify(cell, 'async-def-wrapper') - _run_async = True - else: - code_ast = compiler.ast_parse(cell, filename=cell_name) - else: - code_ast = compiler.ast_parse(cell, filename=cell_name) + code_ast = compiler.ast_parse(cell, filename=cell_name) except self.custom_exceptions as e: etype, value, tb = sys.exc_info() self.CustomTB(etype, value, tb) @@ -3250,8 +3130,6 @@ class InteractiveShell(SingletonConfigurable): # Execute the user code interactivity = "none" if silent else self.ast_node_interactivity - if _run_async: - interactivity = 'async' has_raised = await self.run_ast_nodes(code_ast.body, cell_name, interactivity=interactivity, compiler=compiler, result=result) @@ -3308,8 +3186,8 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- node : ast.Node - The root node to be transformed. Typically called with the ast.Module - produced by parsing user input. + The root node to be transformed. Typically called with the ast.Module + produced by parsing user input. Returns ------- @@ -3356,8 +3234,14 @@ class InteractiveShell(SingletonConfigurable): return code return code.replace(co_name="<cell line: %s>" % (first_real_line,)) - async def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr', - compiler=compile, result=None): + async def run_ast_nodes( + self, + nodelist: ListType[stmt], + cell_name: str, + interactivity="last_expr", + compiler=compile, + result=None, + ): """Run a sequence of AST nodes. The execution mode depends on the interactivity parameter. @@ -3377,11 +3261,6 @@ class InteractiveShell(SingletonConfigurable): or the last assignment. Other values for this parameter will raise a ValueError. - Experimental value: 'async' Will try to run top level interactive - async/await code in default runner, this will not respect the - interactivity setting and will only run the last node if it is an - expression. - compiler : callable A function with the same interface as the built-in compile(), to turn the AST nodes into code objects. Default is the built-in compile(). @@ -3396,6 +3275,7 @@ class InteractiveShell(SingletonConfigurable): if not nodelist: return + if interactivity == 'last_expr_or_assign': if isinstance(nodelist[-1], _assign_nodes): asg = nodelist[-1] @@ -3424,53 +3304,38 @@ class InteractiveShell(SingletonConfigurable): to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:] elif interactivity == 'all': to_run_exec, to_run_interactive = [], nodelist - elif interactivity == 'async': - to_run_exec, to_run_interactive = [], nodelist - _async = True else: raise ValueError("Interactivity was %r" % interactivity) try: - if _async and sys.version_info > (3,8): - raise ValueError("This branch should never happen on Python 3.8 and above, " - "please try to upgrade IPython and open a bug report with your case.") - if _async: - # If interactivity is async the semantics of run_code are - # completely different Skip usual machinery. - mod = Module(nodelist, []) - async_wrapper_code = compiler(mod, cell_name, 'exec') - exec(async_wrapper_code, self.user_global_ns, self.user_ns) - async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__ - if (await self.run_code(async_code, result, async_=True)): + + def compare(code): + is_async = inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE + return is_async + + # refactor that to just change the mod constructor. + to_run = [] + for node in to_run_exec: + to_run.append((node, "exec")) + + for node in to_run_interactive: + to_run.append((node, "single")) + + for node, mode in to_run: + if mode == "exec": + mod = Module([node], []) + elif mode == "single": + mod = ast.Interactive([node]) + with compiler.extra_flags( + getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0) + if self.autoawait + else 0x0 + ): + code = compiler(mod, cell_name, mode) + code = self._update_code_co_name(code) + asy = compare(code) + if await self.run_code(code, result, async_=asy): return True - else: - if sys.version_info > (3, 8): - def compare(code): - is_async = (inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE) - return is_async - else: - def compare(code): - return _async - - # refactor that to just change the mod constructor. - to_run = [] - for node in to_run_exec: - to_run.append((node, 'exec')) - - for node in to_run_interactive: - to_run.append((node, 'single')) - - for node,mode in to_run: - if mode == 'exec': - mod = Module([node], []) - elif mode == 'single': - mod = ast.Interactive([node]) - with compiler.extra_flags(getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0) if self.autoawait else 0x0): - code = compiler(mod, cell_name, mode) - code = self._update_code_co_name(code) - asy = compare(code) - if (await self.run_code(code, result, async_=asy)): - return True # Flush softspace if softspace(sys.stdout, 0): @@ -3493,21 +3358,6 @@ class InteractiveShell(SingletonConfigurable): return False - def _async_exec(self, code_obj: types.CodeType, user_ns: dict): - """ - Evaluate an asynchronous code object using a code runner - - Fake asynchronous execution of code_object in a namespace via a proxy namespace. - - Returns coroutine object, which can be executed via async loop runner - - WARNING: The semantics of `async_exec` are quite different from `exec`, - in particular you can only pass a single namespace. It also return a - handle to the value of the last things returned by code_object. - """ - - return eval(code_obj, user_ns) - async def run_code(self, code_obj, result=None, *, async_=False): """Execute a code object. @@ -3541,12 +3391,7 @@ class InteractiveShell(SingletonConfigurable): outflag = True # happens in more places, so it's easier as default try: try: - self.hooks.pre_run_code_hook() - if async_ and sys.version_info < (3,8): - last_expr = (await self._async_exec(code_obj, self.user_ns)) - code = compile('last_expr', 'fake', "single") - exec(code, {'last_expr': last_expr}) - elif async_ : + if async_: await eval(code_obj, self.user_global_ns, self.user_ns) else: exec(code_obj, self.user_global_ns, self.user_ns) @@ -3579,17 +3424,17 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - source : string - Python input code, which can be multiline. + code : string + Python input code, which can be multiline. Returns ------- status : str - One of 'complete', 'incomplete', or 'invalid' if source is not a - prefix of valid code. + One of 'complete', 'incomplete', or 'invalid' if source is not a + prefix of valid code. indent : str - When status is 'incomplete', this is some whitespace to insert on - the next line of the prompt. + When status is 'incomplete', this is some whitespace to insert on + the next line of the prompt. """ status, nspaces = self.input_transformer_manager.check_complete(code) return status, ' ' * (nspaces or 0) @@ -3616,16 +3461,17 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- gui : optional, string - If given, dictates the choice of matplotlib GUI backend to use - (should be one of IPython's supported backends, 'qt', 'osx', 'tk', - 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by - matplotlib (as dictated by the matplotlib build-time options plus the - user's matplotlibrc configuration file). Note that not all backends - make sense in all contexts, for example a terminal ipython can't - display figures inline. + If given, dictates the choice of matplotlib GUI backend to use + (should be one of IPython's supported backends, 'qt', 'osx', 'tk', + 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by + matplotlib (as dictated by the matplotlib build-time options plus the + user's matplotlibrc configuration file). Note that not all backends + make sense in all contexts, for example a terminal ipython can't + display figures inline. """ - from IPython.core import pylabtools as pt from matplotlib_inline.backend_inline import configure_inline_support + + from IPython.core import pylabtools as pt gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select) if gui != 'inline': @@ -3662,18 +3508,18 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- gui : optional, string - If given, dictates the choice of matplotlib GUI backend to use - (should be one of IPython's supported backends, 'qt', 'osx', 'tk', - 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by - matplotlib (as dictated by the matplotlib build-time options plus the - user's matplotlibrc configuration file). Note that not all backends - make sense in all contexts, for example a terminal ipython can't - display figures inline. + If given, dictates the choice of matplotlib GUI backend to use + (should be one of IPython's supported backends, 'qt', 'osx', 'tk', + 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by + matplotlib (as dictated by the matplotlib build-time options plus the + user's matplotlibrc configuration file). Note that not all backends + make sense in all contexts, for example a terminal ipython can't + display figures inline. import_all : optional, bool, default: True - Whether to do `from numpy import *` and `from pylab import *` - in addition to module imports. + Whether to do `from numpy import *` and `from pylab import *` + in addition to module imports. welcome_message : deprecated - This argument is ignored, no welcome message will be displayed. + This argument is ignored, no welcome message will be displayed. """ from IPython.core.pylabtools import import_pylab @@ -3738,32 +3584,19 @@ class InteractiveShell(SingletonConfigurable): - data(None): if data is given, it gets written out to the temp file immediately, and the file is closed again.""" - dirname = tempfile.mkdtemp(prefix=prefix) - self.tempdirs.append(dirname) + dir_path = Path(tempfile.mkdtemp(prefix=prefix)) + self.tempdirs.append(dir_path) - handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname) + handle, filename = tempfile.mkstemp(".py", prefix, dir=str(dir_path)) os.close(handle) # On Windows, there can only be one open handle on a file - self.tempfiles.append(filename) + + file_path = Path(filename) + self.tempfiles.append(file_path) if data: - with open(filename, 'w') as tmp_file: - tmp_file.write(data) + file_path.write_text(data, encoding="utf-8") return filename - @undoc - def write(self,data): - """DEPRECATED: Write a string to the default output""" - warn('InteractiveShell.write() is deprecated, use sys.stdout instead', - DeprecationWarning, stacklevel=2) - sys.stdout.write(data) - - @undoc - def write_err(self,data): - """DEPRECATED: Write a string to the default error output""" - warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead', - DeprecationWarning, stacklevel=2) - sys.stderr.write(data) - def ask_yes_no(self, prompt, default=None, interrupt=None): if self.quiet: return True @@ -3778,26 +3611,37 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - range_str : string + range_str : str The set of slices is given as a string, like "~5/6-~4/2 4:8 9", since this function is for use by magic functions which get their arguments as strings. The number before the / is the session number: ~n goes n back from the current session. + If empty string is given, returns history of current session + without the last input. + raw : bool, optional By default, the processed input is used. If this is true, the raw input history is used instead. Notes ----- - Slices can be described with two notations: * ``N:M`` -> standard python form, means including items N...(M-1). * ``N-M`` -> include items N..M (closed endpoint). """ lines = self.history_manager.get_range_by_str(range_str, raw=raw) - return "\n".join(x for _, _, x in lines) + text = "\n".join(x for _, _, x in lines) + + # Skip the last line, as it's probably the magic that called this + if not range_str: + if "\n" not in text: + text = "" + else: + text = text[: text.rfind("\n")] + + return text def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False): """Get a code string from history, file, url, or a string or macro. @@ -3806,26 +3650,26 @@ class InteractiveShell(SingletonConfigurable): Parameters ---------- - target : str + A string specifying code to retrieve. This will be tried respectively + as: ranges of input history (see %history for syntax), url, + corresponding .py file, filename, or an expression evaluating to a + string or Macro in the user namespace. - A string specifying code to retrieve. This will be tried respectively - as: ranges of input history (see %history for syntax), url, - corresponding .py file, filename, or an expression evaluating to a - string or Macro in the user namespace. + If empty string is given, returns complete history of current + session, without the last line. raw : bool - If true (default), retrieve raw history. Has no effect on the other - retrieval mechanisms. + If true (default), retrieve raw history. Has no effect on the other + retrieval mechanisms. py_only : bool (default False) - Only try to fetch python code, do not try alternative methods to decode file - if unicode fails. + Only try to fetch python code, do not try alternative methods to decode file + if unicode fails. Returns ------- A string of code. - ValueError is raised if nothing is found, and TypeError if it evaluates to an object of another type. In each case, .args[0] is a printable message. @@ -3836,13 +3680,13 @@ class InteractiveShell(SingletonConfigurable): try: if target.startswith(('http://', 'https://')): return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie) - except UnicodeDecodeError: + except UnicodeDecodeError as e: if not py_only : # Deferred import from urllib.request import urlopen response = urlopen(target) return response.read().decode('latin1') - raise ValueError(("'%s' seem to be unreadable.") % target) + raise ValueError(("'%s' seem to be unreadable.") % target) from e potential_target = [target] try : @@ -3854,11 +3698,11 @@ class InteractiveShell(SingletonConfigurable): if os.path.isfile(tgt): # Read file try : return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie) - except UnicodeDecodeError : + except UnicodeDecodeError as e: if not py_only : with io_open(tgt,'r', encoding='latin1') as f : return f.read() - raise ValueError(("'%s' seem to be unreadable.") % target) + raise ValueError(("'%s' seem to be unreadable.") % target) from e elif os.path.isdir(os.path.expanduser(tgt)): raise ValueError("'%s' is a directory, not a regular file." % target) @@ -3870,9 +3714,9 @@ class InteractiveShell(SingletonConfigurable): try: # User namespace codeobj = eval(target, self.user_ns) - except Exception: + except Exception as e: raise ValueError(("'%s' was not found in history, as a file, url, " - "nor in the user namespace.") % target) + "nor in the user namespace.") % target) from e if isinstance(codeobj, str): return codeobj @@ -3882,6 +3726,22 @@ class InteractiveShell(SingletonConfigurable): raise TypeError("%s is neither a string nor a macro." % target, codeobj) + def _atexit_once(self): + """ + At exist operation that need to be called at most once. + Second call to this function per instance will do nothing. + """ + + if not getattr(self, "_atexit_once_called", False): + self._atexit_once_called = True + # Clear all user namespaces to release all references cleanly. + self.reset(new_session=False) + # Close the history session (this stores the end time and line count) + # this must be *before* the tempfile cleanup, in case of temporary + # history db + self.history_manager.end_session() + self.history_manager = None + #------------------------------------------------------------------------- # Things related to IPython exiting #------------------------------------------------------------------------- @@ -3896,29 +3756,28 @@ class InteractiveShell(SingletonConfigurable): code that has the appropriate information, rather than trying to clutter """ - # Close the history session (this stores the end time and line count) - # this must be *before* the tempfile cleanup, in case of temporary - # history db - self.history_manager.end_session() + self._atexit_once() # Cleanup all tempfiles and folders left around for tfile in self.tempfiles: try: - os.unlink(tfile) - except OSError: + tfile.unlink() + self.tempfiles.remove(tfile) + except FileNotFoundError: pass - + del self.tempfiles for tdir in self.tempdirs: try: - os.rmdir(tdir) - except OSError: + tdir.rmdir() + self.tempdirs.remove(tdir) + except FileNotFoundError: pass + del self.tempdirs - # Clear all user namespaces to release all references cleanly. - self.reset(new_session=False) - - # Run user hooks - self.hooks.shutdown_hook() + # Restore user's cursor + if hasattr(self, "editing_mode") and self.editing_mode == "vi": + sys.stdout.write("\x1b[0 q") + sys.stdout.flush() def cleanup(self): self.restore_sys_module_state() diff --git a/contrib/python/ipython/py3/IPython/core/magic.py b/contrib/python/ipython/py3/IPython/core/magic.py index b41a651f50..cedba61937 100644 --- a/contrib/python/ipython/py3/IPython/core/magic.py +++ b/contrib/python/ipython/py3/IPython/core/magic.py @@ -20,7 +20,6 @@ from traitlets.config.configurable import Configurable from . import oinspect from .error import UsageError from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 -from decorator import decorator from ..utils.ipstruct import Struct from ..utils.process import arg_split from ..utils.text import dedent @@ -115,16 +114,13 @@ def record_magic(dct, magic_kind, magic_name, func): Parameters ---------- dct : dict - A dictionary with 'line' and 'cell' subdicts. - + A dictionary with 'line' and 'cell' subdicts. magic_kind : str - Kind of magic to be stored. - + Kind of magic to be stored. magic_name : str - Key to store the magic as. - + Key to store the magic as. func : function - Callable object to store. + Callable object to store. """ if magic_kind == 'line_cell': dct['line'][magic_name] = dct['cell'][magic_name] = func @@ -184,20 +180,18 @@ def _method_magic_marker(magic_kind): # This is a closure to capture the magic_kind. We could also use a class, # but it's overkill for just that one bit of state. def magic_deco(arg): - call = lambda f, *a, **k: f(*a, **k) - if callable(arg): # "Naked" decorator call (just @foo, no args) func = arg name = func.__name__ - retval = decorator(call, func) + retval = arg record_magic(magics, magic_kind, name, name) elif isinstance(arg, str): # Decorator called with arguments (@foo('bar')) name = arg def mark(func, *a, **kw): record_magic(magics, magic_kind, name, func.__name__) - return decorator(call, func) + return func retval = mark else: raise TypeError("Decorator can only be called with " @@ -217,8 +211,6 @@ def _function_magic_marker(magic_kind): # This is a closure to capture the magic_kind. We could also use a class, # but it's overkill for just that one bit of state. def magic_deco(arg): - call = lambda f, *a, **k: f(*a, **k) - # Find get_ipython() in the caller's namespace caller = sys._getframe(1) for ns in ['f_locals', 'f_globals', 'f_builtins']: @@ -236,13 +228,13 @@ def _function_magic_marker(magic_kind): func = arg name = func.__name__ ip.register_magic_function(func, magic_kind, name) - retval = decorator(call, func) + retval = arg elif isinstance(arg, str): # Decorator called with arguments (@foo('bar')) name = arg def mark(func, *a, **kw): ip.register_magic_function(func, magic_kind, name) - return decorator(call, func) + return func retval = mark else: raise TypeError("Decorator can only be called with " @@ -423,7 +415,7 @@ class MagicsManager(Configurable): def register(self, *magic_objects): """Register one or more instances of Magics. - Take one or more classes or instances of classes that subclass the main + Take one or more classes or instances of classes that subclass the main `core.Magic` class, and register them with IPython to use the magic functions they provide. The registration process will then ensure that any methods that have decorated to provide line and/or cell magics will @@ -438,7 +430,7 @@ class MagicsManager(Configurable): Parameters ---------- - magic_objects : one or more classes or instances + *magic_objects : one or more classes or instances """ # Start by validating them to ensure they have all had their magic # methods registered at the instance level @@ -461,7 +453,7 @@ class MagicsManager(Configurable): This will create an IPython magic (line, cell or both) from a standalone function. The functions should have the following - signatures: + signatures: * For line magics: `def f(line)` * For cell magics: `def f(line, cell)` @@ -473,14 +465,12 @@ class MagicsManager(Configurable): Parameters ---------- func : callable - Function to be registered as a magic. - + Function to be registered as a magic. magic_kind : str - Kind of magic, one of 'line', 'cell' or 'line_cell' - + Kind of magic, one of 'line', 'cell' or 'line_cell' magic_name : optional str - If given, the name the magic will have in the IPython namespace. By - default, the name of the function itself is used. + If given, the name the magic will have in the IPython namespace. By + default, the name of the function itself is used. """ # Create the new method in the user_magics and register it in the @@ -501,13 +491,11 @@ class MagicsManager(Configurable): Parameters ---------- alias_name : str - The name of the magic to be registered. - + The name of the magic to be registered. magic_name : str - The name of an existing magic. - + The name of an existing magic. magic_kind : str - Kind of magic, one of 'line' or 'cell' + Kind of magic, one of 'line' or 'cell' """ # `validate_type` is too permissive, as it allows 'line_cell' @@ -631,25 +619,20 @@ class Magics(Configurable): Parameters ---------- - arg_str : str - The arguments to parse. - + The arguments to parse. opt_str : str - The options specification. - + The options specification. mode : str, default 'string' - If given as 'list', the argument string is returned as a list (split - on whitespace) instead of a string. - + If given as 'list', the argument string is returned as a list (split + on whitespace) instead of a string. list_all : bool, default False - Put all option values in lists. Normally only options - appearing more than once are put in a list. - + Put all option values in lists. Normally only options + appearing more than once are put in a list. posix : bool, default True - Whether to split the input line in POSIX mode or not, as per the - conventions outlined in the :mod:`shlex` module from the standard - library. + Whether to split the input line in POSIX mode or not, as per the + conventions outlined in the :mod:`shlex` module from the standard + library. """ # inject default options at the beginning of the input line @@ -664,6 +647,9 @@ class Magics(Configurable): posix = kw.get('posix', os.name == 'posix') strict = kw.get('strict', True) + preserve_non_opts = kw.get("preserve_non_opts", False) + remainder_arg_str = arg_str + # Check if we have more than one argument to warrant extra processing: odict = {} # Dictionary with options args = arg_str.split() @@ -675,10 +661,18 @@ class Magics(Configurable): try: opts,args = getopt(argv, opt_str, long_opts) except GetoptError as e: - raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, - " ".join(long_opts))) - for o,a in opts: - if o.startswith('--'): + raise UsageError( + '%s ( allowed: "%s" %s)' % (e.msg, opt_str, " ".join(long_opts)) + ) from e + for o, a in opts: + if mode == "string" and preserve_non_opts: + # remove option-parts from the original args-string and preserve remaining-part. + # This relies on the arg_split(...) and getopt(...)'s impl spec, that the parsed options are + # returned in the original order. + remainder_arg_str = remainder_arg_str.replace(o, "", 1).replace( + a, "", 1 + ) + if o.startswith("--"): o = o[2:] else: o = o[1:] @@ -695,7 +689,10 @@ class Magics(Configurable): # Prepare opts,args for return opts = Struct(odict) if mode == 'string': - args = ' '.join(args) + if preserve_non_opts: + args = remainder_arg_str.lstrip() + else: + args = " ".join(args) return opts,args diff --git a/contrib/python/ipython/py3/IPython/core/magic_arguments.py b/contrib/python/ipython/py3/IPython/core/magic_arguments.py index 9231609572..568abd82ae 100644 --- a/contrib/python/ipython/py3/IPython/core/magic_arguments.py +++ b/contrib/python/ipython/py3/IPython/core/magic_arguments.py @@ -37,6 +37,38 @@ arguments:: -o OPTION, --option OPTION An optional argument. +Here is an elaborated example that uses default parameters in `argument` and calls the `args` in the cell magic:: + + from IPython.core.magic import register_cell_magic + from IPython.core.magic_arguments import (argument, magic_arguments, + parse_argstring) + + + @magic_arguments() + @argument( + "--option", + "-o", + help=("Add an option here"), + ) + @argument( + "--style", + "-s", + default="foo", + help=("Add some style arguments"), + ) + @register_cell_magic + def my_cell_magic(line, cell): + args = parse_argstring(my_cell_magic, line) + print(f"{args.option=}") + print(f"{args.style=}") + print(f"{cell=}") + +In a jupyter notebook, this cell magic can be executed like this:: + + %%my_cell_magic -o Hello + print("bar") + i = 42 + Inheritance diagram: .. inheritance-diagram:: IPython.core.magic_arguments diff --git a/contrib/python/ipython/py3/IPython/core/magics/auto.py b/contrib/python/ipython/py3/IPython/core/magics/auto.py index a18542f43d..56aa4f72eb 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/auto.py +++ b/contrib/python/ipython/py3/IPython/core/magics/auto.py @@ -104,16 +104,32 @@ class AutoMagics(Magics): # all-random (note for auto-testing) """ + valid_modes = { + 0: "Off", + 1: "Smart", + 2: "Full", + } + + def errorMessage() -> str: + error = "Valid modes: " + for k, v in valid_modes.items(): + error += str(k) + "->" + v + ", " + error = error[:-2] # remove tailing `, ` after last element + return error + if parameter_s: + if not parameter_s in map(str, valid_modes.keys()): + error(errorMessage()) + return arg = int(parameter_s) else: arg = 'toggle' - if not arg in (0, 1, 2, 'toggle'): - error('Valid modes: (0->Off, 1->Smart, 2->Full') + if not arg in (*list(valid_modes.keys()), "toggle"): + error(errorMessage()) return - if arg in (0, 1, 2): + if arg in (valid_modes.keys()): self.shell.autocall = arg else: # toggle if self.shell.autocall: @@ -125,4 +141,4 @@ class AutoMagics(Magics): except AttributeError: self.shell.autocall = self._magic_state.autocall_save = 1 - print("Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]) + print("Automatic calling is:", list(valid_modes.values())[self.shell.autocall]) diff --git a/contrib/python/ipython/py3/IPython/core/magics/basic.py b/contrib/python/ipython/py3/IPython/core/magics/basic.py index 72cfc80414..c1f6945110 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/basic.py +++ b/contrib/python/ipython/py3/IPython/core/magics/basic.py @@ -4,6 +4,7 @@ import argparse from logging import error import io +import os from pprint import pformat import sys from warnings import warn @@ -45,7 +46,7 @@ class MagicsDisplay(object): def _jsonable(self): """turn magics dict into jsonable dict of the same structure - + replaces object instances with their class names as strings """ magic_dict = {} @@ -74,6 +75,7 @@ class BasicMagics(Magics): These are various magics that don't fit into specific categories but that are all part of the base 'IPython experience'.""" + @skip_doctest @magic_arguments.magic_arguments() @magic_arguments.argument( '-l', '--line', action='store_true', @@ -122,7 +124,7 @@ class BasicMagics(Magics): In [6]: %whereami Out[6]: u'/home/testuser' - + In [7]: %alias_magic h history "-p -l 30" --line Created `%h` as an alias for `%history -l 30`. """ @@ -366,7 +368,7 @@ Currently the magic system has the following functions:""", If called without arguments, acts as a toggle. - When in verbose mode the value --show (and --hide) + When in verbose mode the value --show (and --hide) will respectively show (or hide) frames with ``__tracebackhide__ = True`` value set. """ @@ -560,10 +562,6 @@ Currently the magic system has the following functions:""", @magic_arguments.magic_arguments() @magic_arguments.argument( - '-e', '--export', action='store_true', default=False, - help=argparse.SUPPRESS - ) - @magic_arguments.argument( 'filename', type=str, help='Notebook name or filename' ) @@ -573,11 +571,9 @@ Currently the magic system has the following functions:""", This function can export the current IPython history to a notebook file. For example, to export the history to "foo.ipynb" do "%notebook foo.ipynb". - - The -e or --export flag is deprecated in IPython 5.2, and will be - removed in the future. """ args = magic_arguments.parse_argstring(self.notebook, s) + outfname = os.path.expanduser(args.filename) from nbformat import write, v4 @@ -591,7 +587,7 @@ Currently the magic system has the following functions:""", source=source )) nb = v4.new_notebook(cells=cells) - with io.open(args.filename, 'w', encoding='utf-8') as f: + with io.open(outfname, "w", encoding="utf-8") as f: write(nb, f, version=4) @magics_class @@ -622,12 +618,11 @@ class AsyncMagics(BasicMagics): If the passed parameter does not match any of the above and is a python identifier, get said object from user namespace and set it as the - runner, and activate autoawait. + runner, and activate autoawait. If the object is a fully qualified object name, attempt to import it and set it as the runner, and activate autoawait. - - + The exact behavior of autoawait is experimental and subject to change across version of IPython and Python. """ diff --git a/contrib/python/ipython/py3/IPython/core/magics/code.py b/contrib/python/ipython/py3/IPython/core/magics/code.py index d446d35ac6..65ba52b8bb 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/code.py +++ b/contrib/python/ipython/py3/IPython/core/magics/code.py @@ -22,6 +22,7 @@ import ast from itertools import chain from urllib.request import Request, urlopen from urllib.parse import urlencode +from pathlib import Path # Our own packages from IPython.core.error import TryNext, StdinNotImplementedError, UsageError @@ -184,7 +185,7 @@ class CodeMagics(Magics): """Save a set of lines or a macro to a given filename. Usage:\\ - %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ... + %save [options] filename [history] Options: @@ -198,9 +199,12 @@ class CodeMagics(Magics): -a: append to the file instead of overwriting it. - This function uses the same syntax as %history for input ranges, + The history argument uses the same syntax as %history for input ranges, then saves the lines to the filename you specify. + If no ranges are specified, saves history of the current session up to + this point. + It adds a '.py' extension to the file if you don't do so yourself, and it asks for confirmation before overwriting existing files. @@ -218,6 +222,7 @@ class CodeMagics(Magics): fname, codefrom = args[0], " ".join(args[1:]) if not fname.endswith(('.py','.ipy')): fname += ext + fname = os.path.expanduser(fname) file_exists = os.path.isfile(fname) if file_exists and not force and not append: try: @@ -253,6 +258,9 @@ class CodeMagics(Magics): The argument can be an input history range, a filename, or the name of a string or macro. + If no arguments are given, uploads the history of this session up to + this point. + Options: -d: Pass a custom description. The default will say @@ -314,6 +322,9 @@ class CodeMagics(Magics): where source can be a filename, URL, input history range, macro, or element in the user namespace + If no arguments are given, loads the history of this session up to this + point. + Options: -r <lines>: Specify lines or ranges of lines to load from the source. @@ -332,6 +343,7 @@ class CodeMagics(Magics): confirmation before loading source with more than 200 000 characters, unless -y flag is passed or if the frontend does not support raw_input:: + %load %load myscript.py %load 7-27 %load myMacro @@ -343,13 +355,7 @@ class CodeMagics(Magics): %load -n my_module.wonder_function """ opts,args = self.parse_options(arg_s,'yns:r:') - - if not args: - raise UsageError('Missing filename, URL, input history range, ' - 'macro, or element in the user namespace.') - search_ns = 'n' in opts - contents = self.shell.find_user_code(args, search_ns=search_ns) if 's' in opts: @@ -460,10 +466,10 @@ class CodeMagics(Magics): return (None, None, None) use_temp = False - except DataIsObject: + except DataIsObject as e: # macros have a special edit function if isinstance(data, Macro): - raise MacroToEdit(data) + raise MacroToEdit(data) from e # For objects, try to edit the file where they are defined filename = find_file(data) @@ -487,8 +493,8 @@ class CodeMagics(Magics): m = ipython_input_pat.match(os.path.basename(filename)) if m: - raise InteractivelyDefined(int(m.groups()[0])) - + raise InteractivelyDefined(int(m.groups()[0])) from e + datafile = 1 if filename is None: filename = make_filename(args) @@ -532,8 +538,7 @@ class CodeMagics(Magics): self.shell.hooks.editor(filename) # and make a new macro object, to replace the old one - with open(filename) as mfile: - mvalue = mfile.read() + mvalue = Path(filename).read_text(encoding="utf-8") self.shell.user_ns[mname] = Macro(mvalue) @skip_doctest @@ -708,20 +713,22 @@ class CodeMagics(Magics): # do actual editing here print('Editing...', end=' ') sys.stdout.flush() + filepath = Path(filename) try: - # Quote filenames that may have spaces in them - if ' ' in filename: - filename = "'%s'" % filename - self.shell.hooks.editor(filename,lineno) + # Quote filenames that may have spaces in them when opening + # the editor + quoted = filename = str(filepath.absolute()) + if " " in quoted: + quoted = "'%s'" % quoted + self.shell.hooks.editor(quoted, lineno) except TryNext: warn('Could not open editor') return # XXX TODO: should this be generalized for all string vars? # For now, this is special-cased to blocks created by cpaste - if args.strip() == 'pasted_block': - with open(filename, 'r') as f: - self.shell.user_ns['pasted_block'] = f.read() + if args.strip() == "pasted_block": + self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8") if 'x' in opts: # -x prevents actual execution print() @@ -729,10 +736,9 @@ class CodeMagics(Magics): print('done. Executing edited code...') with preserve_keys(self.shell.user_ns, '__file__'): if not is_temp: - self.shell.user_ns['__file__'] = filename - if 'r' in opts: # Untranslated IPython code - with open(filename, 'r') as f: - source = f.read() + self.shell.user_ns["__file__"] = filename + if "r" in opts: # Untranslated IPython code + source = filepath.read_text(encoding="utf-8") self.shell.run_cell(source, store_history=False) else: self.shell.safe_execfile(filename, self.shell.user_ns, @@ -740,10 +746,9 @@ class CodeMagics(Magics): if is_temp: try: - with open(filename) as f: - return f.read() + return filepath.read_text(encoding="utf-8") except IOError as msg: - if msg.filename == filename: + if Path(msg.filename) == filepath: warn('File not found. Did you forget to save?') return else: diff --git a/contrib/python/ipython/py3/IPython/core/magics/config.py b/contrib/python/ipython/py3/IPython/core/magics/config.py index 97b13df02e..c1387b601b 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/config.py +++ b/contrib/python/ipython/py3/IPython/core/magics/config.py @@ -54,44 +54,73 @@ class ConfigMagics(Magics): In [1]: %config Available objects for config: - TerminalInteractiveShell - HistoryManager - PrefilterManager AliasManager - IPCompleter DisplayFormatter + HistoryManager + IPCompleter + LoggingMagics + MagicsManager + OSMagics + PrefilterManager + ScriptMagics + TerminalInteractiveShell To view what is configurable on a given class, just pass the class name:: In [2]: %config IPCompleter - IPCompleter options - ----------------- - IPCompleter.omit__names=<Enum> - Current: 2 - Choices: (0, 1, 2) - Instruct the completer to omit private method names - Specifically, when completing on ``object.<tab>``. - When 2 [default]: all names that start with '_' will be excluded. - When 1: all 'magic' names (``__foo__``) will be excluded. - When 0: nothing will be excluded. - IPCompleter.merge_completions=<CBool> + IPCompleter(Completer) options + ---------------------------- + IPCompleter.backslash_combining_completions=<Bool> + Enable unicode completions, e.g. \\alpha<tab> . Includes completion of latex + commands, unicode names, and expanding unicode characters back to latex + commands. Current: True - Whether to merge completion results into a single list - If False, only the completion results from the first non-empty - completer will be returned. - IPCompleter.limit_to__all__=<CBool> + IPCompleter.debug=<Bool> + Enable debug for the Completer. Mostly print extra information for + experimental jedi integration. + Current: False + IPCompleter.greedy=<Bool> + Activate greedy completion + PENDING DEPRECATION. this is now mostly taken care of with Jedi. + This will enable completion on elements of lists, results of function calls, etc., + but can be unsafe because the code is actually evaluated on TAB. Current: False + IPCompleter.jedi_compute_type_timeout=<Int> + Experimental: restrict time (in milliseconds) during which Jedi can compute types. + Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt + performance by preventing jedi to build its cache. + Current: 400 + IPCompleter.limit_to__all__=<Bool> + DEPRECATED as of version 5.0. Instruct the completer to use __all__ for the completion Specifically, when completing on ``object.<tab>``. When True: only those names in obj.__all__ will be included. When False [default]: the __all__ attribute is ignored - IPCompleter.greedy=<CBool> Current: False - Activate greedy completion - This will enable completion on elements of lists, results of - function calls, etc., but can be unsafe because the code is - actually evaluated on TAB. + IPCompleter.merge_completions=<Bool> + Whether to merge completion results into a single list + If False, only the completion results from the first non-empty + completer will be returned. + Current: True + IPCompleter.omit__names=<Enum> + Instruct the completer to omit private method names + Specifically, when completing on ``object.<tab>``. + When 2 [default]: all names that start with '_' will be excluded. + When 1: all 'magic' names (``__foo__``) will be excluded. + When 0: nothing will be excluded. + Choices: any of [0, 1, 2] + Current: 2 + IPCompleter.profile_completions=<Bool> + If True, emit profiling data for completion subsystem using cProfile. + Current: False + IPCompleter.profiler_output_dir=<Unicode> + Template for path at which to output profile data for completions. + Current: '.completion_profiles' + IPCompleter.use_jedi=<Bool> + Experimental: Use Jedi to generate autocompletions. Default to True if jedi + is installed. + Current: True but the real use is in setting values:: @@ -118,7 +147,7 @@ class ConfigMagics(Magics): # print available configurable names print("Available objects for config:") for name in classnames: - print(" ", name) + print(" ", name) return elif line in classnames: # `%config TerminalInteractiveShell` will print trait info for @@ -149,7 +178,7 @@ class ConfigMagics(Magics): # leave quotes on args when splitting, because we want # unquoted args to eval in user_ns cfg = Config() - exec("cfg."+line, locals(), self.shell.user_ns) + exec("cfg."+line, self.shell.user_ns, locals()) for configurable in configurables: try: diff --git a/contrib/python/ipython/py3/IPython/core/magics/display.py b/contrib/python/ipython/py3/IPython/core/magics/display.py index 0785394471..6c0eff6884 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/display.py +++ b/contrib/python/ipython/py3/IPython/core/magics/display.py @@ -12,7 +12,7 @@ #----------------------------------------------------------------------------- # Our own packages -from IPython.core.display import display, Javascript, Latex, SVG, HTML, Markdown +from IPython.display import display, Javascript, Latex, SVG, HTML, Markdown from IPython.core.magic import ( Magics, magics_class, cell_magic ) @@ -36,22 +36,33 @@ class DisplayMagics(Magics): """Run the cell block of Javascript code Alias of `%%javascript` + + Starting with IPython 8.0 %%javascript is pending deprecation to be replaced + by a more flexible system + + Please See https://github.com/ipython/ipython/issues/13376 """ self.javascript(line, cell) @cell_magic def javascript(self, line, cell): - """Run the cell block of Javascript code""" + """Run the cell block of Javascript code + + Starting with IPython 8.0 %%javascript is pending deprecation to be replaced + by a more flexible system + + Please See https://github.com/ipython/ipython/issues/13376 + """ display(Javascript(cell)) @cell_magic def latex(self, line, cell): - """Render the cell as a block of latex + """Render the cell as a block of LaTeX - The subset of latex which is support depends on the implementation in + The subset of LaTeX which is supported depends on the implementation in the client. In the Jupyter Notebook, this magic only renders the subset - of latex defined by MathJax + of LaTeX defined by MathJax [here](https://docs.mathjax.org/en/v2.5-latest/tex.html).""" display(Latex(cell)) diff --git a/contrib/python/ipython/py3/IPython/core/magics/execution.py b/contrib/python/ipython/py3/IPython/core/magics/execution.py index 6b651939f8..da7f780b9c 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/execution.py +++ b/contrib/python/ipython/py3/IPython/core/magics/execution.py @@ -8,55 +8,44 @@ import ast import bdb import builtins as builtin_mod +import cProfile as profile import gc import itertools +import math import os +import pstats +import re import shlex import sys import time import timeit -import math -import re +from ast import Module +from io import StringIO +from logging import error +from pathlib import Path from pdb import Restart +from warnings import warn -# cProfile was added in Python2.5 -try: - import cProfile as profile - import pstats -except ImportError: - # profile isn't bundled by default in Debian for license reasons - try: - import profile, pstats - except ImportError: - profile = pstats = None - -from IPython.core import oinspect -from IPython.core import magic_arguments -from IPython.core import page +from IPython.core import magic_arguments, oinspect, page from IPython.core.error import UsageError from IPython.core.macro import Macro -from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic, - line_cell_magic, on_off, needs_local_scope, - no_var_expand) +from IPython.core.magic import ( + Magics, + cell_magic, + line_cell_magic, + line_magic, + magics_class, + needs_local_scope, + no_var_expand, + on_off, +) from IPython.testing.skipdoctest import skip_doctest -from IPython.utils.contexts import preserve_keys from IPython.utils.capture import capture_output +from IPython.utils.contexts import preserve_keys from IPython.utils.ipstruct import Struct from IPython.utils.module_paths import find_mod from IPython.utils.path import get_py_filename, shellglob from IPython.utils.timing import clock, clock2 -from warnings import warn -from logging import error -from io import StringIO - -if sys.version_info > (3,8): - from ast import Module -else : - # mock the new API, ignore second argument - # see https://github.com/ipython/ipython/issues/11590 - from ast import Module as OriginalModule - Module = lambda nodelist, type_ignores: OriginalModule(nodelist) - #----------------------------------------------------------------------------- # Magic implementation classes @@ -103,17 +92,15 @@ class TimeitResult(object): pm = u'\xb1' except: pass - return ( - u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)" - .format( - pm = pm, - runs = self.repeat, - loops = self.loops, - loop_plural = "" if self.loops == 1 else "s", - run_plural = "" if self.repeat == 1 else "s", - mean = _format_time(self.average, self._precision), - std = _format_time(self.stdev, self._precision)) - ) + return "{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( + pm=pm, + runs=self.repeat, + loops=self.loops, + loop_plural="" if self.loops == 1 else "s", + run_plural="" if self.repeat == 1 else "s", + mean=_format_time(self.average, self._precision), + std=_format_time(self.stdev, self._precision), + ) def _repr_pretty_(self, p , cycle): unic = self.__str__() @@ -181,17 +168,9 @@ class ExecutionMagics(Magics): def __init__(self, shell): super(ExecutionMagics, self).__init__(shell) - if profile is None: - self.prun = self.profile_missing_notice # Default execution function used to actually run user code. self.default_runner = None - def profile_missing_notice(self, *args, **kwargs): - error("""\ -The profile module could not be found. It has been removed from the standard -python packages because of its non-free license. To use profiling, install the -python-profiler package from non-free.""") - @skip_doctest @no_var_expand @line_cell_magic @@ -375,18 +354,22 @@ python-profiler package from non-free.""") text_file = opts.T[0] if dump_file: prof.dump_stats(dump_file) - print('\n*** Profile stats marshalled to file',\ - repr(dump_file)+'.',sys_exit) + print( + f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}" + ) if text_file: - with open(text_file, 'w') as pfile: - pfile.write(output) - print('\n*** Profile printout saved to text file',\ - repr(text_file)+'.',sys_exit) + pfile = Path(text_file) + pfile.touch(exist_ok=True) + pfile.write_text(output, encoding="utf-8") + + print( + f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}" + ) if 'r' in opts: return stats - else: - return None + + return None @line_magic def pdb(self, parameter_s=''): @@ -423,7 +406,6 @@ python-profiler package from non-free.""") self.shell.call_pdb = new_pdb print('Automatic pdb calling has been turned',on_off(new_pdb)) - @skip_doctest @magic_arguments.magic_arguments() @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE', help=""" @@ -529,7 +511,7 @@ python-profiler package from non-free.""") """Run the named file inside IPython as a program. Usage:: - + %run [-n -i -e -G] [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )] ( -m mod | filename ) [args] @@ -570,7 +552,7 @@ python-profiler package from non-free.""") *two* back slashes (e.g. ``\\\\*``) to suppress expansions. To completely disable these expansions, you can use -G flag. - On Windows systems, the use of single quotes `'` when specifying + On Windows systems, the use of single quotes `'` when specifying a file is not supported. Use double quotes `"`. Options: @@ -712,9 +694,9 @@ python-profiler package from non-free.""") fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) - except IndexError: + except IndexError as e: msg = 'you must provide at least a filename.' - raise Exception(msg) + raise Exception(msg) from e except IOError as e: try: msg = str(e) @@ -722,7 +704,7 @@ python-profiler package from non-free.""") msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - raise Exception(msg) + raise Exception(msg) from e except TypeError: if fpath in sys.meta_path: filename = "" @@ -751,7 +733,7 @@ python-profiler package from non-free.""") sys.argv = [filename] + args # put in the proper filename if 'n' in opts: - name = os.path.splitext(os.path.basename(filename))[0] + name = Path(filename).stem else: name = '__main__' @@ -1085,7 +1067,6 @@ python-profiler package from non-free.""") In [6]: %timeit -n1 time.sleep(2) - The times reported by %timeit will be slightly higher than those reported by the timeit.py script when variables are accessed. This is due to the fact that %timeit executes the statement in the namespace @@ -1094,8 +1075,9 @@ python-profiler package from non-free.""") does not matter as long as results from timeit.py are not mixed with those from %timeit.""" - opts, stmt = self.parse_options(line,'n:r:tcp:qo', - posix=False, strict=False) + opts, stmt = self.parse_options( + line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True + ) if stmt == "" and cell is None: return @@ -1218,7 +1200,7 @@ python-profiler package from non-free.""") The CPU and wall clock times are printed, and the value of the expression (if any) is returned. Note that under Win32, system time is always reported as 0, since it can not be measured. - + This function can be used both as a line and cell magic: - In line mode you can time a single-line statement (though multiple @@ -1255,7 +1237,6 @@ python-profiler package from non-free.""") CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s Wall time: 0.00 - .. note:: The time needed by Python to compile the given expression will be reported if it is more than 0.1s. @@ -1345,19 +1326,22 @@ python-profiler package from non-free.""") wall_end = wtime() # Compute actual times and report - wall_time = wall_end-wall_st - cpu_user = end[0]-st[0] - cpu_sys = end[1]-st[1] - cpu_tot = cpu_user+cpu_sys - # On windows cpu_sys is always zero, so no new information to the next print - if sys.platform != 'win32': - print("CPU times: user %s, sys: %s, total: %s" % \ - (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot))) - print("Wall time: %s" % _format_time(wall_time)) + wall_time = wall_end - wall_st + cpu_user = end[0] - st[0] + cpu_sys = end[1] - st[1] + cpu_tot = cpu_user + cpu_sys + # On windows cpu_sys is always zero, so only total is displayed + if sys.platform != "win32": + print( + f"CPU times: user {_format_time(cpu_user)}, sys: {_format_time(cpu_sys)}, total: {_format_time(cpu_tot)}" + ) + else: + print(f"CPU times: total: {_format_time(cpu_tot)}") + print(f"Wall time: {_format_time(wall_time)}") if tc > tc_min: - print("Compiler : %s" % _format_time(tc)) + print(f"Compiler : {_format_time(tc)}") if tp > tp_min: - print("Parser : %s" % _format_time(tp)) + print(f"Parser : {_format_time(tp)}") return out @skip_doctest diff --git a/contrib/python/ipython/py3/IPython/core/magics/extension.py b/contrib/python/ipython/py3/IPython/core/magics/extension.py index ba93b3be75..2bc76b2d55 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/extension.py +++ b/contrib/python/ipython/py3/IPython/core/magics/extension.py @@ -41,7 +41,7 @@ class ExtensionMagics(Magics): @line_magic def unload_ext(self, module_str): """Unload an IPython extension by its module name. - + Not all extensions can be unloaded, only those which define an ``unload_ipython_extension`` function. """ diff --git a/contrib/python/ipython/py3/IPython/core/magics/history.py b/contrib/python/ipython/py3/IPython/core/magics/history.py index 5af09e5ce1..faa4335faa 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/history.py +++ b/contrib/python/ipython/py3/IPython/core/magics/history.py @@ -16,6 +16,7 @@ import os import sys from io import open as io_open +import fnmatch # Our own packages from IPython.core.error import StdinNotImplementedError @@ -104,7 +105,7 @@ class HistoryMagics(Magics): By default, all input history from the current session is displayed. Ranges of history can be indicated using the syntax: - + ``4`` Line 4, current session ``4-6`` @@ -116,7 +117,7 @@ class HistoryMagics(Magics): ``~8/1-~6/5`` From the first line of 8 sessions ago, to the fifth line of 6 sessions ago. - + Multiple ranges can be entered, separated by spaces The same syntax is used by %macro, %save, %edit, %rerun @@ -150,6 +151,7 @@ class HistoryMagics(Magics): # We don't want to close stdout at the end! close_at_end = False else: + outfname = os.path.expanduser(outfname) if os.path.exists(outfname): try: ans = io.ask_yes_no("File %r exists. Overwrite?" % outfname) @@ -170,7 +172,8 @@ class HistoryMagics(Magics): pattern = None limit = None if args.limit is _unspecified else args.limit - if args.pattern is not None: + range_pattern = False + if args.pattern is not None and not args.range: if args.pattern: pattern = "*" + " ".join(args.pattern) + "*" else: @@ -182,11 +185,12 @@ class HistoryMagics(Magics): n = 10 if limit is None else limit hist = history_manager.get_tail(n, raw=raw, output=get_output) else: - if args.range: # Get history by ranges - hist = history_manager.get_range_by_str(" ".join(args.range), - raw, get_output) - else: # Just get history for the current session - hist = history_manager.get_range(raw=raw, output=get_output) + if args.pattern: + range_pattern = "*" + " ".join(args.pattern) + "*" + print_nums = True + hist = history_manager.get_range_by_str( + " ".join(args.range), raw, get_output + ) # We could be displaying the entire history, so let's not try to pull # it into a list in memory. Anything that needs more space will just @@ -200,6 +204,9 @@ class HistoryMagics(Magics): # into an editor. if get_output: inline, output = inline + if range_pattern: + if not fnmatch.fnmatch(inline, range_pattern): + continue inline = inline.expandtabs(4).rstrip() multiline = "\n" in inline @@ -293,7 +300,19 @@ class HistoryMagics(Magics): """ opts, args = self.parse_options(parameter_s, 'l:g:', mode='string') if "l" in opts: # Last n lines - n = int(opts['l']) + try: + n = int(opts["l"]) + except ValueError: + print("Number of lines must be an integer") + return + + if n == 0: + print("Requested 0 last lines - nothing to run") + return + elif n < 0: + print("Number of lines to rerun cannot be negative") + return + hist = self.shell.history_manager.get_tail(n) elif "g" in opts: # Search p = "*"+opts['g']+"*" diff --git a/contrib/python/ipython/py3/IPython/core/magics/namespace.py b/contrib/python/ipython/py3/IPython/core/magics/namespace.py index 5cc2d81ca2..c86d3de9b6 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/namespace.py +++ b/contrib/python/ipython/py3/IPython/core/magics/namespace.py @@ -173,7 +173,7 @@ class NamespaceMagics(Magics): 'builtin', 'user', 'user_global','internal', 'alias', where 'builtin' and 'user' are the search defaults. Note that you should not use quotes when specifying namespaces. - + -l: List all available object types for object matching. This function can be used without arguments. @@ -203,9 +203,9 @@ class NamespaceMagics(Magics): Show objects beginning with a single _:: %psearch -a _* list objects beginning with a single underscore - + List available objects:: - + %psearch -l list all available object types """ # default namespaces to be searched @@ -252,7 +252,6 @@ class NamespaceMagics(Magics): Examples -------- - Define two variables and list them with who_ls:: In [1]: alpha = 123 @@ -367,7 +366,6 @@ class NamespaceMagics(Magics): Examples -------- - Define two variables and list them with whos:: In [1]: alpha = 123 @@ -484,24 +482,26 @@ class NamespaceMagics(Magics): Parameters ---------- - -f : force reset without asking for confirmation. - - -s : 'Soft' reset: Only clears your namespace, leaving history intact. + -f + force reset without asking for confirmation. + -s + 'Soft' reset: Only clears your namespace, leaving history intact. References to objects may be kept. By default (without this option), we do a 'hard' reset, giving you a new session and removing all references to objects from the current session. - - --aggressive: Try to aggressively remove modules from sys.modules ; this + --aggressive + Try to aggressively remove modules from sys.modules ; this may allow you to reimport Python modules that have been updated and pick up changes, but can have unattended consequences. - in : reset input history - - out : reset output history - - dhist : reset directory history - - array : reset only variables that are NumPy arrays + in + reset input history + out + reset output history + dhist + reset directory history + array + reset only variables that are NumPy arrays See Also -------- @@ -624,7 +624,6 @@ class NamespaceMagics(Magics): Examples -------- - We first fully reset the namespace so your output looks identical to this example for pedagogical reasons; in practice you do not need a full reset:: @@ -687,8 +686,8 @@ class NamespaceMagics(Magics): else: try: m = re.compile(regex) - except TypeError: - raise TypeError('regex must be a string or compiled pattern') + except TypeError as e: + raise TypeError('regex must be a string or compiled pattern') from e for i in self.who_ls(): if m.search(i): del(user_ns[i]) diff --git a/contrib/python/ipython/py3/IPython/core/magics/osm.py b/contrib/python/ipython/py3/IPython/core/magics/osm.py index 90da7e2280..41957a2850 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/osm.py +++ b/contrib/python/ipython/py3/IPython/core/magics/osm.py @@ -63,10 +63,9 @@ class OSMagics(Magics): super().__init__(shell=shell, **kwargs) - @skip_doctest def _isexec_POSIX(self, file): """ - Test for executable on a POSIX system + Test for executable on a POSIX system """ if os.access(file.path, os.X_OK): # will fail on maxOS if access is not X_OK @@ -75,17 +74,15 @@ class OSMagics(Magics): - @skip_doctest def _isexec_WIN(self, file): """ - Test for executable file on non POSIX system + Test for executable file on non POSIX system """ return file.is_file() and self.execre.match(file.name) is not None - @skip_doctest def isexec(self, file): """ - Test for executable file on non POSIX system + Test for executable file on non POSIX system """ if self.is_posix: return self._isexec_POSIX(file) @@ -130,7 +127,7 @@ class OSMagics(Magics): Aliases expand Python variables just like system calls using ! or !! do: all expressions prefixed with '$' get expanded. For details of the semantic rules, see PEP-215: - http://www.python.org/peps/pep-0215.html. This is the library used by + https://peps.python.org/pep-0215/. This is the library used by IPython for variable expansion. If you want to access a true shell variable, an extra $ is necessary to prevent its expansion by IPython:: @@ -293,8 +290,8 @@ class OSMagics(Magics): """ try: return os.getcwd() - except FileNotFoundError: - raise UsageError("CWD no longer exists - please use %cd to change directory.") + except FileNotFoundError as e: + raise UsageError("CWD no longer exists - please use %cd to change directory.") from e @skip_doctest @line_magic @@ -302,33 +299,34 @@ class OSMagics(Magics): """Change the current working directory. This command automatically maintains an internal list of directories - you visit during your IPython session, in the variable _dh. The - command %dhist shows this history nicely formatted. You can also - do 'cd -<tab>' to see directory history conveniently. - + you visit during your IPython session, in the variable ``_dh``. The + command :magic:`%dhist` shows this history nicely formatted. You can + also do ``cd -<tab>`` to see directory history conveniently. Usage: - cd 'dir': changes to directory 'dir'. - - cd -: changes to the last visited directory. - - cd -<n>: changes to the n-th directory in the directory history. - - cd --foo: change to directory that matches 'foo' in history + - ``cd 'dir'``: changes to directory 'dir'. + - ``cd -``: changes to the last visited directory. + - ``cd -<n>``: changes to the n-th directory in the directory history. + - ``cd --foo``: change to directory that matches 'foo' in history + - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark + - Hitting a tab key after ``cd -b`` allows you to tab-complete + bookmark names. - cd -b <bookmark_name>: jump to a bookmark set by %bookmark - (note: cd <bookmark_name> is enough if there is no - directory <bookmark_name>, but a bookmark with the name exists.) - 'cd -b <tab>' allows you to tab-complete bookmark names. + .. note:: + ``cd <bookmark_name>`` is enough if there is no directory + ``<bookmark_name>``, but a bookmark with the name exists. Options: - -q: quiet. Do not print the working directory after the cd command is - executed. By default IPython's cd command does print this directory, - since the default prompts do not display path information. + -q Be quiet. Do not print the working directory after the + cd command is executed. By default IPython's cd + command does print this directory, since the default + prompts do not display path information. - Note that !cd doesn't work for this purpose because the shell where - !command runs is immediately discarded after executing 'command'. + .. note:: + Note that ``!cd`` doesn't work for this purpose because the shell + where ``!command`` runs is immediately discarded after executing + 'command'. Examples -------- @@ -386,8 +384,8 @@ class OSMagics(Magics): if ps == '-': try: ps = self.shell.user_ns['_dh'][-2] - except IndexError: - raise UsageError('%cd -: No previous directory to change to.') + except IndexError as e: + raise UsageError('%cd -: No previous directory to change to.') from e # jump to bookmark if needed else: if not os.path.isdir(ps) or 'b' in opts: @@ -436,11 +434,11 @@ class OSMagics(Magics): Usage:\\ - %env: lists all environment variables/values - %env var: get value for var - %env var val: set value for var - %env var=val: set value for var - %env var=$val: set value for var, using python expansion if possible + :``%env``: lists all environment variables/values + :``%env var``: get value for var + :``%env var val``: set value for var + :``%env var=val``: set value for var + :``%env var=$val``: set value for var, using python expansion if possible """ if parameter_s.strip(): split = '=' if '=' in parameter_s else ' ' @@ -506,7 +504,7 @@ class OSMagics(Magics): if tgt: self.cd(parameter_s) dir_s.insert(0,cwd) - return self.shell.magic('dirs') + return self.shell.run_line_magic('dirs', '') @line_magic def popd(self, parameter_s=''): @@ -630,8 +628,8 @@ class OSMagics(Magics): # while the list form is useful to loop over: In [6]: for f in a.l: - ...: !wc -l $f - ...: + ...: !wc -l $f + ...: 146 setup.py 130 win32_manual_post_install.py @@ -764,15 +762,15 @@ class OSMagics(Magics): if 'd' in opts: try: todel = args[0] - except IndexError: + except IndexError as e: raise UsageError( - "%bookmark -d: must provide a bookmark to delete") + "%bookmark -d: must provide a bookmark to delete") from e else: try: del bkms[todel] - except KeyError: + except KeyError as e: raise UsageError( - "%%bookmark -d: Can't delete bookmark '%s'" % todel) + "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e elif 'r' in opts: bkms = {} @@ -803,18 +801,17 @@ class OSMagics(Magics): to be Python source and will show it with syntax highlighting. This magic command can either take a local filename, an url, - an history range (see %history) or a macro as argument :: + an history range (see %history) or a macro as argument. + + If no parameter is given, prints out history of current session up to + this point. :: %pycat myscript.py %pycat 7-27 %pycat myMacro %pycat http://www.example.com/myscript.py """ - if not parameter_s: - raise UsageError('Missing filename, URL, input history range, ' - 'or macro.') - - try : + try: cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False) except (ValueError, IOError): print("Error: no such file, variable, URL, history range or macro") @@ -835,7 +832,7 @@ class OSMagics(Magics): @cell_magic def writefile(self, line, cell): """Write the contents of the cell to a file. - + The file will be overwritten unless the -a (--append) flag is specified. """ args = magic_arguments.parse_argstring(self.writefile, line) diff --git a/contrib/python/ipython/py3/IPython/core/magics/packaging.py b/contrib/python/ipython/py3/IPython/core/magics/packaging.py index 04bde051ae..2f7652c169 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/packaging.py +++ b/contrib/python/ipython/py3/IPython/core/magics/packaging.py @@ -8,10 +8,10 @@ # The full license is in the file COPYING.txt, distributed with this software. #----------------------------------------------------------------------------- -import os import re import shlex import sys +from pathlib import Path from IPython.core.magic import Magics, magics_class, line_magic @@ -19,27 +19,28 @@ from IPython.core.magic import Magics, magics_class, line_magic def _is_conda_environment(): """Return True if the current Python executable is in a conda env""" # TODO: does this need to change on windows? - conda_history = os.path.join(sys.prefix, 'conda-meta', 'history') - return os.path.exists(conda_history) + return Path(sys.prefix, "conda-meta", "history").exists() def _get_conda_executable(): """Find the path to the conda executable""" # Check if there is a conda executable in the same directory as the Python executable. # This is the case within conda's root environment. - conda = os.path.join(os.path.dirname(sys.executable), 'conda') - if os.path.isfile(conda): - return conda + conda = Path(sys.executable).parent / "conda" + if conda.is_file(): + return str(conda) # Otherwise, attempt to extract the executable from conda history. # This applies in any conda environment. - R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]") - with open(os.path.join(sys.prefix, 'conda-meta', 'history')) as f: - for line in f: - match = R.match(line) - if match: - return match.groupdict()['command'] - + history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8") + match = re.search( + r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]", + history, + flags=re.MULTILINE, + ) + if match: + return match.groupdict()["command"] + # Fallback: assume conda is available on the system path. return "conda" @@ -78,18 +79,19 @@ class PackagingMagics(Magics): @line_magic def conda(self, line): """Run the conda package manager within the current kernel. - + Usage: %conda install [pkgs] """ if not _is_conda_environment(): raise ValueError("The python kernel does not appear to be a conda environment. " "Please use ``%pip install`` instead.") - + conda = _get_conda_executable() args = shlex.split(line) - command = args[0] - args = args[1:] + command = args[0] if len(args) > 0 else "" + args = args[1:] if len(args) > 1 else [""] + extra_args = [] # When the subprocess does not allow us to respond "yes" during the installation, diff --git a/contrib/python/ipython/py3/IPython/core/magics/pylab.py b/contrib/python/ipython/py3/IPython/core/magics/pylab.py index 9ec441a3e2..0f3fff62fa 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/pylab.py +++ b/contrib/python/ipython/py3/IPython/core/magics/pylab.py @@ -154,6 +154,9 @@ class PylabMagics(Magics): gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all) self._show_matplotlib_backend(args.gui, backend) + print( + "%pylab is deprecated, use %matplotlib inline and import the required libraries." + ) print("Populating the interactive namespace from numpy and matplotlib") if clobbered: warn("pylab import has clobbered these variables: %s" % clobbered + diff --git a/contrib/python/ipython/py3/IPython/core/magics/script.py b/contrib/python/ipython/py3/IPython/core/magics/script.py index 8b7f6f94e0..9fd2fc6c0d 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/script.py +++ b/contrib/python/ipython/py3/IPython/core/magics/script.py @@ -3,22 +3,22 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio +import atexit import errno import os -import sys import signal +import sys import time -from subprocess import Popen, PIPE, CalledProcessError -import atexit +from subprocess import CalledProcessError +from threading import Thread + +from traitlets import Any, Dict, List, default from IPython.core import magic_arguments -from IPython.core.magic import ( - Magics, magics_class, line_magic, cell_magic -) -from IPython.lib.backgroundjobs import BackgroundJobManager -from IPython.utils import py3compat +from IPython.core.async_helpers import _AsyncIOProxy +from IPython.core.magic import Magics, cell_magic, line_magic, magics_class from IPython.utils.process import arg_split -from traitlets import List, Dict, default #----------------------------------------------------------------------------- # Magic implementation classes @@ -56,15 +56,16 @@ def script_args(f): ), magic_arguments.argument( '--no-raise-error', action="store_false", dest='raise_error', - help="""Whether you should raise an error message in addition to + help="""Whether you should raise an error message in addition to a stream on stderr if you get a nonzero exit code. - """ - ) + """, + ), ] for arg in args: f = arg(f) return f + @magics_class class ScriptMagics(Magics): """Magics for talking to scripts @@ -73,6 +74,17 @@ class ScriptMagics(Magics): with a program in a subprocess, and registers a few top-level magics that call %%script with common interpreters. """ + + event_loop = Any( + help=""" + The event loop on which to run subprocesses + + Not the main event loop, + because we want to be able to make blocking calls + and have certain requirements we don't want to impose on the main loop. + """ + ) + script_magics = List( help="""Extra script cell magics to define @@ -114,7 +126,6 @@ class ScriptMagics(Magics): def __init__(self, shell=None): super(ScriptMagics, self).__init__(shell=shell) self._generate_script_magics() - self.job_manager = BackgroundJobManager() self.bg_processes = [] atexit.register(self.kill_bg_processes) @@ -136,7 +147,7 @@ class ScriptMagics(Magics): def named_script_magic(line, cell): # if line, add it as cl-flags if line: - line = "%s %s" % (script, line) + line = "%s %s" % (script, line) else: line = script return self.shebang(line, cell) @@ -157,16 +168,16 @@ class ScriptMagics(Magics): @cell_magic("script") def shebang(self, line, cell): """Run a cell via a shell command - + The `%%script` line is like the #! line of script, specifying a program (bash, perl, ruby, etc.) with which to run. - + The rest of the cell is run by that program. - + Examples -------- :: - + In [1]: %%script bash ...: for i in 1 2 3; do ...: echo $i @@ -175,18 +186,70 @@ class ScriptMagics(Magics): 2 3 """ - argv = arg_split(line, posix = not sys.platform.startswith('win')) + + # Create the event loop in which to run script magics + # this operates on a background thread + if self.event_loop is None: + if sys.platform == "win32": + # don't override the current policy, + # just create an event loop + event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop() + else: + event_loop = asyncio.new_event_loop() + self.event_loop = event_loop + + # start the loop in a background thread + asyncio_thread = Thread(target=event_loop.run_forever, daemon=True) + asyncio_thread.start() + else: + event_loop = self.event_loop + + def in_thread(coro): + """Call a coroutine on the asyncio thread""" + return asyncio.run_coroutine_threadsafe(coro, event_loop).result() + + async def _handle_stream(stream, stream_arg, file_object): + while True: + line = (await stream.readline()).decode("utf8") + if not line: + break + if stream_arg: + self.shell.user_ns[stream_arg] = line + else: + file_object.write(line) + file_object.flush() + + async def _stream_communicate(process, cell): + process.stdin.write(cell) + process.stdin.close() + stdout_task = asyncio.create_task( + _handle_stream(process.stdout, args.out, sys.stdout) + ) + stderr_task = asyncio.create_task( + _handle_stream(process.stderr, args.err, sys.stderr) + ) + await asyncio.wait([stdout_task, stderr_task]) + await process.wait() + + argv = arg_split(line, posix=not sys.platform.startswith("win")) args, cmd = self.shebang.parser.parse_known_args(argv) - + try: - p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) + p = in_thread( + asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE, + ) + ) except OSError as e: if e.errno == errno.ENOENT: print("Couldn't find program: %r" % cmd[0]) return else: raise - + if not cell.endswith('\n'): cell += '\n' cell = cell.encode('utf8', 'replace') @@ -195,30 +258,35 @@ class ScriptMagics(Magics): self._gc_bg_processes() to_close = [] if args.out: - self.shell.user_ns[args.out] = p.stdout + self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop) else: to_close.append(p.stdout) if args.err: - self.shell.user_ns[args.err] = p.stderr + self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop) else: to_close.append(p.stderr) - self.job_manager.new(self._run_script, p, cell, to_close, daemon=True) + event_loop.call_soon_threadsafe( + lambda: asyncio.Task(self._run_script(p, cell, to_close)) + ) if args.proc: - self.shell.user_ns[args.proc] = p + proc_proxy = _AsyncIOProxy(p, event_loop) + proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop) + proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop) + self.shell.user_ns[args.proc] = proc_proxy return - + try: - out, err = p.communicate(cell) + in_thread(_stream_communicate(p, cell)) except KeyboardInterrupt: try: p.send_signal(signal.SIGINT) - time.sleep(0.1) - if p.poll() is not None: + in_thread(asyncio.wait_for(p.wait(), timeout=0.1)) + if p.returncode is not None: print("Process is interrupted.") return p.terminate() - time.sleep(0.1) - if p.poll() is not None: + in_thread(asyncio.wait_for(p.wait(), timeout=0.1)) + if p.returncode is not None: print("Process is terminated.") return p.kill() @@ -226,31 +294,31 @@ class ScriptMagics(Magics): except OSError: pass except Exception as e: - print("Error while terminating subprocess (pid=%i): %s" \ - % (p.pid, e)) + print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e)) return - out = py3compat.decode(out) - err = py3compat.decode(err) - if args.out: - self.shell.user_ns[args.out] = out - else: - sys.stdout.write(out) - sys.stdout.flush() - if args.err: - self.shell.user_ns[args.err] = err - else: - sys.stderr.write(err) - sys.stderr.flush() - if args.raise_error and p.returncode!=0: - raise CalledProcessError(p.returncode, cell, output=out, stderr=err) - - def _run_script(self, p, cell, to_close): + + if args.raise_error and p.returncode != 0: + # If we get here and p.returncode is still None, we must have + # killed it but not yet seen its return code. We don't wait for it, + # in case it's stuck in uninterruptible sleep. -9 = SIGKILL + rc = p.returncode or -9 + raise CalledProcessError(rc, cell) + + shebang.__skip_doctest__ = os.name != "posix" + + async def _run_script(self, p, cell, to_close): """callback for running the script in the background""" + p.stdin.write(cell) + await p.stdin.drain() p.stdin.close() + await p.stdin.wait_closed() + await p.wait() + # asyncio read pipes have no close + # but we should drain the data anyway for s in to_close: - s.close() - p.wait() + await s.read() + self._gc_bg_processes() @line_magic("killbgscripts") def killbgscripts(self, _nouse_=''): @@ -263,7 +331,7 @@ class ScriptMagics(Magics): if not self.bg_processes: return for p in self.bg_processes: - if p.poll() is None: + if p.returncode is None: try: p.send_signal(signal.SIGINT) except: @@ -273,7 +341,7 @@ class ScriptMagics(Magics): if not self.bg_processes: return for p in self.bg_processes: - if p.poll() is None: + if p.returncode is None: try: p.terminate() except: @@ -283,7 +351,7 @@ class ScriptMagics(Magics): if not self.bg_processes: return for p in self.bg_processes: - if p.poll() is None: + if p.returncode is None: try: p.kill() except: @@ -291,4 +359,4 @@ class ScriptMagics(Magics): self._gc_bg_processes() def _gc_bg_processes(self): - self.bg_processes = [p for p in self.bg_processes if p.poll() is None] + self.bg_processes = [p for p in self.bg_processes if p.returncode is None] diff --git a/contrib/python/ipython/py3/IPython/core/oinspect.py b/contrib/python/ipython/py3/IPython/core/oinspect.py index 272916c966..1a5c0ae070 100644 --- a/contrib/python/ipython/py3/IPython/core/oinspect.py +++ b/contrib/python/ipython/py3/IPython/core/oinspect.py @@ -222,7 +222,7 @@ def format_argspec(argspec): This takes a dict instead of ordered arguments and calls inspect.format_argspec with the arguments in the necessary order. - DEPRECATED: Do not use; will be removed in future versions. + DEPRECATED (since 7.10): Do not use; will be removed in future versions. """ warnings.warn('`format_argspec` function is deprecated as of IPython 7.10' @@ -234,10 +234,13 @@ def format_argspec(argspec): @undoc def call_tip(oinfo, format_call=True): - """DEPRECATED. Extract call tip data from an oinfo dict. - """ - warnings.warn('`call_tip` function is deprecated as of IPython 6.0' - 'and will be removed in future versions.', DeprecationWarning, stacklevel=2) + """DEPRECATED since 6.0. Extract call tip data from an oinfo dict.""" + warnings.warn( + "`call_tip` function is deprecated as of IPython 6.0" + "and will be removed in future versions.", + DeprecationWarning, + stacklevel=2, + ) # Get call definition argspec = oinfo.get('argspec') if argspec is None: @@ -299,7 +302,7 @@ def find_file(obj) -> str: Returns ------- fname : str - The absolute path to the file where the object was defined. + The absolute path to the file where the object was defined. """ obj = _get_wrapped(obj) @@ -334,7 +337,7 @@ def find_source_lines(obj): Returns ------- lineno : int - The line number where the object definition starts. + The line number where the object definition starts. """ obj = _get_wrapped(obj) @@ -425,7 +428,6 @@ class Inspector(Colorable): Examples -------- - In [1]: class NoInit: ...: pass @@ -516,12 +518,12 @@ class Inspector(Colorable): """Return a mime bundle representation of the input text. - if `formatter` is None, the returned mime bundle has - a `text/plain` field, with the input text. - a `text/html` field with a `<pre>` tag containing the input text. + a ``text/plain`` field, with the input text. + a ``text/html`` field with a ``<pre>`` tag containing the input text. - - if `formatter` is not None, it must be a callable transforming the - input text into a mime bundle. Default values for `text/plain` and - `text/html` representations are the ones described above. + - if ``formatter`` is not None, it must be a callable transforming the + input text into a mime bundle. Default values for ``text/plain`` and + ``text/html`` representations are the ones described above. Note: @@ -566,24 +568,27 @@ class Inspector(Colorable): bundle['text/plain'] = text return bundle - def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0): + def _get_info( + self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + ): """Retrieve an info dict and format it. Parameters - ========== - - obj: any + ---------- + obj : any Object to inspect and return info from - oname: str (default: ''): + oname : str (default: ''): Name of the variable pointing to `obj`. - formatter: callable - info: + formatter : callable + info already computed information - detail_level: integer + detail_level : integer Granularity of detail level, if set to 1, give more information. + omit_sections : container[str] + Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) """ - info = self._info(obj, oname=oname, info=info, detail_level=detail_level) + info = self.info(obj, oname=oname, info=info, detail_level=detail_level) _mime = { 'text/plain': [], @@ -591,6 +596,8 @@ class Inspector(Colorable): } def append_field(bundle, title:str, key:str, formatter=None): + if title in omit_sections or key in omit_sections: + return field = info[key] if field is not None: formatted_field = self._mime_format(field, formatter) @@ -655,7 +662,16 @@ class Inspector(Colorable): return self.format_mime(_mime) - def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0, enable_html_pager=True): + def pinfo( + self, + obj, + oname="", + formatter=None, + info=None, + detail_level=0, + enable_html_pager=True, + omit_sections=(), + ): """Show detailed information about an object. Optional arguments: @@ -676,40 +692,48 @@ class Inspector(Colorable): precomputed already. - detail_level: if set to 1, more information is given. + + - omit_sections: set of section keys and titles to omit """ - info = self._get_info(obj, oname, formatter, info, detail_level) + info = self._get_info( + obj, oname, formatter, info, detail_level, omit_sections=omit_sections + ) if not enable_html_pager: del info['text/html'] page.page(info) - def info(self, obj, oname='', formatter=None, info=None, detail_level=0): - """DEPRECATED. Compute a dict with detailed information about an object. + def _info(self, obj, oname="", info=None, detail_level=0): """ - if formatter is not None: - warnings.warn('The `formatter` keyword argument to `Inspector.info`' - 'is deprecated as of IPython 5.0 and will have no effects.', - DeprecationWarning, stacklevel=2) - return self._info(obj, oname=oname, info=info, detail_level=detail_level) + Inspector.info() was likely improperly marked as deprecated + while only a parameter was deprecated. We "un-deprecate" it. + """ + + warnings.warn( + "The `Inspector.info()` method has been un-deprecated as of 8.0 " + "and the `formatter=` keyword removed. `Inspector._info` is now " + "an alias, and you can just call `.info()` directly.", + DeprecationWarning, + stacklevel=2, + ) + return self.info(obj, oname=oname, info=info, detail_level=detail_level) - def _info(self, obj, oname='', info=None, detail_level=0) -> dict: + def info(self, obj, oname="", info=None, detail_level=0) -> dict: """Compute a dict with detailed information about an object. Parameters - ========== - - obj: any + ---------- + obj : any An object to find information about - oname: str (default: ''): + oname : str (default: '') Name of the variable pointing to `obj`. - info: (default: None) + info : (default: None) A struct (dict like with attr access) with some information fields which may have been precomputed already. - detail_level: int (default:0) + detail_level : int (default:0) If set to 1, more information is given. Returns - ======= - + ------- An object info dict with known fields from `info_fields`. Keys are strings, values are string or None. """ @@ -941,7 +965,7 @@ class Inspector(Colorable): - show_all(False): show all names, including those starting with underscores. - + - list_types(False): list all available object types for object matching. """ #print 'ps pattern:<%r>' % pattern # dbg diff --git a/contrib/python/ipython/py3/IPython/core/page.py b/contrib/python/ipython/py3/IPython/core/page.py index ed16b61781..d3e6a9eef5 100644 --- a/contrib/python/ipython/py3/IPython/core/page.py +++ b/contrib/python/ipython/py3/IPython/core/page.py @@ -22,9 +22,10 @@ import tempfile import subprocess from io import UnsupportedOperation +from pathlib import Path from IPython import get_ipython -from IPython.core.display import display +from IPython.display import display from IPython.core.error import TryNext from IPython.utils.data import chop from IPython.utils.process import system @@ -45,7 +46,7 @@ def display_page(strng, start=0, screen_lines=25): def as_hook(page_func): """Wrap a pager func to strip the `self` arg - + so it can be called as a hook. """ return lambda self, *args, **kwargs: page_func(*args, **kwargs) @@ -106,7 +107,7 @@ def _detect_screen_size(screen_lines_def): term_flags = termios.tcgetattr(sys.stdout) except termios.error as err: # can fail on Linux 2.6, pager_page will catch the TypeError - raise TypeError('termios error: {0}'.format(err)) + raise TypeError('termios error: {0}'.format(err)) from err try: scr = curses.initscr() @@ -126,7 +127,7 @@ def _detect_screen_size(screen_lines_def): def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): """Display a string, piping through a pager after a certain length. - + strng can be a mime-bundle dict, supplying multiple representations, keyed by mime-type. @@ -195,28 +196,32 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): retval = 1 else: fd, tmpname = tempfile.mkstemp('.txt') + tmppath = Path(tmpname) try: os.close(fd) - with open(tmpname, 'wt') as tmpfile: + with tmppath.open("wt", encoding="utf-8") as tmpfile: tmpfile.write(strng) - cmd = "%s < %s" % (pager_cmd, tmpname) + cmd = "%s < %s" % (pager_cmd, tmppath) # tmpfile needs to be closed for windows if os.system(cmd): retval = 1 else: retval = None finally: - os.remove(tmpname) + Path.unlink(tmppath) else: try: retval = None # Emulate os.popen, but redirect stderr - proc = subprocess.Popen(pager_cmd, - shell=True, - stdin=subprocess.PIPE, - stderr=subprocess.DEVNULL - ) - pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc) + proc = subprocess.Popen( + pager_cmd, + shell=True, + stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + pager = os._wrap_close( + io.TextIOWrapper(proc.stdin, encoding="utf-8"), proc + ) try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) @@ -236,10 +241,10 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): def page(data, start=0, screen_lines=0, pager_cmd=None): """Display content in a pager, piping through a pager after a certain length. - + data can be a mime-bundle dict, supplying multiple representations, keyed by mime-type, or text. - + Pager is dispatched via the `show_in_pager` IPython hook. If no hook is registered, `pager_page` will be used. """ @@ -275,7 +280,7 @@ def page_file(fname, start=0, pager_cmd=None): try: if start > 0: start -= 1 - page(open(fname).read(),start) + page(open(fname, encoding="utf-8").read(), start) except: print('Unable to show file',repr(fname)) diff --git a/contrib/python/ipython/py3/IPython/core/payloadpage.py b/contrib/python/ipython/py3/IPython/core/payloadpage.py index eb613445dd..4958108076 100644 --- a/contrib/python/ipython/py3/IPython/core/payloadpage.py +++ b/contrib/python/ipython/py3/IPython/core/payloadpage.py @@ -17,10 +17,9 @@ def page(strng, start=0, screen_lines=0, pager_cmd=None): Parameters ---------- strng : str or mime-dict - Text to page, or a mime-type keyed dict of already formatted data. - + Text to page, or a mime-type keyed dict of already formatted data. start : int - Starting line at which to place the display. + Starting line at which to place the display. """ # Some routines may auto-compute start offsets incorrectly and pass a @@ -42,7 +41,7 @@ def page(strng, start=0, screen_lines=0, pager_cmd=None): def install_payload_page(): """DEPRECATED, use show_in_pager hook - + Install this version of page as IPython.core.page.page. """ warnings.warn("""install_payload_page is deprecated. diff --git a/contrib/python/ipython/py3/IPython/core/prefilter.py b/contrib/python/ipython/py3/IPython/core/prefilter.py index bf801f999c..0038e5c673 100644 --- a/contrib/python/ipython/py3/IPython/core/prefilter.py +++ b/contrib/python/ipython/py3/IPython/core/prefilter.py @@ -120,7 +120,7 @@ class PrefilterManager(Configurable): def __init__(self, shell=None, **kwargs): super(PrefilterManager, self).__init__(shell=shell, **kwargs) self.shell = shell - self.init_transformers() + self._transformers = [] self.init_handlers() self.init_checkers() @@ -128,14 +128,6 @@ class PrefilterManager(Configurable): # API for managing transformers #------------------------------------------------------------------------- - def init_transformers(self): - """Create the default transformers.""" - self._transformers = [] - for transformer_cls in _default_transformers: - transformer_cls( - shell=self.shell, prefilter_manager=self, parent=self - ) - def sort_transformers(self): """Sort the transformers by priority. @@ -687,9 +679,6 @@ class EmacsHandler(PrefilterHandler): #----------------------------------------------------------------------------- -_default_transformers = [ -] - _default_checkers = [ EmacsChecker, MacroChecker, diff --git a/contrib/python/ipython/py3/IPython/core/profiledir.py b/contrib/python/ipython/py3/IPython/core/profiledir.py index ba8f82b7d9..cb4d39339a 100644 --- a/contrib/python/ipython/py3/IPython/core/profiledir.py +++ b/contrib/python/ipython/py3/IPython/core/profiledir.py @@ -7,6 +7,7 @@ import os import shutil import errno +from pathlib import Path from traitlets.config.configurable import LoggingConfigurable from ..paths import get_ipython_package_dir @@ -131,19 +132,20 @@ class ProfileDir(LoggingConfigurable): self.check_pid_dir() self.check_startup_dir() - def copy_config_file(self, config_file, path=None, overwrite=False): + def copy_config_file(self, config_file: str, path: Path, overwrite=False) -> bool: """Copy a default config file into the active profile directory. Default configuration files are kept in :mod:`IPython.core.profile`. This function moves these from that location to the working profile directory. """ - dst = os.path.join(self.location, config_file) - if os.path.isfile(dst) and not overwrite: + dst = Path(os.path.join(self.location, config_file)) + if dst.exists() and not overwrite: return False if path is None: path = os.path.join(get_ipython_package_dir(), u'core', u'profile', u'default') - src = os.path.join(path, config_file) + assert isinstance(path, Path) + src = path / config_file shutil.copy(src, dst) return True diff --git a/contrib/python/ipython/py3/IPython/core/pylabtools.py b/contrib/python/ipython/py3/IPython/core/pylabtools.py index c9c8e14aa2..68e100f7d0 100644 --- a/contrib/python/ipython/py3/IPython/core/pylabtools.py +++ b/contrib/python/ipython/py3/IPython/core/pylabtools.py @@ -185,8 +185,8 @@ def mpl_runner(safe_execfile): Parameters ---------- safe_execfile : function - This must be a function with the same interface as the - :meth:`safe_execfile` method of IPython. + This must be a function with the same interface as the + :meth:`safe_execfile` method of IPython. Returns ------- @@ -231,8 +231,8 @@ def _reshow_nbagg_figure(fig): """reshow an nbagg figure""" try: reshow = fig.canvas.manager.reshow - except AttributeError: - raise NotImplementedError() + except AttributeError as e: + raise NotImplementedError() from e else: reshow() @@ -241,7 +241,7 @@ def select_figure_formats(shell, formats, **kwargs): """Select figure formats for the inline backend. Parameters - ========== + ---------- shell : InteractiveShell The main IPython instance. formats : str or set @@ -391,7 +391,7 @@ def import_pylab(user_ns, import_all=True): # IPython symbols to add user_ns['figsize'] = figsize - from IPython.core.display import display + from IPython.display import display # Add display and getfigs to the user's namespace user_ns['display'] = display user_ns['getfigs'] = getfigs @@ -408,7 +408,6 @@ def configure_inline_support(shell, backend): Parameters ---------- shell : InteractiveShell instance - backend : matplotlib backend """ warnings.warn( @@ -418,6 +417,8 @@ def configure_inline_support(shell, backend): stacklevel=2, ) - from matplotlib_inline.backend_inline import configure_inline_support as configure_inline_support_orig + from matplotlib_inline.backend_inline import ( + configure_inline_support as configure_inline_support_orig, + ) configure_inline_support_orig(shell, backend) diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py index 879d8cc4f8..4e8c82cdb7 100644 --- a/contrib/python/ipython/py3/IPython/core/release.py +++ b/contrib/python/ipython/py3/IPython/core/release.py @@ -12,18 +12,14 @@ # The full license is in the file COPYING.txt, distributed with this software. #----------------------------------------------------------------------------- -# Name of the package for release purposes. This is the name which labels -# the tarballs and RPMs made by distutils, so it's best to lowercase it. -name = 'ipython' - # IPython version information. An empty _version_extra corresponds to a full # release. 'dev' as a _version_extra string means this is a development # version -_version_major = 7 -_version_minor = 33 +_version_major = 8 +_version_minor = 3 _version_patch = 0 -_version_extra = '.dev' -# _version_extra = 'b1' +_version_extra = ".dev" +# _version_extra = "rc1" _version_extra = "" # Uncomment this for full releases # Construct full version string from these. @@ -40,49 +36,6 @@ version_info = (_version_major, _version_minor, _version_patch, _version_extra) kernel_protocol_version_info = (5, 0) kernel_protocol_version = "%i.%i" % kernel_protocol_version_info -description = "IPython: Productive Interactive Computing" - -long_description = \ -""" -IPython provides a rich toolkit to help you make the most out of using Python -interactively. Its main components are: - -* A powerful interactive Python shell -* A `Jupyter <https://jupyter.org/>`_ kernel to work with Python code in Jupyter - notebooks and other interactive frontends. - -The enhanced interactive Python shells have the following main features: - -* Comprehensive object introspection. - -* Input history, persistent across sessions. - -* Caching of output results during a session with automatically generated - references. - -* Extensible tab completion, with support by default for completion of python - variables and keywords, filenames and function keywords. - -* Extensible system of 'magic' commands for controlling the environment and - performing many tasks related either to IPython or the operating system. - -* A rich configuration system with easy switching between different setups - (simpler than changing $PYTHONSTARTUP environment variables every time). - -* Session logging and reloading. - -* Extensible syntax processing for special purpose situations. - -* Access to the system shell with user-extensible alias system. - -* Easily embeddable in other Python programs and GUIs. - -* Integrated access to the pdb debugger and the Python profiler. - -The latest development version is always available from IPython's `GitHub -site <http://github.com/ipython>`_. -""" - license = 'BSD' authors = {'Fernando' : ('Fernando Perez','fperez.net@gmail.com'), @@ -99,21 +52,3 @@ authors = {'Fernando' : ('Fernando Perez','fperez.net@gmail.com'), author = 'The IPython Development Team' author_email = 'ipython-dev@python.org' - -url = 'https://ipython.org' - - -platforms = ['Linux','Mac OSX','Windows'] - -keywords = ['Interactive','Interpreter','Shell', 'Embedding'] - -classifiers = [ - 'Framework :: IPython', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Topic :: System :: Shells' - ] diff --git a/contrib/python/ipython/py3/IPython/core/shellapp.py b/contrib/python/ipython/py3/IPython/core/shellapp.py index c442658ae7..f737bcb56b 100644 --- a/contrib/python/ipython/py3/IPython/core/shellapp.py +++ b/contrib/python/ipython/py3/IPython/core/shellapp.py @@ -98,11 +98,6 @@ shell_aliases = dict( ) shell_aliases['cache-size'] = 'InteractiveShell.cache_size' -if traitlets.version_info < (5, 0): - # traitlets 4 doesn't handle lists on CLI - shell_aliases["ext"] = "InteractiveShellApp.extra_extension" - - #----------------------------------------------------------------------------- # Main classes and functions #----------------------------------------------------------------------------- @@ -126,17 +121,6 @@ class InteractiveShellApp(Configurable): help="A list of dotted module names of IPython extensions to load." ).tag(config=True) - extra_extension = Unicode( - "", - help=""" - DEPRECATED. Dotted module name of a single extra IPython extension to load. - - Only one extension can be added this way. - - Only used with traitlets < 5.0, plural extra_extensions list is used in traitlets 5. - """, - ).tag(config=True) - extra_extensions = List( DottedObjectName(), help=""" @@ -293,8 +277,6 @@ class InteractiveShellApp(Configurable): extensions = ( self.default_extensions + self.extensions + self.extra_extensions ) - if self.extra_extension: - extensions.append(self.extra_extension) for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) diff --git a/contrib/python/ipython/py3/IPython/core/ultratb.py b/contrib/python/ipython/py3/IPython/core/ultratb.py index de85a1f8ea..4447080f35 100644 --- a/contrib/python/ipython/py3/IPython/core/ultratb.py +++ b/contrib/python/ipython/py3/IPython/core/ultratb.py @@ -89,39 +89,28 @@ Inheritance diagram: #***************************************************************************** -import dis import inspect -import keyword import linecache -import os import pydoc -import re import sys import time -import tokenize import traceback +from types import TracebackType +from typing import Tuple, List, Any, Optional -from tokenize import generate_tokens - -# For purposes of monkeypatching inspect to fix a bug in it. -from inspect import getsourcefile, getfile, getmodule, \ - ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode +import stack_data +from pygments.formatters.terminal256 import Terminal256Formatter +from pygments.styles import get_style_by_name # IPython's own modules from IPython import get_ipython from IPython.core import debugger from IPython.core.display_trap import DisplayTrap from IPython.core.excolors import exception_colors -from IPython.utils import PyColorize from IPython.utils import path as util_path from IPython.utils import py3compat -from IPython.utils.data import uniq_stable from IPython.utils.terminal import get_terminal_size -from logging import info, error, debug - -from importlib.util import source_from_cache - import IPython.utils.colorable as colorable # Globals @@ -134,276 +123,81 @@ INDENT_SIZE = 8 # to users of ultratb who are NOT running inside ipython. DEFAULT_SCHEME = 'NoColor' - -# Number of frame above which we are likely to have a recursion and will -# **attempt** to detect it. Made modifiable mostly to speedup test suite -# as detecting recursion is one of our slowest test -_FRAME_RECURSION_LIMIT = 500 - # --------------------------------------------------------------------------- # Code begins -# Utility functions -def inspect_error(): - """Print a message about internal inspect errors. - - These are unfortunately quite common.""" - - error('Internal Python error in the inspect module.\n' - 'Below is the traceback from this internal error.\n') - - -# This function is a monkeypatch we apply to the Python inspect module. We have -# now found when it's needed (see discussion on issue gh-1456), and we have a -# test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if -# the monkeypatch is not applied. TK, Aug 2012. -def findsource(object): - """Return the entire source file and starting line number for an object. - - The argument may be a module, class, method, function, traceback, frame, - or code object. The source code is returned as a list of all the lines - in the file and the line number indexes a line in that list. An IOError - is raised if the source code cannot be retrieved. - - FIXED version with which we monkeypatch the stdlib to work around a bug.""" - - file = getsourcefile(object) or getfile(object) - # If the object is a frame, then trying to get the globals dict from its - # module won't work. Instead, the frame object itself has the globals - # dictionary. - globals_dict = None - if inspect.isframe(object): - # XXX: can this ever be false? - globals_dict = object.f_globals - else: - module = getmodule(object, file) - if module: - globals_dict = module.__dict__ - lines = linecache.getlines(file, globals_dict) - if not lines: - raise IOError('could not get source code') - - if ismodule(object): - return lines, 0 - - if isclass(object): - name = object.__name__ - pat = re.compile(r'^(\s*)class\s*' + name + r'\b') - # make some effort to find the best matching class definition: - # use the one with the least indentation, which is the one - # that's most probably not inside a function definition. - candidates = [] - for i, line in enumerate(lines): - match = pat.match(line) - if match: - # if it's at toplevel, it's already the best one - if line[0] == 'c': - return lines, i - # else add whitespace to candidate list - candidates.append((match.group(1), i)) - if candidates: - # this will sort by whitespace, and by line number, - # less whitespace first - candidates.sort() - return lines, candidates[0][1] - else: - raise IOError('could not find class definition') - - if ismethod(object): - object = object.__func__ - if isfunction(object): - object = object.__code__ - if istraceback(object): - object = object.tb_frame - if isframe(object): - object = object.f_code - if iscode(object): - if not hasattr(object, 'co_firstlineno'): - raise IOError('could not find function definition') - pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)') - pmatch = pat.match - # fperez - fix: sometimes, co_firstlineno can give a number larger than - # the length of lines, which causes an error. Safeguard against that. - lnum = min(object.co_firstlineno, len(lines)) - 1 - while lnum > 0: - if pmatch(lines[lnum]): - break - lnum -= 1 - - return lines, lnum - raise IOError('could not find code object') - - -# Monkeypatch inspect to apply our bugfix. -def with_patch_inspect(f): - """ - Deprecated since IPython 6.0 - decorator for monkeypatching inspect.findsource - """ - - def wrapped(*args, **kwargs): - save_findsource = inspect.findsource - inspect.findsource = findsource - try: - return f(*args, **kwargs) - finally: - inspect.findsource = save_findsource - - return wrapped - - -def fix_frame_records_filenames(records): - """Try to fix the filenames in each record from inspect.getinnerframes(). - - Particularly, modules loaded from within zip files have useless filenames - attached to their code object, and inspect.getinnerframes() just uses it. - """ - fixed_records = [] - for frame, filename, line_no, func_name, lines, index in records: - # Look inside the frame's globals dictionary for __file__, - # which should be better. However, keep Cython filenames since - # we prefer the source filenames over the compiled .so file. - if not filename.endswith(('.pyx', '.pxd', '.pxi')): - better_fn = frame.f_globals.get('__file__', None) - if isinstance(better_fn, str): - # Check the type just in case someone did something weird with - # __file__. It might also be None if the error occurred during - # import. - filename = better_fn - fixed_records.append((frame, filename, line_no, func_name, lines, index)) - return fixed_records - - -@with_patch_inspect -def _fixed_getinnerframes(etb, context=1, tb_offset=0): - LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5 - - records = fix_frame_records_filenames(inspect.getinnerframes(etb, context)) - # If the error is at the console, don't build any context, since it would - # otherwise produce 5 blank lines printed out (there is no file at the - # console) - rec_check = records[tb_offset:] - try: - rname = rec_check[0][1] - if rname == '<ipython console>' or rname.endswith('<string>'): - return rec_check - except IndexError: - pass - - aux = traceback.extract_tb(etb) - assert len(records) == len(aux) - for i, (file, lnum, _, _) in enumerate(aux): - maybeStart = lnum - 1 - context // 2 - start = max(maybeStart, 0) - end = start + context - lines = linecache.getlines(file)[start:end] - buf = list(records[i]) - buf[LNUM_POS] = lnum - buf[INDEX_POS] = lnum - 1 - start - buf[LINES_POS] = lines - records[i] = tuple(buf) - return records[tb_offset:] - # Helper function -- largely belongs to VerboseTB, but we need the same # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they # can be recognized properly by ipython.el's py-traceback-line-re # (SyntaxErrors have to be treated specially because they have no traceback) -def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format): +def _format_traceback_lines(lines, Colors, has_colors: bool, lvals): """ Format tracebacks lines with pointing arrow, leading numbers... Parameters - ========== - - lnum: int - index: int - lines: list[string] - Colors: + ---------- + lines : list[Line] + Colors ColorScheme used. - lvals: bytes + lvals : str Values of local variables, already colored, to inject just after the error line. - _line_format: f (str) -> (str, bool) - return (colorized version of str, failure to do so) """ numbers_width = INDENT_SIZE - 1 res = [] - for i,line in enumerate(lines, lnum-index): - line = py3compat.cast_unicode(line) + for stack_line in lines: + if stack_line is stack_data.LINE_GAP: + res.append('%s (...)%s\n' % (Colors.linenoEm, Colors.Normal)) + continue - new_line, err = _line_format(line, 'str') - if not err: - line = new_line - - if i == lnum: + line = stack_line.render(pygmented=has_colors).rstrip('\n') + '\n' + lineno = stack_line.lineno + if stack_line.is_current: # This is the line with the error - pad = numbers_width - len(str(i)) - num = '%s%s' % (debugger.make_arrow(pad), str(lnum)) - line = '%s%s%s %s%s' % (Colors.linenoEm, num, - Colors.line, line, Colors.Normal) + pad = numbers_width - len(str(lineno)) + num = '%s%s' % (debugger.make_arrow(pad), str(lineno)) + start_color = Colors.linenoEm else: - num = '%*s' % (numbers_width, i) - line = '%s%s%s %s' % (Colors.lineno, num, - Colors.Normal, line) + num = '%*s' % (numbers_width, lineno) + start_color = Colors.lineno + + line = '%s%s%s %s' % (start_color, num, Colors.Normal, line) res.append(line) - if lvals and i == lnum: + if lvals and stack_line.is_current: res.append(lvals + '\n') return res -def is_recursion_error(etype, value, records): - try: - # RecursionError is new in Python 3.5 - recursion_error_type = RecursionError - except NameError: - recursion_error_type = RuntimeError - - # The default recursion limit is 1000, but some of that will be taken up - # by stack frames in IPython itself. >500 frames probably indicates - # a recursion error. - return (etype is recursion_error_type) \ - and "recursion" in str(value).lower() \ - and len(records) > _FRAME_RECURSION_LIMIT - -def find_recursion(etype, value, records): - """Identify the repeating stack frames from a RecursionError traceback - 'records' is a list as returned by VerboseTB.get_records() +def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): + """ + Format filename lines with `In [n]` if it's the nth code cell or `File *.py` if it's a module. - Returns (last_unique, repeat_length) + Parameters + ---------- + file : str + ColorFilename + ColorScheme's filename coloring to be used. + ColorNormal + ColorScheme's normal coloring to be used. """ - # This involves a bit of guesswork - we want to show enough of the traceback - # to indicate where the recursion is occurring. We guess that the innermost - # quarter of the traceback (250 frames by default) is repeats, and find the - # first frame (from in to out) that looks different. - if not is_recursion_error(etype, value, records): - return len(records), 0 - - # Select filename, lineno, func_name to track frames with - records = [r[1:4] for r in records] - inner_frames = records[-(len(records)//4):] - frames_repeated = set(inner_frames) - - last_seen_at = {} - longest_repeat = 0 - i = len(records) - for frame in reversed(records): - i -= 1 - if frame not in frames_repeated: - last_unique = i - break - - if frame in last_seen_at: - distance = last_seen_at[frame] - i - longest_repeat = max(longest_repeat, distance) - - last_seen_at[frame] = i + ipinst = get_ipython() + + if ipinst is not None and file in ipinst.compile._filename_map: + file = "[%s]" % ipinst.compile._filename_map[file] + tpl_link = f"Input {ColorFilename}In {{file}}{ColorNormal}" else: - last_unique = 0 # The whole traceback was recursion + file = util_path.compress_user( + py3compat.cast_unicode(file, util_path.fs_encoding) + ) + if lineno is None: + tpl_link = f"File {ColorFilename}{{file}}{ColorNormal}" + else: + tpl_link = f"File {ColorFilename}{{file}}:{{lineno}}{ColorNormal}" - return last_unique, longest_repeat + return tpl_link.format(file=file, lineno=lineno) #--------------------------------------------------------------------------- # Module classes @@ -413,7 +207,16 @@ class TBTools(colorable.Colorable): # Number of frames to skip when reporting tracebacks tb_offset = 0 - def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None): + def __init__( + self, + color_scheme="NoColor", + call_pdb=False, + ostream=None, + parent=None, + config=None, + *, + debugger_cls=None, + ): # Whether to call the interactive pdb debugger after printing # tracebacks or not super(TBTools, self).__init__(parent=parent, config=config) @@ -433,9 +236,10 @@ class TBTools(colorable.Colorable): self.set_colors(color_scheme) self.old_scheme = color_scheme # save initial value for toggles + self.debugger_cls = debugger_cls or debugger.Pdb if call_pdb: - self.pdb = debugger.Pdb() + self.pdb = self.debugger_cls() else: self.pdb = None @@ -458,21 +262,26 @@ class TBTools(colorable.Colorable): ostream = property(_get_ostream, _set_ostream) - def get_parts_of_chained_exception(self, evalue): - def get_chained_exception(exception_value): - cause = getattr(exception_value, '__cause__', None) - if cause: - return cause - if getattr(exception_value, '__suppress_context__', False): - return None - return getattr(exception_value, '__context__', None) + @staticmethod + def _get_chained_exception(exception_value): + cause = getattr(exception_value, "__cause__", None) + if cause: + return cause + if getattr(exception_value, "__suppress_context__", False): + return None + return getattr(exception_value, "__context__", None) + + def get_parts_of_chained_exception( + self, evalue + ) -> Optional[Tuple[type, BaseException, TracebackType]]: - chained_evalue = get_chained_exception(evalue) + chained_evalue = self._get_chained_exception(evalue) if chained_evalue: return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + return None - def prepare_chained_exception_message(self, cause): + def prepare_chained_exception_message(self, cause) -> List[Any]: direct_cause = "\nThe above exception was the direct cause of the following exception:\n" exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" @@ -482,6 +291,10 @@ class TBTools(colorable.Colorable): message = [[exception_during_handling]] return message + @property + def has_colors(self) -> bool: + return self.color_scheme_table.active_scheme_name.lower() != "nocolor" + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" @@ -508,7 +321,7 @@ class TBTools(colorable.Colorable): """Convert a structured traceback (a list) to a string.""" return '\n'.join(stb) - def text(self, etype, value, tb, tb_offset=None, context=5): + def text(self, etype, value, tb, tb_offset: Optional[int] = None, context=5): """Return formatted traceback. Subclasses may override this if they add extra arguments. @@ -517,8 +330,9 @@ class TBTools(colorable.Colorable): tb_offset, context) return self.stb2text(tb_list) - def structured_traceback(self, etype, evalue, tb, tb_offset=None, - context=5, mode=None): + def structured_traceback( + self, etype, evalue, tb, tb_offset: Optional[int] = None, context=5, mode=None + ): """Return a list of traceback frames. Must be implemented by each class. @@ -532,7 +346,7 @@ class ListTB(TBTools): Calling requires 3 arguments: (etype, evalue, elist) as would be obtained by:: - + etype, evalue, tb = sys.exc_info() if tb: elist = traceback.extract_tb(tb) @@ -546,9 +360,6 @@ class ListTB(TBTools): Because they are meant to be called without a full traceback (only a list), instances of this class can't call the interactive pdb debugger.""" - def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None): - TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, - ostream=ostream, parent=parent,config=config) def __call__(self, etype, value, elist): self.ostream.flush() @@ -561,28 +372,30 @@ class ListTB(TBTools): else: return None - def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, - context=5): + def structured_traceback( + self, + etype: type, + evalue: BaseException, + etb: Optional[TracebackType] = None, + tb_offset: Optional[int] = None, + context=5, + ): """Return a color formatted string with the traceback info. Parameters ---------- etype : exception type - Type of the exception raised. - + Type of the exception raised. evalue : object - Data stored in the exception - - etb : object - If list: List of frames, see class docstring for details. - If Traceback: Traceback of the exception. - + Data stored in the exception + etb : list | TracebackType | None + If list: List of frames, see class docstring for details. + If Traceback: Traceback of the exception. tb_offset : int, optional - Number of frames in the traceback to skip. If not given, the - instance evalue is used (set in constructor). - + Number of frames in the traceback to skip. If not given, the + instance evalue is used (set in constructor). context : int, optional - Number of lines of context information to print. + Number of lines of context information to print. Returns ------- @@ -602,6 +415,7 @@ class ListTB(TBTools): else: elist = [] tb_offset = self.tb_offset if tb_offset is None else tb_offset + assert isinstance(tb_offset, int) Colors = self.Colors out_list = [] if elist: @@ -650,21 +464,29 @@ class ListTB(TBTools): Colors = self.Colors list = [] for filename, lineno, name, line in extracted_list[:-1]: - item = ' File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \ - (Colors.filename, filename, Colors.Normal, - Colors.lineno, lineno, Colors.Normal, - Colors.name, name, Colors.Normal) + item = " %s in %s%s%s\n" % ( + _format_filename( + filename, Colors.filename, Colors.Normal, lineno=lineno + ), + Colors.name, + name, + Colors.Normal, + ) if line: item += ' %s\n' % line.strip() list.append(item) # Emphasize the last entry filename, lineno, name, line = extracted_list[-1] - item = '%s File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \ - (Colors.normalEm, - Colors.filenameEm, filename, Colors.normalEm, - Colors.linenoEm, lineno, Colors.normalEm, - Colors.nameEm, name, Colors.normalEm, - Colors.Normal) + item = "%s %s in %s%s%s%s\n" % ( + Colors.normalEm, + _format_filename( + filename, Colors.filenameEm, Colors.normalEm, lineno=lineno + ), + Colors.nameEm, + name, + Colors.normalEm, + Colors.Normal, + ) if line: item += '%s %s%s\n' % (Colors.line, line.strip(), Colors.Normal) @@ -699,13 +521,22 @@ class ListTB(TBTools): lineno = value.lineno textline = linecache.getline(value.filename, value.lineno) else: - lineno = 'unknown' - textline = '' - list.append('%s File %s"%s"%s, line %s%s%s\n' % \ - (Colors.normalEm, - Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm, - Colors.linenoEm, lineno, Colors.Normal )) - if textline == '': + lineno = "unknown" + textline = "" + list.append( + "%s %s%s\n" + % ( + Colors.normalEm, + _format_filename( + value.filename, + Colors.filenameEm, + Colors.normalEm, + lineno=(None if lineno == "unknown" else lineno), + ), + Colors.Normal, + ) + ) + if textline == "": textline = py3compat.cast_unicode(value.text, "utf-8") if textline is not None: @@ -759,7 +590,7 @@ class ListTB(TBTools): Parameters ---------- etype : exception type - value : exception value + evalue : exception value """ # This method needs to use __call__ from *this* class, not the one from # a subclass whose signature or behavior may be different @@ -785,18 +616,34 @@ class VerboseTB(TBTools): traceback, to be used with alternate interpreters (because their own code would appear in the traceback).""" - def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, - tb_offset=0, long_header=False, include_vars=True, - check_cache=None, debugger_cls = None, - parent=None, config=None): + def __init__( + self, + color_scheme: str = "Linux", + call_pdb: bool = False, + ostream=None, + tb_offset: int = 0, + long_header: bool = False, + include_vars: bool = True, + check_cache=None, + debugger_cls=None, + parent=None, + config=None, + ): """Specify traceback offset, headers and color scheme. Define how many frames to drop from the tracebacks. Calling it with tb_offset=1 allows use of this handler in interpreters which will have their own code at the top of the traceback (VerboseTB will first remove that frame before printing the traceback info).""" - TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, - ostream=ostream, parent=parent, config=config) + TBTools.__init__( + self, + color_scheme=color_scheme, + call_pdb=call_pdb, + ostream=ostream, + parent=parent, + config=config, + debugger_cls=debugger_cls, + ) self.tb_offset = tb_offset self.long_header = long_header self.include_vars = include_vars @@ -809,97 +656,46 @@ class VerboseTB(TBTools): check_cache = linecache.checkcache self.check_cache = check_cache - self.debugger_cls = debugger_cls or debugger.Pdb self.skip_hidden = True - def format_records(self, records, last_unique, recursion_repeat): - """Format the stack frames of the traceback""" - frames = [] - - skipped = 0 - lastrecord = len(records) - 1 - for i, r in enumerate(records[: last_unique + recursion_repeat + 1]): - if self.skip_hidden: - if r[0].f_locals.get("__tracebackhide__", 0) and i != lastrecord: - skipped += 1 - continue - if skipped: - Colors = self.Colors # just a shorthand + quicker name lookup - ColorsNormal = Colors.Normal # used a lot - frames.append( - " %s[... skipping hidden %s frame]%s\n" - % (Colors.excName, skipped, ColorsNormal) - ) - skipped = 0 - - frames.append(self.format_record(*r)) - - if skipped: - Colors = self.Colors # just a shorthand + quicker name lookup - ColorsNormal = Colors.Normal # used a lot - frames.append( - " %s[... skipping hidden %s frame]%s\n" - % (Colors.excName, skipped, ColorsNormal) - ) - - if recursion_repeat: - frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) - frames.append(self.format_record(*records[last_unique+recursion_repeat+1])) - - return frames - - def format_record(self, frame, file, lnum, func, lines, index): + def format_record(self, frame_info): """Format a single stack frame""" Colors = self.Colors # just a shorthand + quicker name lookup ColorsNormal = Colors.Normal # used a lot - col_scheme = self.color_scheme_table.active_scheme_name - indent = ' ' * INDENT_SIZE - em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal) - undefined = '%sundefined%s' % (Colors.em, ColorsNormal) - tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal) - tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm, - ColorsNormal) - tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \ - (Colors.vName, Colors.valEm, ColorsNormal) - tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal) - tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal, - Colors.vName, ColorsNormal) - tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal) - - if not file: - file = '?' - elif file.startswith(str("<")) and file.endswith(str(">")): - # Not a real filename, no problem... - pass - elif not os.path.isabs(file): - # Try to make the filename absolute by trying all - # sys.path entries (which is also what linecache does) - for dirname in sys.path: - try: - fullname = os.path.join(dirname, file) - if os.path.isfile(fullname): - file = os.path.abspath(fullname) - break - except Exception: - # Just in case that sys.path contains very - # strange entries... - pass - - file = py3compat.cast_unicode(file, util_path.fs_encoding) - link = tpl_link % util_path.compress_user(file) - args, varargs, varkw, locals_ = inspect.getargvalues(frame) - - if func == '?': - call = '' - elif func == '<module>': - call = tpl_call % (func, '') + + if isinstance(frame_info, stack_data.RepeatedFrames): + return ' %s[... skipping similar frames: %s]%s\n' % ( + Colors.excName, frame_info.description, ColorsNormal) + + indent = " " * INDENT_SIZE + em_normal = "%s\n%s%s" % (Colors.valEm, indent, ColorsNormal) + tpl_call = f"in {Colors.vName}{{file}}{Colors.valEm}{{scope}}{ColorsNormal}" + tpl_call_fail = "in %s%%s%s(***failed resolving arguments***)%s" % ( + Colors.vName, + Colors.valEm, + ColorsNormal, + ) + tpl_name_val = "%%s %s= %%s%s" % (Colors.valEm, ColorsNormal) + + link = _format_filename( + frame_info.filename, + Colors.filenameEm, + ColorsNormal, + lineno=frame_info.lineno, + ) + args, varargs, varkw, locals_ = inspect.getargvalues(frame_info.frame) + + func = frame_info.executing.code_qualname() + if func == "<module>": + call = tpl_call.format(file=func, scope="") else: # Decide whether to include variable details or not var_repr = eqrepr if self.include_vars else nullrepr try: - call = tpl_call % (func, inspect.formatargvalues(args, - varargs, varkw, - locals_, formatvalue=var_repr)) + scope = inspect.formatargvalues( + args, varargs, varkw, locals_, formatvalue=var_repr + ) + call = tpl_call.format(file=func, scope=scope) except KeyError: # This happens in situations like errors inside generator # expressions, where local variables are listed in the @@ -919,111 +715,26 @@ class VerboseTB(TBTools): # disabled. call = tpl_call_fail % func - # Don't attempt to tokenize binary files. - if file.endswith(('.so', '.pyd', '.dll')): - return '%s %s\n' % (link, call) - - elif file.endswith(('.pyc', '.pyo')): - # Look up the corresponding source file. - try: - file = source_from_cache(file) - except ValueError: - # Failed to get the source file for some reason - # E.g. https://github.com/ipython/ipython/issues/9486 - return '%s %s\n' % (link, call) - - def linereader(file=file, lnum=[lnum], getline=linecache.getline): - line = getline(file, lnum[0]) - lnum[0] += 1 - return line - - # Build the list of names on this line of code where the exception - # occurred. - try: - names = [] - name_cont = False - - for token_type, token, start, end, line in generate_tokens(linereader): - # build composite names - if token_type == tokenize.NAME and token not in keyword.kwlist: - if name_cont: - # Continuation of a dotted name - try: - names[-1].append(token) - except IndexError: - names.append([token]) - name_cont = False - else: - # Regular new names. We append everything, the caller - # will be responsible for pruning the list later. It's - # very tricky to try to prune as we go, b/c composite - # names can fool us. The pruning at the end is easy - # to do (or the caller can print a list with repeated - # names if so desired. - names.append([token]) - elif token == '.': - name_cont = True - elif token_type == tokenize.NEWLINE: - break - - except (IndexError, UnicodeDecodeError, SyntaxError): - # signals exit of tokenizer - # SyntaxError can occur if the file is not actually Python - # - see gh-6300 - pass - except tokenize.TokenError as msg: - # Tokenizing may fail for various reasons, many of which are - # harmless. (A good example is when the line in question is the - # close of a triple-quoted string, cf gh-6864). We don't want to - # show this to users, but want make it available for debugging - # purposes. - _m = ("An unexpected error occurred while tokenizing input\n" - "The following traceback may be corrupted or invalid\n" - "The error message is: %s\n" % msg) - debug(_m) - - # Join composite names (e.g. "dict.fromkeys") - names = ['.'.join(n) for n in names] - # prune names list of duplicates, but keep the right order - unique_names = uniq_stable(names) - - # Start loop over vars lvals = '' lvals_list = [] if self.include_vars: - for name_full in unique_names: - name_base = name_full.split('.', 1)[0] - if name_base in frame.f_code.co_varnames: - if name_base in locals_: - try: - value = repr(eval(name_full, locals_)) - except: - value = undefined - else: - value = undefined - name = tpl_local_var % name_full - else: - if name_base in frame.f_globals: - try: - value = repr(eval(name_full, frame.f_globals)) - except: - value = undefined - else: - value = undefined - name = tpl_global_var % name_full - lvals_list.append(tpl_name_val % (name, value)) + try: + # we likely want to fix stackdata at some point, but + # still need a workaround. + fibp = frame_info.variables_in_executing_piece + for var in fibp: + lvals_list.append(tpl_name_val % (var.name, repr(var.value))) + except Exception: + lvals_list.append( + "Exception trying to inspect frame. No more locals available." + ) if lvals_list: lvals = '%s%s' % (indent, em_normal.join(lvals_list)) - level = '%s %s\n' % (link, call) + result = "%s, %s\n" % (link, call) - if index is None: - return level - else: - _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2 - return '%s%s' % (level, ''.join( - _format_traceback_lines(lnum, index, lines, Colors, lvals, - _line_format))) + result += ''.join(_format_traceback_lines(frame_info.lines, Colors, self.has_colors, lvals)) + return result def prepare_header(self, etype, long_version=False): colors = self.Colors # just a shorthand + quicker name lookup @@ -1061,7 +772,14 @@ class VerboseTB(TBTools): return ['%s%s%s: %s' % (colors.excName, etype_str, colorsnormal, py3compat.cast_unicode(evalue_str))] - def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset): + def format_exception_as_a_whole( + self, + etype: type, + evalue: BaseException, + etb: Optional[TracebackType], + number_of_lines_of_context, + tb_offset: Optional[int], + ): """Formats the header, traceback and exception message for a single exception. This may be called multiple times by Python 3 exception chaining @@ -1075,52 +793,75 @@ class VerboseTB(TBTools): pass tb_offset = self.tb_offset if tb_offset is None else tb_offset + assert isinstance(tb_offset, int) head = self.prepare_header(etype, self.long_header) - records = self.get_records(etb, number_of_lines_of_context, tb_offset) + records = ( + self.get_records(etb, number_of_lines_of_context, tb_offset) if etb else [] + ) - - last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) - - frames = self.format_records(records, last_unique, recursion_repeat) + frames = [] + skipped = 0 + lastrecord = len(records) - 1 + for i, r in enumerate(records): + if not isinstance(r, stack_data.RepeatedFrames) and self.skip_hidden: + if r.frame.f_locals.get("__tracebackhide__", 0) and i != lastrecord: + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(r)) + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) formatted_exception = self.format_exception(etype, evalue) if records: - filepath, lnum = records[-1][1:3] - filepath = os.path.abspath(filepath) + frame_info = records[-1] ipinst = get_ipython() if ipinst is not None: - ipinst.hooks.synchronize_with_editor(filepath, lnum, 0) + ipinst.hooks.synchronize_with_editor(frame_info.filename, frame_info.lineno, 0) return [[head] + frames + [''.join(formatted_exception[0])]] - def get_records(self, etb, number_of_lines_of_context, tb_offset): - try: - # Try the default getinnerframes and Alex's: Alex's fixes some - # problems, but it generates empty tracebacks for console errors - # (5 blanks lines) where none should be returned. - return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset) - except UnicodeDecodeError: - # This can occur if a file's encoding magic comment is wrong. - # I can't see a way to recover without duplicating a bunch of code - # from the stdlib traceback module. --TK - error('\nUnicodeDecodeError while processing traceback.\n') - return None - except: - # FIXME: I've been getting many crash reports from python 2.3 - # users, traceable to inspect.py. If I can find a small test-case - # to reproduce this, I should either write a better workaround or - # file a bug report against inspect (if that's the real problem). - # So far, I haven't been able to find an isolated example to - # reproduce the problem. - inspect_error() - traceback.print_exc(file=self.ostream) - info('\nUnfortunately, your original traceback can not be constructed.\n') - return None - - def structured_traceback(self, etype, evalue, etb, tb_offset=None, - number_of_lines_of_context=5): + def get_records( + self, etb: TracebackType, number_of_lines_of_context: int, tb_offset: int + ): + assert etb is not None + context = number_of_lines_of_context - 1 + after = context // 2 + before = context - after + if self.has_colors: + style = get_style_by_name("default") + style = stack_data.style_with_executing_node(style, "") + formatter = Terminal256Formatter(style=style) + else: + formatter = None + options = stack_data.Options( + before=before, + after=after, + pygments_formatter=formatter, + ) + return list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:] + + def structured_traceback( + self, + etype: type, + evalue: Optional[BaseException], + etb: Optional[TracebackType], + tb_offset: Optional[int] = None, + number_of_lines_of_context: int = 5, + ): """Return a nice text document describing the traceback.""" - formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context, tb_offset) @@ -1133,6 +874,7 @@ class VerboseTB(TBTools): formatted_exceptions = formatted_exception exception = self.get_parts_of_chained_exception(evalue) if exception: + assert evalue is not None formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) etype, evalue, etb = exception else: @@ -1157,7 +899,7 @@ class VerboseTB(TBTools): return structured_traceback_parts - def debugger(self, force=False): + def debugger(self, force: bool = False): """Call up the pdb debugger if desired, always clean up the tb reference. @@ -1191,6 +933,7 @@ class VerboseTB(TBTools): else: etb = self.tb = sys.last_traceback while self.tb is not None and self.tb.tb_next is not None: + assert self.tb.tb_next is not None self.tb = self.tb.tb_next if etb and etb.tb_next: etb = etb.tb_next @@ -1236,6 +979,8 @@ class FormattedTB(VerboseTB, ListTB): occurs with python programs that themselves execute other python code, like Python shells). """ + mode: str + def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, ostream=None, tb_offset=0, long_header=False, include_vars=False, @@ -1282,8 +1027,7 @@ class FormattedTB(VerboseTB, ListTB): """Convert a structured traceback (a list) to a string.""" return self.tb_join_char.join(stb) - - def set_mode(self, mode=None): + def set_mode(self, mode: Optional[str] = None): """Switch to the desired mode. If mode is not specified, cycles through the available modes.""" @@ -1293,9 +1037,12 @@ class FormattedTB(VerboseTB, ListTB): len(self.valid_modes) self.mode = self.valid_modes[new_idx] elif mode not in self.valid_modes: - raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n' - 'Valid modes: ' + str(self.valid_modes)) + raise ValueError( + "Unrecognized mode in FormattedTB: <" + mode + ">\n" + "Valid modes: " + str(self.valid_modes) + ) else: + assert isinstance(mode, str) self.mode = mode # include variable details only in 'Verbose' mode self.include_vars = (self.mode == self.valid_modes[2]) @@ -1340,7 +1087,7 @@ class AutoFormattedTB(FormattedTB): - tb_offset: the number of frames to skip over in the stack, on a per-call basis (this overrides temporarily the instance's tb_offset - given at initialization time. """ + given at initialization time.""" if out is None: out = self.ostream @@ -1357,6 +1104,10 @@ class AutoFormattedTB(FormattedTB): def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): + + etype: type + value: BaseException + # tb: TracebackType or tupleof tb types ? if etype is None: etype, value, tb = sys.exc_info() if isinstance(tb, tuple): |