From e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 Mon Sep 17 00:00:00 2001 From: shadchin Date: Thu, 10 Feb 2022 16:44:39 +0300 Subject: Restoring authorship annotation for . Commit 2 of 2. --- contrib/python/ipython/py3/IPython/core/alias.py | 516 +- .../python/ipython/py3/IPython/core/application.py | 924 +-- .../ipython/py3/IPython/core/async_helpers.py | 366 +- .../python/ipython/py3/IPython/core/autocall.py | 140 +- .../ipython/py3/IPython/core/builtin_trap.py | 172 +- .../python/ipython/py3/IPython/core/compilerop.py | 376 +- .../python/ipython/py3/IPython/core/completer.py | 4236 +++++------ .../ipython/py3/IPython/core/completerlib.py | 804 +- .../ipython/py3/IPython/core/crashhandler.py | 456 +- .../python/ipython/py3/IPython/core/debugger.py | 2198 +++--- contrib/python/ipython/py3/IPython/core/display.py | 3112 ++++---- .../ipython/py3/IPython/core/display_trap.py | 140 +- .../python/ipython/py3/IPython/core/displayhook.py | 650 +- .../python/ipython/py3/IPython/core/displaypub.py | 276 +- contrib/python/ipython/py3/IPython/core/error.py | 120 +- contrib/python/ipython/py3/IPython/core/events.py | 322 +- .../python/ipython/py3/IPython/core/excolors.py | 368 +- .../python/ipython/py3/IPython/core/extensions.py | 300 +- .../python/ipython/py3/IPython/core/formatters.py | 2048 +++--- .../python/ipython/py3/IPython/core/getipython.py | 48 +- contrib/python/ipython/py3/IPython/core/history.py | 1812 ++--- .../python/ipython/py3/IPython/core/historyapp.py | 322 +- contrib/python/ipython/py3/IPython/core/hooks.py | 380 +- .../ipython/py3/IPython/core/inputsplitter.py | 1544 ++-- .../ipython/py3/IPython/core/inputtransformer.py | 1072 +-- .../ipython/py3/IPython/core/inputtransformer2.py | 1500 ++-- .../ipython/py3/IPython/core/interactiveshell.py | 7680 ++++++++++---------- .../ipython/py3/IPython/core/latex_symbols.py | 2602 +++---- contrib/python/ipython/py3/IPython/core/logger.py | 436 +- contrib/python/ipython/py3/IPython/core/macro.py | 106 +- contrib/python/ipython/py3/IPython/core/magic.py | 1406 ++-- .../ipython/py3/IPython/core/magic_arguments.py | 556 +- .../ipython/py3/IPython/core/magics/__init__.py | 84 +- .../python/ipython/py3/IPython/core/magics/auto.py | 256 +- .../ipython/py3/IPython/core/magics/basic.py | 1328 ++-- .../python/ipython/py3/IPython/core/magics/code.py | 1500 ++-- .../ipython/py3/IPython/core/magics/config.py | 316 +- .../ipython/py3/IPython/core/magics/display.py | 164 +- .../ipython/py3/IPython/core/magics/execution.py | 3048 ++++---- .../ipython/py3/IPython/core/magics/extension.py | 126 +- .../ipython/py3/IPython/core/magics/history.py | 638 +- .../ipython/py3/IPython/core/magics/logging.py | 390 +- .../ipython/py3/IPython/core/magics/namespace.py | 1424 ++-- .../python/ipython/py3/IPython/core/magics/osm.py | 1714 ++--- .../ipython/py3/IPython/core/magics/packaging.py | 220 +- .../ipython/py3/IPython/core/magics/pylab.py | 332 +- .../ipython/py3/IPython/core/magics/script.py | 588 +- .../python/ipython/py3/IPython/core/oinspect.py | 2062 +++--- contrib/python/ipython/py3/IPython/core/page.py | 686 +- contrib/python/ipython/py3/IPython/core/payload.py | 110 +- .../python/ipython/py3/IPython/core/payloadpage.py | 104 +- .../python/ipython/py3/IPython/core/prefilter.py | 1418 ++-- .../py3/IPython/core/profile/README_STARTUP | 22 +- .../python/ipython/py3/IPython/core/profileapp.py | 624 +- .../python/ipython/py3/IPython/core/profiledir.py | 442 +- contrib/python/ipython/py3/IPython/core/prompts.py | 42 +- .../python/ipython/py3/IPython/core/pylabtools.py | 846 +-- contrib/python/ipython/py3/IPython/core/release.py | 238 +- .../python/ipython/py3/IPython/core/shellapp.py | 940 +-- .../python/ipython/py3/IPython/core/splitinput.py | 274 +- contrib/python/ipython/py3/IPython/core/ultratb.py | 2912 ++++---- contrib/python/ipython/py3/IPython/core/usage.py | 680 +- 62 files changed, 30258 insertions(+), 30258 deletions(-) (limited to 'contrib/python/ipython/py3/IPython/core') diff --git a/contrib/python/ipython/py3/IPython/core/alias.py b/contrib/python/ipython/py3/IPython/core/alias.py index c5aa00129d5..2ad990231a0 100644 --- a/contrib/python/ipython/py3/IPython/core/alias.py +++ b/contrib/python/ipython/py3/IPython/core/alias.py @@ -1,258 +1,258 @@ -# encoding: utf-8 -""" -System command aliases. - -Authors: - -* Fernando Perez -* Brian Granger -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import os -import re -import sys - -from traitlets.config.configurable import Configurable -from .error import UsageError - -from traitlets import List, Instance -from logging import error - -#----------------------------------------------------------------------------- -# Utilities -#----------------------------------------------------------------------------- - -# This is used as the pattern for calls to split_user_input. -shell_line_split = re.compile(r'^(\s*)()(\S+)(.*$)') - -def default_aliases(): - """Return list of shell aliases to auto-define. - """ - # Note: the aliases defined here should be safe to use on a kernel - # regardless of what frontend it is attached to. Frontends that use a - # kernel in-process can define additional aliases that will only work in - # their case. For example, things like 'less' or 'clear' that manipulate - # the terminal should NOT be declared here, as they will only work if the - # kernel is running inside a true terminal, and not over the network. - - if os.name == 'posix': - default_aliases = [('mkdir', 'mkdir'), ('rmdir', 'rmdir'), - ('mv', 'mv'), ('rm', 'rm'), ('cp', 'cp'), - ('cat', 'cat'), - ] - # Useful set of ls aliases. The GNU and BSD options are a little - # different, so we make aliases that provide as similar as possible - # behavior in ipython, by passing the right flags for each platform - if sys.platform.startswith('linux'): - ls_aliases = [('ls', 'ls -F --color'), - # long ls - ('ll', 'ls -F -o --color'), - # ls normal files only - ('lf', 'ls -F -o --color %l | grep ^-'), - # ls symbolic links - ('lk', 'ls -F -o --color %l | grep ^l'), - # directories or links to directories, - ('ldir', 'ls -F -o --color %l | grep /$'), - # things which are executable - ('lx', 'ls -F -o --color %l | grep ^-..x'), - ] - elif sys.platform.startswith('openbsd') or sys.platform.startswith('netbsd'): - # OpenBSD, NetBSD. The ls implementation on these platforms do not support - # the -G switch and lack the ability to use colorized output. - ls_aliases = [('ls', 'ls -F'), - # long ls - ('ll', 'ls -F -l'), - # ls normal files only - ('lf', 'ls -F -l %l | grep ^-'), - # ls symbolic links - ('lk', 'ls -F -l %l | grep ^l'), - # directories or links to directories, - ('ldir', 'ls -F -l %l | grep /$'), - # things which are executable - ('lx', 'ls -F -l %l | grep ^-..x'), - ] - else: - # BSD, OSX, etc. - ls_aliases = [('ls', 'ls -F -G'), - # long ls - ('ll', 'ls -F -l -G'), - # ls normal files only - ('lf', 'ls -F -l -G %l | grep ^-'), - # ls symbolic links - ('lk', 'ls -F -l -G %l | grep ^l'), - # directories or links to directories, - ('ldir', 'ls -F -G -l %l | grep /$'), - # things which are executable - ('lx', 'ls -F -l -G %l | grep ^-..x'), - ] - default_aliases = default_aliases + ls_aliases - elif os.name in ['nt', 'dos']: - default_aliases = [('ls', 'dir /on'), - ('ddir', 'dir /ad /on'), ('ldir', 'dir /ad /on'), - ('mkdir', 'mkdir'), ('rmdir', 'rmdir'), - ('echo', 'echo'), ('ren', 'ren'), ('copy', 'copy'), - ] - else: - default_aliases = [] - - return default_aliases - - -class AliasError(Exception): - pass - - -class InvalidAliasError(AliasError): - pass - -class Alias(object): - """Callable object storing the details of one alias. - - Instances are registered as magic functions to allow use of aliases. - """ - - # Prepare blacklist - blacklist = {'cd','popd','pushd','dhist','alias','unalias'} - - def __init__(self, shell, name, cmd): - self.shell = shell - self.name = name - self.cmd = cmd - self.__doc__ = "Alias for `!{}`".format(cmd) - self.nargs = self.validate() - - def validate(self): - """Validate the alias, and return the number of arguments.""" - if self.name in self.blacklist: - raise InvalidAliasError("The name %s can't be aliased " - "because it is a keyword or builtin." % self.name) - try: - caller = self.shell.magics_manager.magics['line'][self.name] - except KeyError: - pass - else: - if not isinstance(caller, Alias): - raise InvalidAliasError("The name %s can't be aliased " - "because it is another magic command." % self.name) - - if not (isinstance(self.cmd, str)): - raise InvalidAliasError("An alias command must be a string, " - "got: %r" % self.cmd) - - nargs = self.cmd.count('%s') - self.cmd.count('%%s') - - if (nargs > 0) and (self.cmd.find('%l') >= 0): - raise InvalidAliasError('The %s and %l specifiers are mutually ' - 'exclusive in alias definitions.') - - return nargs - - def __repr__(self): - return "".format(self.name, self.cmd) - - def __call__(self, rest=''): - cmd = self.cmd - nargs = self.nargs - # Expand the %l special to be the user's input line - if cmd.find('%l') >= 0: - cmd = cmd.replace('%l', rest) - rest = '' - - if nargs==0: - if cmd.find('%%s') >= 1: - cmd = cmd.replace('%%s', '%s') - # Simple, argument-less aliases - cmd = '%s %s' % (cmd, rest) - else: - # Handle aliases with positional arguments - args = rest.split(None, nargs) - if len(args) < nargs: - raise UsageError('Alias <%s> requires %s arguments, %s given.' % - (self.name, nargs, len(args))) - cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:])) - - self.shell.system(cmd) - -#----------------------------------------------------------------------------- -# Main AliasManager class -#----------------------------------------------------------------------------- - -class AliasManager(Configurable): - - default_aliases = List(default_aliases()).tag(config=True) - user_aliases = List(default_value=[]).tag(config=True) - shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) - - def __init__(self, shell=None, **kwargs): - super(AliasManager, self).__init__(shell=shell, **kwargs) - # For convenient access - self.linemagics = self.shell.magics_manager.magics['line'] - self.init_aliases() - - def init_aliases(self): - # Load default & user aliases - for name, cmd in self.default_aliases + self.user_aliases: - if cmd.startswith('ls ') and self.shell.colors == 'NoColor': - cmd = cmd.replace(' --color', '') - self.soft_define_alias(name, cmd) - - @property - def aliases(self): - return [(n, func.cmd) for (n, func) in self.linemagics.items() - if isinstance(func, Alias)] - - def soft_define_alias(self, name, cmd): - """Define an alias, but don't raise on an AliasError.""" - try: - self.define_alias(name, cmd) - except AliasError as e: - error("Invalid alias: %s" % e) - - def define_alias(self, name, cmd): - """Define a new alias after validating it. - - This will raise an :exc:`AliasError` if there are validation - problems. - """ - caller = Alias(shell=self.shell, name=name, cmd=cmd) - self.shell.magics_manager.register_function(caller, magic_kind='line', - magic_name=name) - - def get_alias(self, name): - """Return an alias, or None if no alias by that name exists.""" - aname = self.linemagics.get(name, None) - return aname if isinstance(aname, Alias) else None - - def is_alias(self, name): - """Return whether or not a given name has been defined as an alias""" - return self.get_alias(name) is not None - - def undefine_alias(self, name): - if self.is_alias(name): - del self.linemagics[name] - else: - raise ValueError('%s is not an alias' % name) - - def clear_aliases(self): - for name, cmd in self.aliases: - self.undefine_alias(name) - - def retrieve_alias(self, name): - """Retrieve the command to which an alias expands.""" - caller = self.get_alias(name) - if caller: - return caller.cmd - else: - raise ValueError('%s is not an alias' % name) +# encoding: utf-8 +""" +System command aliases. + +Authors: + +* Fernando Perez +* Brian Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import re +import sys + +from traitlets.config.configurable import Configurable +from .error import UsageError + +from traitlets import List, Instance +from logging import error + +#----------------------------------------------------------------------------- +# Utilities +#----------------------------------------------------------------------------- + +# This is used as the pattern for calls to split_user_input. +shell_line_split = re.compile(r'^(\s*)()(\S+)(.*$)') + +def default_aliases(): + """Return list of shell aliases to auto-define. + """ + # Note: the aliases defined here should be safe to use on a kernel + # regardless of what frontend it is attached to. Frontends that use a + # kernel in-process can define additional aliases that will only work in + # their case. For example, things like 'less' or 'clear' that manipulate + # the terminal should NOT be declared here, as they will only work if the + # kernel is running inside a true terminal, and not over the network. + + if os.name == 'posix': + default_aliases = [('mkdir', 'mkdir'), ('rmdir', 'rmdir'), + ('mv', 'mv'), ('rm', 'rm'), ('cp', 'cp'), + ('cat', 'cat'), + ] + # Useful set of ls aliases. The GNU and BSD options are a little + # different, so we make aliases that provide as similar as possible + # behavior in ipython, by passing the right flags for each platform + if sys.platform.startswith('linux'): + ls_aliases = [('ls', 'ls -F --color'), + # long ls + ('ll', 'ls -F -o --color'), + # ls normal files only + ('lf', 'ls -F -o --color %l | grep ^-'), + # ls symbolic links + ('lk', 'ls -F -o --color %l | grep ^l'), + # directories or links to directories, + ('ldir', 'ls -F -o --color %l | grep /$'), + # things which are executable + ('lx', 'ls -F -o --color %l | grep ^-..x'), + ] + elif sys.platform.startswith('openbsd') or sys.platform.startswith('netbsd'): + # OpenBSD, NetBSD. The ls implementation on these platforms do not support + # the -G switch and lack the ability to use colorized output. + ls_aliases = [('ls', 'ls -F'), + # long ls + ('ll', 'ls -F -l'), + # ls normal files only + ('lf', 'ls -F -l %l | grep ^-'), + # ls symbolic links + ('lk', 'ls -F -l %l | grep ^l'), + # directories or links to directories, + ('ldir', 'ls -F -l %l | grep /$'), + # things which are executable + ('lx', 'ls -F -l %l | grep ^-..x'), + ] + else: + # BSD, OSX, etc. + ls_aliases = [('ls', 'ls -F -G'), + # long ls + ('ll', 'ls -F -l -G'), + # ls normal files only + ('lf', 'ls -F -l -G %l | grep ^-'), + # ls symbolic links + ('lk', 'ls -F -l -G %l | grep ^l'), + # directories or links to directories, + ('ldir', 'ls -F -G -l %l | grep /$'), + # things which are executable + ('lx', 'ls -F -l -G %l | grep ^-..x'), + ] + default_aliases = default_aliases + ls_aliases + elif os.name in ['nt', 'dos']: + default_aliases = [('ls', 'dir /on'), + ('ddir', 'dir /ad /on'), ('ldir', 'dir /ad /on'), + ('mkdir', 'mkdir'), ('rmdir', 'rmdir'), + ('echo', 'echo'), ('ren', 'ren'), ('copy', 'copy'), + ] + else: + default_aliases = [] + + return default_aliases + + +class AliasError(Exception): + pass + + +class InvalidAliasError(AliasError): + pass + +class Alias(object): + """Callable object storing the details of one alias. + + Instances are registered as magic functions to allow use of aliases. + """ + + # Prepare blacklist + blacklist = {'cd','popd','pushd','dhist','alias','unalias'} + + def __init__(self, shell, name, cmd): + self.shell = shell + self.name = name + self.cmd = cmd + self.__doc__ = "Alias for `!{}`".format(cmd) + self.nargs = self.validate() + + def validate(self): + """Validate the alias, and return the number of arguments.""" + if self.name in self.blacklist: + raise InvalidAliasError("The name %s can't be aliased " + "because it is a keyword or builtin." % self.name) + try: + caller = self.shell.magics_manager.magics['line'][self.name] + except KeyError: + pass + else: + if not isinstance(caller, Alias): + raise InvalidAliasError("The name %s can't be aliased " + "because it is another magic command." % self.name) + + if not (isinstance(self.cmd, str)): + raise InvalidAliasError("An alias command must be a string, " + "got: %r" % self.cmd) + + nargs = self.cmd.count('%s') - self.cmd.count('%%s') + + if (nargs > 0) and (self.cmd.find('%l') >= 0): + raise InvalidAliasError('The %s and %l specifiers are mutually ' + 'exclusive in alias definitions.') + + return nargs + + def __repr__(self): + return "".format(self.name, self.cmd) + + def __call__(self, rest=''): + cmd = self.cmd + nargs = self.nargs + # Expand the %l special to be the user's input line + if cmd.find('%l') >= 0: + cmd = cmd.replace('%l', rest) + rest = '' + + if nargs==0: + if cmd.find('%%s') >= 1: + cmd = cmd.replace('%%s', '%s') + # Simple, argument-less aliases + cmd = '%s %s' % (cmd, rest) + else: + # Handle aliases with positional arguments + args = rest.split(None, nargs) + if len(args) < nargs: + raise UsageError('Alias <%s> requires %s arguments, %s given.' % + (self.name, nargs, len(args))) + cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:])) + + self.shell.system(cmd) + +#----------------------------------------------------------------------------- +# Main AliasManager class +#----------------------------------------------------------------------------- + +class AliasManager(Configurable): + + default_aliases = List(default_aliases()).tag(config=True) + user_aliases = List(default_value=[]).tag(config=True) + shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) + + def __init__(self, shell=None, **kwargs): + super(AliasManager, self).__init__(shell=shell, **kwargs) + # For convenient access + self.linemagics = self.shell.magics_manager.magics['line'] + self.init_aliases() + + def init_aliases(self): + # Load default & user aliases + for name, cmd in self.default_aliases + self.user_aliases: + if cmd.startswith('ls ') and self.shell.colors == 'NoColor': + cmd = cmd.replace(' --color', '') + self.soft_define_alias(name, cmd) + + @property + def aliases(self): + return [(n, func.cmd) for (n, func) in self.linemagics.items() + if isinstance(func, Alias)] + + def soft_define_alias(self, name, cmd): + """Define an alias, but don't raise on an AliasError.""" + try: + self.define_alias(name, cmd) + except AliasError as e: + error("Invalid alias: %s" % e) + + def define_alias(self, name, cmd): + """Define a new alias after validating it. + + This will raise an :exc:`AliasError` if there are validation + problems. + """ + caller = Alias(shell=self.shell, name=name, cmd=cmd) + self.shell.magics_manager.register_function(caller, magic_kind='line', + magic_name=name) + + def get_alias(self, name): + """Return an alias, or None if no alias by that name exists.""" + aname = self.linemagics.get(name, None) + return aname if isinstance(aname, Alias) else None + + def is_alias(self, name): + """Return whether or not a given name has been defined as an alias""" + return self.get_alias(name) is not None + + def undefine_alias(self, name): + if self.is_alias(name): + del self.linemagics[name] + else: + raise ValueError('%s is not an alias' % name) + + def clear_aliases(self): + for name, cmd in self.aliases: + self.undefine_alias(name) + + def retrieve_alias(self, name): + """Retrieve the command to which an alias expands.""" + caller = self.get_alias(name) + if caller: + return caller.cmd + else: + raise ValueError('%s is not an alias' % name) diff --git a/contrib/python/ipython/py3/IPython/core/application.py b/contrib/python/ipython/py3/IPython/core/application.py index cad86198035..b319888b59b 100644 --- a/contrib/python/ipython/py3/IPython/core/application.py +++ b/contrib/python/ipython/py3/IPython/core/application.py @@ -1,462 +1,462 @@ -# encoding: utf-8 -""" -An application for IPython. - -All top-level applications should use the classes in this module for -handling configuration and creating configurables. - -The job of an :class:`Application` is to create the master configuration -object and then create the configurable objects, passing the config to them. -""" - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - -import atexit -from copy import deepcopy -import glob -import logging -import os -import shutil -import sys - -from traitlets.config.application import Application, catch_config_error -from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader -from IPython.core import release, crashhandler -from IPython.core.profiledir import ProfileDir, ProfileDirError -from IPython.paths import get_ipython_dir, get_ipython_package_dir -from IPython.utils.path import ensure_dir_exists -from traitlets import ( - List, Unicode, Type, Bool, Set, Instance, Undefined, - default, observe, -) - -if os.name == 'nt': - programdata = os.environ.get('PROGRAMDATA', None) - if programdata: - SYSTEM_CONFIG_DIRS = [os.path.join(programdata, 'ipython')] - else: # PROGRAMDATA is not defined by default on XP. - SYSTEM_CONFIG_DIRS = [] -else: - SYSTEM_CONFIG_DIRS = [ - "/usr/local/etc/ipython", - "/etc/ipython", - ] - - -ENV_CONFIG_DIRS = [] -_env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython') -if _env_config_dir not in SYSTEM_CONFIG_DIRS: - # only add ENV_CONFIG if sys.prefix is not already included - ENV_CONFIG_DIRS.append(_env_config_dir) - - -_envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS') -if _envvar in {None, ''}: - IPYTHON_SUPPRESS_CONFIG_ERRORS = None -else: - if _envvar.lower() in {'1','true'}: - IPYTHON_SUPPRESS_CONFIG_ERRORS = True - elif _envvar.lower() in {'0','false'} : - IPYTHON_SUPPRESS_CONFIG_ERRORS = False - else: - sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of {'0', '1', 'false', 'true', ''}."% _envvar ) - -# aliases and flags - -base_aliases = { - 'profile-dir' : 'ProfileDir.location', - 'profile' : 'BaseIPythonApplication.profile', - 'ipython-dir' : 'BaseIPythonApplication.ipython_dir', - 'log-level' : 'Application.log_level', - 'config' : 'BaseIPythonApplication.extra_config_file', -} - -base_flags = dict( - debug = ({'Application' : {'log_level' : logging.DEBUG}}, - "set log level to logging.DEBUG (maximize logging output)"), - quiet = ({'Application' : {'log_level' : logging.CRITICAL}}, - "set log level to logging.CRITICAL (minimize logging output)"), - init = ({'BaseIPythonApplication' : { - 'copy_config_files' : True, - 'auto_create' : True} - }, """Initialize profile with default config files. This is equivalent - to running `ipython profile create ` prior to startup. - """) -) - -class ProfileAwareConfigLoader(PyFileConfigLoader): - """A Python file config loader that is aware of IPython profiles.""" - def load_subconfig(self, fname, path=None, profile=None): - if profile is not None: - try: - profile_dir = ProfileDir.find_profile_dir_by_name( - get_ipython_dir(), - profile, - ) - except ProfileDirError: - return - path = profile_dir.location - return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path) - -class BaseIPythonApplication(Application): - - name = u'ipython' - description = Unicode(u'IPython: an enhanced interactive Python shell.') - version = Unicode(release.version) - - aliases = base_aliases - flags = base_flags - classes = List([ProfileDir]) - - # enable `load_subconfig('cfg.py', profile='name')` - python_config_loader_class = ProfileAwareConfigLoader - - # Track whether the config_file has changed, - # because some logic happens only if we aren't using the default. - config_file_specified = Set() - - config_file_name = Unicode() - @default('config_file_name') - def _config_file_name_default(self): - return self.name.replace('-','_') + u'_config.py' - @observe('config_file_name') - def _config_file_name_changed(self, change): - if change['new'] != change['old']: - self.config_file_specified.add(change['new']) - - # The directory that contains IPython's builtin profiles. - builtin_profile_dir = Unicode( - os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default') - ) - - config_file_paths = List(Unicode()) - @default('config_file_paths') - def _config_file_paths_default(self): - return [] - - extra_config_file = Unicode( - help="""Path to an extra config file to load. - - If specified, load this config file in addition to any other IPython config. - """).tag(config=True) - @observe('extra_config_file') - def _extra_config_file_changed(self, change): - old = change['old'] - new = change['new'] - try: - self.config_files.remove(old) - except ValueError: - pass - self.config_file_specified.add(new) - self.config_files.append(new) - - profile = Unicode(u'default', - help="""The IPython profile to use.""" - ).tag(config=True) - - @observe('profile') - def _profile_changed(self, change): - self.builtin_profile_dir = os.path.join( - get_ipython_package_dir(), u'config', u'profile', change['new'] - ) - - ipython_dir = Unicode( - help=""" - The name of the IPython directory. This directory is used for logging - configuration (through profiles), history storage, etc. The default - is usually $HOME/.ipython. This option can also be specified through - the environment variable IPYTHONDIR. - """ - ).tag(config=True) - @default('ipython_dir') - def _ipython_dir_default(self): - d = get_ipython_dir() - self._ipython_dir_changed({ - 'name': 'ipython_dir', - 'old': d, - 'new': d, - }) - return d - - _in_init_profile_dir = False - profile_dir = Instance(ProfileDir, allow_none=True) - @default('profile_dir') - def _profile_dir_default(self): - # avoid recursion - if self._in_init_profile_dir: - return - # profile_dir requested early, force initialization - self.init_profile_dir() - return self.profile_dir - - overwrite = Bool(False, - help="""Whether to overwrite existing config files when copying""" - ).tag(config=True) - auto_create = Bool(False, - help="""Whether to create profile dir if it doesn't exist""" - ).tag(config=True) - - config_files = List(Unicode()) - @default('config_files') - def _config_files_default(self): - return [self.config_file_name] - - copy_config_files = Bool(False, - help="""Whether to install the default config files into the profile dir. - If a new profile is being created, and IPython contains config files for that - profile, then they will be staged into the new directory. Otherwise, - default config files will be automatically generated. - """).tag(config=True) - - verbose_crash = Bool(False, - help="""Create a massive crash report when IPython encounters what may be an - internal error. The default is to append a short message to the - usual traceback""").tag(config=True) - - # The class to use as the crash handler. - crash_handler_class = Type(crashhandler.CrashHandler) - - @catch_config_error - def __init__(self, **kwargs): - super(BaseIPythonApplication, self).__init__(**kwargs) - # ensure current working directory exists - try: - os.getcwd() - except: - # exit if cwd doesn't exist - self.log.error("Current working directory doesn't exist.") - self.exit(1) - - #------------------------------------------------------------------------- - # Various stages of Application creation - #------------------------------------------------------------------------- - - deprecated_subcommands = {} - - def initialize_subcommand(self, subc, argv=None): - if subc in self.deprecated_subcommands: - self.log.warning("Subcommand `ipython {sub}` is deprecated and will be removed " - "in future versions.".format(sub=subc)) - self.log.warning("You likely want to use `jupyter {sub}` in the " - "future".format(sub=subc)) - return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv) - - def init_crash_handler(self): - """Create a crash handler, typically setting sys.excepthook to it.""" - self.crash_handler = self.crash_handler_class(self) - sys.excepthook = self.excepthook - def unset_crashhandler(): - sys.excepthook = sys.__excepthook__ - atexit.register(unset_crashhandler) - - def excepthook(self, etype, evalue, tb): - """this is sys.excepthook after init_crashhandler - - set self.verbose_crash=True to use our full crashhandler, instead of - a regular traceback with a short message (crash_handler_lite) - """ - - if self.verbose_crash: - return self.crash_handler(etype, evalue, tb) - else: - return crashhandler.crash_handler_lite(etype, evalue, tb) - - @observe('ipython_dir') - def _ipython_dir_changed(self, change): - old = change['old'] - new = change['new'] - if old is not Undefined: - str_old = os.path.abspath(old) - if str_old in sys.path: - sys.path.remove(str_old) - str_path = os.path.abspath(new) - sys.path.append(str_path) - ensure_dir_exists(new) - readme = os.path.join(new, 'README') - readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') - if not os.path.exists(readme) and os.path.exists(readme_src): - shutil.copy(readme_src, readme) - for d in ('extensions', 'nbextensions'): - path = os.path.join(new, d) - try: - ensure_dir_exists(path) - except OSError as e: - # this will not be EEXIST - self.log.error("couldn't create path %s: %s", path, e) - self.log.debug("IPYTHONDIR set to: %s" % new) - - def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): - """Load the config file. - - By default, errors in loading config are handled, and a warning - printed on screen. For testing, the suppress_errors option is set - to False, so errors will make tests fail. - - `suppress_errors` default value is to be `None` in which case the - behavior default to the one of `traitlets.Application`. - - The default value can be set : - - to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive). - - to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive). - - to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset. - - Any other value are invalid, and will make IPython exit with a non-zero return code. - """ - - - self.log.debug("Searching path %s for config files", self.config_file_paths) - base_config = 'ipython_config.py' - self.log.debug("Attempting to load config file: %s" % - base_config) - try: - if suppress_errors is not None: - old_value = Application.raise_config_file_errors - Application.raise_config_file_errors = not suppress_errors; - Application.load_config_file( - self, - base_config, - path=self.config_file_paths - ) - except ConfigFileNotFound: - # ignore errors loading parent - self.log.debug("Config file %s not found", base_config) - pass - if suppress_errors is not None: - Application.raise_config_file_errors = old_value - - for config_file_name in self.config_files: - if not config_file_name or config_file_name == base_config: - continue - self.log.debug("Attempting to load config file: %s" % - self.config_file_name) - try: - Application.load_config_file( - self, - config_file_name, - path=self.config_file_paths - ) - except ConfigFileNotFound: - # Only warn if the default config file was NOT being used. - if config_file_name in self.config_file_specified: - msg = self.log.warning - else: - msg = self.log.debug - msg("Config file not found, skipping: %s", config_file_name) - except Exception: - # For testing purposes. - if not suppress_errors: - raise - self.log.warning("Error loading config file: %s" % - self.config_file_name, exc_info=True) - - def init_profile_dir(self): - """initialize the profile dir""" - self._in_init_profile_dir = True - if self.profile_dir is not None: - # already ran - return - if 'ProfileDir.location' not in self.config: - # location not specified, find by profile name - try: - p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config) - except ProfileDirError: - # not found, maybe create it (always create default profile) - if self.auto_create or self.profile == 'default': - try: - p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config) - except ProfileDirError: - self.log.fatal("Could not create profile: %r"%self.profile) - self.exit(1) - else: - self.log.info("Created profile dir: %r"%p.location) - else: - self.log.fatal("Profile %r not found."%self.profile) - self.exit(1) - else: - self.log.debug(f"Using existing profile dir: {p.location!r}") - else: - location = self.config.ProfileDir.location - # location is fully specified - try: - p = ProfileDir.find_profile_dir(location, self.config) - except ProfileDirError: - # not found, maybe create it - if self.auto_create: - try: - p = ProfileDir.create_profile_dir(location, self.config) - except ProfileDirError: - self.log.fatal("Could not create profile directory: %r"%location) - self.exit(1) - else: - self.log.debug("Creating new profile dir: %r"%location) - else: - self.log.fatal("Profile directory %r not found."%location) - self.exit(1) - else: - self.log.debug(f"Using existing profile dir: {p.location!r}") - # if profile_dir is specified explicitly, set profile name - dir_name = os.path.basename(p.location) - if dir_name.startswith('profile_'): - self.profile = dir_name[8:] - - self.profile_dir = p - self.config_file_paths.append(p.location) - self._in_init_profile_dir = False - - def init_config_files(self): - """[optionally] copy default config files into profile dir.""" - self.config_file_paths.extend(ENV_CONFIG_DIRS) - self.config_file_paths.extend(SYSTEM_CONFIG_DIRS) - # copy config files - path = self.builtin_profile_dir - if self.copy_config_files: - src = self.profile - - cfg = self.config_file_name - if path and os.path.exists(os.path.join(path, cfg)): - self.log.warning("Staging %r from %s into %r [overwrite=%s]"%( - cfg, src, self.profile_dir.location, self.overwrite) - ) - self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) - else: - self.stage_default_config_file() - else: - # Still stage *bundled* config files, but not generated ones - # This is necessary for `ipython profile=sympy` to load the profile - # on the first go - files = glob.glob(os.path.join(path, '*.py')) - for fullpath in files: - cfg = os.path.basename(fullpath) - if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False): - # file was copied - self.log.warning("Staging bundled %s from %s into %r"%( - cfg, self.profile, self.profile_dir.location) - ) - - - def stage_default_config_file(self): - """auto generate default config file, and stage it into the profile.""" - s = self.generate_config_file() - fname = os.path.join(self.profile_dir.location, self.config_file_name) - if self.overwrite or not os.path.exists(fname): - self.log.warning("Generating default config file: %r"%(fname)) - with open(fname, 'w') as f: - f.write(s) - - @catch_config_error - def initialize(self, argv=None): - # don't hook up crash handler before parsing command-line - self.parse_command_line(argv) - self.init_crash_handler() - if self.subapp is not None: - # stop here if subapp is taking over - return - # save a copy of CLI config to re-load after config files - # so that it has highest priority - cl_config = deepcopy(self.config) - self.init_profile_dir() - self.init_config_files() - self.load_config_file() - # enforce cl-opts override configfile opts: - self.update_config(cl_config) +# encoding: utf-8 +""" +An application for IPython. + +All top-level applications should use the classes in this module for +handling configuration and creating configurables. + +The job of an :class:`Application` is to create the master configuration +object and then create the configurable objects, passing the config to them. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import atexit +from copy import deepcopy +import glob +import logging +import os +import shutil +import sys + +from traitlets.config.application import Application, catch_config_error +from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader +from IPython.core import release, crashhandler +from IPython.core.profiledir import ProfileDir, ProfileDirError +from IPython.paths import get_ipython_dir, get_ipython_package_dir +from IPython.utils.path import ensure_dir_exists +from traitlets import ( + List, Unicode, Type, Bool, Set, Instance, Undefined, + default, observe, +) + +if os.name == 'nt': + programdata = os.environ.get('PROGRAMDATA', None) + if programdata: + SYSTEM_CONFIG_DIRS = [os.path.join(programdata, 'ipython')] + else: # PROGRAMDATA is not defined by default on XP. + SYSTEM_CONFIG_DIRS = [] +else: + SYSTEM_CONFIG_DIRS = [ + "/usr/local/etc/ipython", + "/etc/ipython", + ] + + +ENV_CONFIG_DIRS = [] +_env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython') +if _env_config_dir not in SYSTEM_CONFIG_DIRS: + # only add ENV_CONFIG if sys.prefix is not already included + ENV_CONFIG_DIRS.append(_env_config_dir) + + +_envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS') +if _envvar in {None, ''}: + IPYTHON_SUPPRESS_CONFIG_ERRORS = None +else: + if _envvar.lower() in {'1','true'}: + IPYTHON_SUPPRESS_CONFIG_ERRORS = True + elif _envvar.lower() in {'0','false'} : + IPYTHON_SUPPRESS_CONFIG_ERRORS = False + else: + sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of {'0', '1', 'false', 'true', ''}."% _envvar ) + +# aliases and flags + +base_aliases = { + 'profile-dir' : 'ProfileDir.location', + 'profile' : 'BaseIPythonApplication.profile', + 'ipython-dir' : 'BaseIPythonApplication.ipython_dir', + 'log-level' : 'Application.log_level', + 'config' : 'BaseIPythonApplication.extra_config_file', +} + +base_flags = dict( + debug = ({'Application' : {'log_level' : logging.DEBUG}}, + "set log level to logging.DEBUG (maximize logging output)"), + quiet = ({'Application' : {'log_level' : logging.CRITICAL}}, + "set log level to logging.CRITICAL (minimize logging output)"), + init = ({'BaseIPythonApplication' : { + 'copy_config_files' : True, + 'auto_create' : True} + }, """Initialize profile with default config files. This is equivalent + to running `ipython profile create ` prior to startup. + """) +) + +class ProfileAwareConfigLoader(PyFileConfigLoader): + """A Python file config loader that is aware of IPython profiles.""" + def load_subconfig(self, fname, path=None, profile=None): + if profile is not None: + try: + profile_dir = ProfileDir.find_profile_dir_by_name( + get_ipython_dir(), + profile, + ) + except ProfileDirError: + return + path = profile_dir.location + return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path) + +class BaseIPythonApplication(Application): + + name = u'ipython' + description = Unicode(u'IPython: an enhanced interactive Python shell.') + version = Unicode(release.version) + + aliases = base_aliases + flags = base_flags + classes = List([ProfileDir]) + + # enable `load_subconfig('cfg.py', profile='name')` + python_config_loader_class = ProfileAwareConfigLoader + + # Track whether the config_file has changed, + # because some logic happens only if we aren't using the default. + config_file_specified = Set() + + config_file_name = Unicode() + @default('config_file_name') + def _config_file_name_default(self): + return self.name.replace('-','_') + u'_config.py' + @observe('config_file_name') + def _config_file_name_changed(self, change): + if change['new'] != change['old']: + self.config_file_specified.add(change['new']) + + # The directory that contains IPython's builtin profiles. + builtin_profile_dir = Unicode( + os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default') + ) + + config_file_paths = List(Unicode()) + @default('config_file_paths') + def _config_file_paths_default(self): + return [] + + extra_config_file = Unicode( + help="""Path to an extra config file to load. + + If specified, load this config file in addition to any other IPython config. + """).tag(config=True) + @observe('extra_config_file') + def _extra_config_file_changed(self, change): + old = change['old'] + new = change['new'] + try: + self.config_files.remove(old) + except ValueError: + pass + self.config_file_specified.add(new) + self.config_files.append(new) + + profile = Unicode(u'default', + help="""The IPython profile to use.""" + ).tag(config=True) + + @observe('profile') + def _profile_changed(self, change): + self.builtin_profile_dir = os.path.join( + get_ipython_package_dir(), u'config', u'profile', change['new'] + ) + + ipython_dir = Unicode( + help=""" + The name of the IPython directory. This directory is used for logging + configuration (through profiles), history storage, etc. The default + is usually $HOME/.ipython. This option can also be specified through + the environment variable IPYTHONDIR. + """ + ).tag(config=True) + @default('ipython_dir') + def _ipython_dir_default(self): + d = get_ipython_dir() + self._ipython_dir_changed({ + 'name': 'ipython_dir', + 'old': d, + 'new': d, + }) + return d + + _in_init_profile_dir = False + profile_dir = Instance(ProfileDir, allow_none=True) + @default('profile_dir') + def _profile_dir_default(self): + # avoid recursion + if self._in_init_profile_dir: + return + # profile_dir requested early, force initialization + self.init_profile_dir() + return self.profile_dir + + overwrite = Bool(False, + help="""Whether to overwrite existing config files when copying""" + ).tag(config=True) + auto_create = Bool(False, + help="""Whether to create profile dir if it doesn't exist""" + ).tag(config=True) + + config_files = List(Unicode()) + @default('config_files') + def _config_files_default(self): + return [self.config_file_name] + + copy_config_files = Bool(False, + help="""Whether to install the default config files into the profile dir. + If a new profile is being created, and IPython contains config files for that + profile, then they will be staged into the new directory. Otherwise, + default config files will be automatically generated. + """).tag(config=True) + + verbose_crash = Bool(False, + help="""Create a massive crash report when IPython encounters what may be an + internal error. The default is to append a short message to the + usual traceback""").tag(config=True) + + # The class to use as the crash handler. + crash_handler_class = Type(crashhandler.CrashHandler) + + @catch_config_error + def __init__(self, **kwargs): + super(BaseIPythonApplication, self).__init__(**kwargs) + # ensure current working directory exists + try: + os.getcwd() + except: + # exit if cwd doesn't exist + self.log.error("Current working directory doesn't exist.") + self.exit(1) + + #------------------------------------------------------------------------- + # Various stages of Application creation + #------------------------------------------------------------------------- + + deprecated_subcommands = {} + + def initialize_subcommand(self, subc, argv=None): + if subc in self.deprecated_subcommands: + self.log.warning("Subcommand `ipython {sub}` is deprecated and will be removed " + "in future versions.".format(sub=subc)) + self.log.warning("You likely want to use `jupyter {sub}` in the " + "future".format(sub=subc)) + return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv) + + def init_crash_handler(self): + """Create a crash handler, typically setting sys.excepthook to it.""" + self.crash_handler = self.crash_handler_class(self) + sys.excepthook = self.excepthook + def unset_crashhandler(): + sys.excepthook = sys.__excepthook__ + atexit.register(unset_crashhandler) + + def excepthook(self, etype, evalue, tb): + """this is sys.excepthook after init_crashhandler + + set self.verbose_crash=True to use our full crashhandler, instead of + a regular traceback with a short message (crash_handler_lite) + """ + + if self.verbose_crash: + return self.crash_handler(etype, evalue, tb) + else: + return crashhandler.crash_handler_lite(etype, evalue, tb) + + @observe('ipython_dir') + def _ipython_dir_changed(self, change): + old = change['old'] + new = change['new'] + if old is not Undefined: + str_old = os.path.abspath(old) + if str_old in sys.path: + sys.path.remove(str_old) + str_path = os.path.abspath(new) + sys.path.append(str_path) + ensure_dir_exists(new) + readme = os.path.join(new, 'README') + readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') + if not os.path.exists(readme) and os.path.exists(readme_src): + shutil.copy(readme_src, readme) + for d in ('extensions', 'nbextensions'): + path = os.path.join(new, d) + try: + ensure_dir_exists(path) + except OSError as e: + # this will not be EEXIST + self.log.error("couldn't create path %s: %s", path, e) + self.log.debug("IPYTHONDIR set to: %s" % new) + + def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): + """Load the config file. + + By default, errors in loading config are handled, and a warning + printed on screen. For testing, the suppress_errors option is set + to False, so errors will make tests fail. + + `suppress_errors` default value is to be `None` in which case the + behavior default to the one of `traitlets.Application`. + + The default value can be set : + - to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive). + - to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive). + - to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset. + + Any other value are invalid, and will make IPython exit with a non-zero return code. + """ + + + self.log.debug("Searching path %s for config files", self.config_file_paths) + base_config = 'ipython_config.py' + self.log.debug("Attempting to load config file: %s" % + base_config) + try: + if suppress_errors is not None: + old_value = Application.raise_config_file_errors + Application.raise_config_file_errors = not suppress_errors; + Application.load_config_file( + self, + base_config, + path=self.config_file_paths + ) + except ConfigFileNotFound: + # ignore errors loading parent + self.log.debug("Config file %s not found", base_config) + pass + if suppress_errors is not None: + Application.raise_config_file_errors = old_value + + for config_file_name in self.config_files: + if not config_file_name or config_file_name == base_config: + continue + self.log.debug("Attempting to load config file: %s" % + self.config_file_name) + try: + Application.load_config_file( + self, + config_file_name, + path=self.config_file_paths + ) + except ConfigFileNotFound: + # Only warn if the default config file was NOT being used. + if config_file_name in self.config_file_specified: + msg = self.log.warning + else: + msg = self.log.debug + msg("Config file not found, skipping: %s", config_file_name) + except Exception: + # For testing purposes. + if not suppress_errors: + raise + self.log.warning("Error loading config file: %s" % + self.config_file_name, exc_info=True) + + def init_profile_dir(self): + """initialize the profile dir""" + self._in_init_profile_dir = True + if self.profile_dir is not None: + # already ran + return + if 'ProfileDir.location' not in self.config: + # location not specified, find by profile name + try: + p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config) + except ProfileDirError: + # not found, maybe create it (always create default profile) + if self.auto_create or self.profile == 'default': + try: + p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config) + except ProfileDirError: + self.log.fatal("Could not create profile: %r"%self.profile) + self.exit(1) + else: + self.log.info("Created profile dir: %r"%p.location) + else: + self.log.fatal("Profile %r not found."%self.profile) + self.exit(1) + else: + self.log.debug(f"Using existing profile dir: {p.location!r}") + else: + location = self.config.ProfileDir.location + # location is fully specified + try: + p = ProfileDir.find_profile_dir(location, self.config) + except ProfileDirError: + # not found, maybe create it + if self.auto_create: + try: + p = ProfileDir.create_profile_dir(location, self.config) + except ProfileDirError: + self.log.fatal("Could not create profile directory: %r"%location) + self.exit(1) + else: + self.log.debug("Creating new profile dir: %r"%location) + else: + self.log.fatal("Profile directory %r not found."%location) + self.exit(1) + else: + self.log.debug(f"Using existing profile dir: {p.location!r}") + # if profile_dir is specified explicitly, set profile name + dir_name = os.path.basename(p.location) + if dir_name.startswith('profile_'): + self.profile = dir_name[8:] + + self.profile_dir = p + self.config_file_paths.append(p.location) + self._in_init_profile_dir = False + + def init_config_files(self): + """[optionally] copy default config files into profile dir.""" + self.config_file_paths.extend(ENV_CONFIG_DIRS) + self.config_file_paths.extend(SYSTEM_CONFIG_DIRS) + # copy config files + path = self.builtin_profile_dir + if self.copy_config_files: + src = self.profile + + cfg = self.config_file_name + if path and os.path.exists(os.path.join(path, cfg)): + self.log.warning("Staging %r from %s into %r [overwrite=%s]"%( + cfg, src, self.profile_dir.location, self.overwrite) + ) + self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) + else: + self.stage_default_config_file() + else: + # Still stage *bundled* config files, but not generated ones + # This is necessary for `ipython profile=sympy` to load the profile + # on the first go + files = glob.glob(os.path.join(path, '*.py')) + for fullpath in files: + cfg = os.path.basename(fullpath) + if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False): + # file was copied + self.log.warning("Staging bundled %s from %s into %r"%( + cfg, self.profile, self.profile_dir.location) + ) + + + def stage_default_config_file(self): + """auto generate default config file, and stage it into the profile.""" + s = self.generate_config_file() + fname = os.path.join(self.profile_dir.location, self.config_file_name) + if self.overwrite or not os.path.exists(fname): + self.log.warning("Generating default config file: %r"%(fname)) + with open(fname, 'w') as f: + f.write(s) + + @catch_config_error + def initialize(self, argv=None): + # don't hook up crash handler before parsing command-line + self.parse_command_line(argv) + self.init_crash_handler() + if self.subapp is not None: + # stop here if subapp is taking over + return + # save a copy of CLI config to re-load after config files + # so that it has highest priority + cl_config = deepcopy(self.config) + self.init_profile_dir() + self.init_config_files() + self.load_config_file() + # enforce cl-opts override configfile opts: + self.update_config(cl_config) diff --git a/contrib/python/ipython/py3/IPython/core/async_helpers.py b/contrib/python/ipython/py3/IPython/core/async_helpers.py index 070468e70ae..fca78def85a 100644 --- a/contrib/python/ipython/py3/IPython/core/async_helpers.py +++ b/contrib/python/ipython/py3/IPython/core/async_helpers.py @@ -1,183 +1,183 @@ -""" -Async helper function that are invalid syntax on Python 3.5 and below. - -This code is best effort, and may have edge cases not behaving as expected. In -particular it contain a number of heuristics to detect whether code is -effectively async and need to run in an event loop or not. - -Some constructs (like top-level `return`, or `yield`) are taken care of -explicitly to actually raise a SyntaxError and stay as close as possible to -Python semantics. -""" - - -import ast -import sys -import asyncio -import inspect -from textwrap import dedent, indent - - -class _AsyncIORunner: - def __init__(self): - self._loop = None - - @property - def loop(self): - """Always returns a non-closed event loop""" - if self._loop is None or self._loop.is_closed(): - policy = asyncio.get_event_loop_policy() - self._loop = policy.new_event_loop() - policy.set_event_loop(self._loop) - return self._loop - - def __call__(self, coro): - """ - Handler for asyncio autoawait - """ - return self.loop.run_until_complete(coro) - - def __str__(self): - return 'asyncio' - -_asyncio_runner = _AsyncIORunner() - - -def _curio_runner(coroutine): - """ - handler for curio autoawait - """ - import curio - - return curio.run(coroutine) - - -def _trio_runner(async_fn): - import trio - - async def loc(coro): - """ - We need the dummy no-op async def to protect from - trio's internal. See https://github.com/python-trio/trio/issues/89 - """ - return await coro - - return trio.run(loc, async_fn) - - -def _pseudo_sync_runner(coro): - """ - A runner that does not really allow async execution, and just advance the coroutine. - - See discussion in https://github.com/python-trio/trio/issues/608, - - Credit to Nathaniel Smith - - """ - try: - coro.send(None) - except StopIteration as exc: - return exc.value - else: - # TODO: do not raise but return an execution result with the right info. - raise RuntimeError( - "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__) - ) - - -def _asyncify(code: str) -> str: - """wrap code in async def definition. - - And setup a bit of context to run it later. - """ - res = dedent( - """ - async def __wrapper__(): - try: - {usercode} - finally: - locals() - """ - ).format(usercode=indent(code, " " * 8)) - return res - - -class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): - """ - Find syntax errors that would be an error in an async repl, but because - the implementation involves wrapping the repl in an async function, it - is erroneously allowed (e.g. yield or return at the top level) - """ - def __init__(self): - if sys.version_info >= (3,8): - raise ValueError('DEPRECATED in Python 3.8+') - self.depth = 0 - super().__init__() - - def generic_visit(self, node): - func_types = (ast.FunctionDef, ast.AsyncFunctionDef) - invalid_types_by_depth = { - 0: (ast.Return, ast.Yield, ast.YieldFrom), - 1: (ast.Nonlocal,) - } - - should_traverse = self.depth < max(invalid_types_by_depth.keys()) - if isinstance(node, func_types) and should_traverse: - self.depth += 1 - super().generic_visit(node) - self.depth -= 1 - elif isinstance(node, invalid_types_by_depth[self.depth]): - raise SyntaxError() - else: - super().generic_visit(node) - - -def _async_parse_cell(cell: str) -> ast.AST: - """ - This is a compatibility shim for pre-3.7 when async outside of a function - is a syntax error at the parse stage. - - It will return an abstract syntax tree parsed as if async and await outside - of a function were not a syntax error. - """ - if sys.version_info < (3, 7): - # Prior to 3.7 you need to asyncify before parse - wrapped_parse_tree = ast.parse(_asyncify(cell)) - return wrapped_parse_tree.body[0].body[0] - else: - return ast.parse(cell) - - -def _should_be_async(cell: str) -> bool: - """Detect if a block of code need to be wrapped in an `async def` - - Attempt to parse the block of code, it it compile we're fine. - Otherwise we wrap if and try to compile. - - If it works, assume it should be async. Otherwise Return False. - - Not handled yet: If the block of code has a return statement as the top - level, it will be seen as async. This is a know limitation. - """ - if sys.version_info > (3, 8): - try: - code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) - return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except (SyntaxError, MemoryError): - return False - try: - # we can't limit ourself to ast.parse, as it __accepts__ to parse on - # 3.7+, but just does not _compile_ - code = compile(cell, "<>", "exec") - except (SyntaxError, MemoryError): - try: - parse_tree = _async_parse_cell(cell) - - # Raise a SyntaxError if there are top-level return or yields - v = _AsyncSyntaxErrorVisitor() - v.visit(parse_tree) - - except (SyntaxError, MemoryError): - return False - return True - return False +""" +Async helper function that are invalid syntax on Python 3.5 and below. + +This code is best effort, and may have edge cases not behaving as expected. In +particular it contain a number of heuristics to detect whether code is +effectively async and need to run in an event loop or not. + +Some constructs (like top-level `return`, or `yield`) are taken care of +explicitly to actually raise a SyntaxError and stay as close as possible to +Python semantics. +""" + + +import ast +import sys +import asyncio +import inspect +from textwrap import dedent, indent + + +class _AsyncIORunner: + def __init__(self): + self._loop = None + + @property + def loop(self): + """Always returns a non-closed event loop""" + if self._loop is None or self._loop.is_closed(): + policy = asyncio.get_event_loop_policy() + self._loop = policy.new_event_loop() + policy.set_event_loop(self._loop) + return self._loop + + def __call__(self, coro): + """ + Handler for asyncio autoawait + """ + return self.loop.run_until_complete(coro) + + def __str__(self): + return 'asyncio' + +_asyncio_runner = _AsyncIORunner() + + +def _curio_runner(coroutine): + """ + handler for curio autoawait + """ + import curio + + return curio.run(coroutine) + + +def _trio_runner(async_fn): + import trio + + async def loc(coro): + """ + We need the dummy no-op async def to protect from + trio's internal. See https://github.com/python-trio/trio/issues/89 + """ + return await coro + + return trio.run(loc, async_fn) + + +def _pseudo_sync_runner(coro): + """ + A runner that does not really allow async execution, and just advance the coroutine. + + See discussion in https://github.com/python-trio/trio/issues/608, + + Credit to Nathaniel Smith + + """ + try: + coro.send(None) + except StopIteration as exc: + return exc.value + else: + # TODO: do not raise but return an execution result with the right info. + raise RuntimeError( + "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__) + ) + + +def _asyncify(code: str) -> str: + """wrap code in async def definition. + + And setup a bit of context to run it later. + """ + res = dedent( + """ + async def __wrapper__(): + try: + {usercode} + finally: + locals() + """ + ).format(usercode=indent(code, " " * 8)) + return res + + +class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): + """ + Find syntax errors that would be an error in an async repl, but because + the implementation involves wrapping the repl in an async function, it + is erroneously allowed (e.g. yield or return at the top level) + """ + def __init__(self): + if sys.version_info >= (3,8): + raise ValueError('DEPRECATED in Python 3.8+') + self.depth = 0 + super().__init__() + + def generic_visit(self, node): + func_types = (ast.FunctionDef, ast.AsyncFunctionDef) + invalid_types_by_depth = { + 0: (ast.Return, ast.Yield, ast.YieldFrom), + 1: (ast.Nonlocal,) + } + + should_traverse = self.depth < max(invalid_types_by_depth.keys()) + if isinstance(node, func_types) and should_traverse: + self.depth += 1 + super().generic_visit(node) + self.depth -= 1 + elif isinstance(node, invalid_types_by_depth[self.depth]): + raise SyntaxError() + else: + super().generic_visit(node) + + +def _async_parse_cell(cell: str) -> ast.AST: + """ + This is a compatibility shim for pre-3.7 when async outside of a function + is a syntax error at the parse stage. + + It will return an abstract syntax tree parsed as if async and await outside + of a function were not a syntax error. + """ + if sys.version_info < (3, 7): + # Prior to 3.7 you need to asyncify before parse + wrapped_parse_tree = ast.parse(_asyncify(cell)) + return wrapped_parse_tree.body[0].body[0] + else: + return ast.parse(cell) + + +def _should_be_async(cell: str) -> bool: + """Detect if a block of code need to be wrapped in an `async def` + + Attempt to parse the block of code, it it compile we're fine. + Otherwise we wrap if and try to compile. + + If it works, assume it should be async. Otherwise Return False. + + Not handled yet: If the block of code has a return statement as the top + level, it will be seen as async. This is a know limitation. + """ + if sys.version_info > (3, 8): + try: + code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE + except (SyntaxError, MemoryError): + return False + try: + # we can't limit ourself to ast.parse, as it __accepts__ to parse on + # 3.7+, but just does not _compile_ + code = compile(cell, "<>", "exec") + except (SyntaxError, MemoryError): + try: + parse_tree = _async_parse_cell(cell) + + # Raise a SyntaxError if there are top-level return or yields + v = _AsyncSyntaxErrorVisitor() + v.visit(parse_tree) + + except (SyntaxError, MemoryError): + return False + return True + return False diff --git a/contrib/python/ipython/py3/IPython/core/autocall.py b/contrib/python/ipython/py3/IPython/core/autocall.py index 4ef2bce59c1..bab7f859c96 100644 --- a/contrib/python/ipython/py3/IPython/core/autocall.py +++ b/contrib/python/ipython/py3/IPython/core/autocall.py @@ -1,70 +1,70 @@ -# encoding: utf-8 -""" -Autocall capabilities for IPython.core. - -Authors: - -* Brian Granger -* Fernando Perez -* Thomas Kluyver - -Notes ------ -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - - -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - -class IPyAutocall(object): - """ Instances of this class are always autocalled - - This happens regardless of 'autocall' variable state. Use this to - develop macro-like mechanisms. - """ - _ip = None - rewrite = True - def __init__(self, ip=None): - self._ip = ip - - def set_ip(self, ip): - """ Will be used to set _ip point to current ipython instance b/f call - - Override this method if you don't want this to happen. - - """ - self._ip = ip - - -class ExitAutocall(IPyAutocall): - """An autocallable object which will be added to the user namespace so that - exit, exit(), quit or quit() are all valid ways to close the shell.""" - rewrite = False - - def __call__(self): - self._ip.ask_exit() - -class ZMQExitAutocall(ExitAutocall): - """Exit IPython. Autocallable, so it needn't be explicitly called. - - Parameters - ---------- - keep_kernel : bool - If True, leave the kernel alive. Otherwise, tell the kernel to exit too - (default). - """ - def __call__(self, keep_kernel=False): - self._ip.keepkernel_on_exit = keep_kernel - self._ip.ask_exit() +# encoding: utf-8 +""" +Autocall capabilities for IPython.core. + +Authors: + +* Brian Granger +* Fernando Perez +* Thomas Kluyver + +Notes +----- +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + +class IPyAutocall(object): + """ Instances of this class are always autocalled + + This happens regardless of 'autocall' variable state. Use this to + develop macro-like mechanisms. + """ + _ip = None + rewrite = True + def __init__(self, ip=None): + self._ip = ip + + def set_ip(self, ip): + """ Will be used to set _ip point to current ipython instance b/f call + + Override this method if you don't want this to happen. + + """ + self._ip = ip + + +class ExitAutocall(IPyAutocall): + """An autocallable object which will be added to the user namespace so that + exit, exit(), quit or quit() are all valid ways to close the shell.""" + rewrite = False + + def __call__(self): + self._ip.ask_exit() + +class ZMQExitAutocall(ExitAutocall): + """Exit IPython. Autocallable, so it needn't be explicitly called. + + Parameters + ---------- + keep_kernel : bool + If True, leave the kernel alive. Otherwise, tell the kernel to exit too + (default). + """ + def __call__(self, keep_kernel=False): + self._ip.keepkernel_on_exit = keep_kernel + self._ip.ask_exit() diff --git a/contrib/python/ipython/py3/IPython/core/builtin_trap.py b/contrib/python/ipython/py3/IPython/core/builtin_trap.py index c3f0b1eda81..a8ea4abcd9d 100644 --- a/contrib/python/ipython/py3/IPython/core/builtin_trap.py +++ b/contrib/python/ipython/py3/IPython/core/builtin_trap.py @@ -1,86 +1,86 @@ -""" -A context manager for managing things injected into :mod:`builtins`. -""" -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. -import builtins as builtin_mod - -from traitlets.config.configurable import Configurable - -from traitlets import Instance - - -class __BuiltinUndefined(object): pass -BuiltinUndefined = __BuiltinUndefined() - -class __HideBuiltin(object): pass -HideBuiltin = __HideBuiltin() - - -class BuiltinTrap(Configurable): - - shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', - allow_none=True) - - def __init__(self, shell=None): - super(BuiltinTrap, self).__init__(shell=shell, config=None) - self._orig_builtins = {} - # We define this to track if a single BuiltinTrap is nested. - # Only turn off the trap when the outermost call to __exit__ is made. - self._nested_level = 0 - self.shell = shell - # builtins we always add - if set to HideBuiltin, they will just - # be removed instead of being replaced by something else - self.auto_builtins = {'exit': HideBuiltin, - 'quit': HideBuiltin, - 'get_ipython': self.shell.get_ipython, - } - - def __enter__(self): - if self._nested_level == 0: - self.activate() - self._nested_level += 1 - # I return self, so callers can use add_builtin in a with clause. - return self - - def __exit__(self, type, value, traceback): - if self._nested_level == 1: - self.deactivate() - self._nested_level -= 1 - # Returning False will cause exceptions to propagate - return False - - def add_builtin(self, key, value): - """Add a builtin and save the original.""" - bdict = builtin_mod.__dict__ - orig = bdict.get(key, BuiltinUndefined) - if value is HideBuiltin: - if orig is not BuiltinUndefined: #same as 'key in bdict' - self._orig_builtins[key] = orig - del bdict[key] - else: - self._orig_builtins[key] = orig - bdict[key] = value - - def remove_builtin(self, key, orig): - """Remove an added builtin and re-set the original.""" - if orig is BuiltinUndefined: - del builtin_mod.__dict__[key] - else: - builtin_mod.__dict__[key] = orig - - def activate(self): - """Store ipython references in the __builtin__ namespace.""" - - add_builtin = self.add_builtin - for name, func in self.auto_builtins.items(): - add_builtin(name, func) - - def deactivate(self): - """Remove any builtins which might have been added by add_builtins, or - restore overwritten ones to their previous values.""" - remove_builtin = self.remove_builtin - for key, val in self._orig_builtins.items(): - remove_builtin(key, val) - self._orig_builtins.clear() - self._builtins_added = False +""" +A context manager for managing things injected into :mod:`builtins`. +""" +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. +import builtins as builtin_mod + +from traitlets.config.configurable import Configurable + +from traitlets import Instance + + +class __BuiltinUndefined(object): pass +BuiltinUndefined = __BuiltinUndefined() + +class __HideBuiltin(object): pass +HideBuiltin = __HideBuiltin() + + +class BuiltinTrap(Configurable): + + shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', + allow_none=True) + + def __init__(self, shell=None): + super(BuiltinTrap, self).__init__(shell=shell, config=None) + self._orig_builtins = {} + # We define this to track if a single BuiltinTrap is nested. + # Only turn off the trap when the outermost call to __exit__ is made. + self._nested_level = 0 + self.shell = shell + # builtins we always add - if set to HideBuiltin, they will just + # be removed instead of being replaced by something else + self.auto_builtins = {'exit': HideBuiltin, + 'quit': HideBuiltin, + 'get_ipython': self.shell.get_ipython, + } + + def __enter__(self): + if self._nested_level == 0: + self.activate() + self._nested_level += 1 + # I return self, so callers can use add_builtin in a with clause. + return self + + def __exit__(self, type, value, traceback): + if self._nested_level == 1: + self.deactivate() + self._nested_level -= 1 + # Returning False will cause exceptions to propagate + return False + + def add_builtin(self, key, value): + """Add a builtin and save the original.""" + bdict = builtin_mod.__dict__ + orig = bdict.get(key, BuiltinUndefined) + if value is HideBuiltin: + if orig is not BuiltinUndefined: #same as 'key in bdict' + self._orig_builtins[key] = orig + del bdict[key] + else: + self._orig_builtins[key] = orig + bdict[key] = value + + def remove_builtin(self, key, orig): + """Remove an added builtin and re-set the original.""" + if orig is BuiltinUndefined: + del builtin_mod.__dict__[key] + else: + builtin_mod.__dict__[key] = orig + + def activate(self): + """Store ipython references in the __builtin__ namespace.""" + + add_builtin = self.add_builtin + for name, func in self.auto_builtins.items(): + add_builtin(name, func) + + def deactivate(self): + """Remove any builtins which might have been added by add_builtins, or + restore overwritten ones to their previous values.""" + remove_builtin = self.remove_builtin + for key, val in self._orig_builtins.items(): + remove_builtin(key, val) + self._orig_builtins.clear() + self._builtins_added = False diff --git a/contrib/python/ipython/py3/IPython/core/compilerop.py b/contrib/python/ipython/py3/IPython/core/compilerop.py index 68b1d3ed140..50672a19541 100644 --- a/contrib/python/ipython/py3/IPython/core/compilerop.py +++ b/contrib/python/ipython/py3/IPython/core/compilerop.py @@ -1,188 +1,188 @@ -"""Compiler tools with improved interactive support. - -Provides compilation machinery similar to codeop, but with caching support so -we can provide interactive tracebacks. - -Authors -------- -* Robert Kern -* Fernando Perez -* Thomas Kluyver -""" - -# Note: though it might be more natural to name this module 'compiler', that -# name is in the stdlib and name collisions with the stdlib tend to produce -# weird problems (often with third-party tools). - -#----------------------------------------------------------------------------- -# Copyright (C) 2010-2011 The IPython Development Team. -# -# Distributed under the terms of the BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -# Stdlib imports -import __future__ -from ast import PyCF_ONLY_AST -import codeop -import functools -import hashlib -import linecache -import operator -import time -from contextlib import contextmanager - -#----------------------------------------------------------------------------- -# Constants -#----------------------------------------------------------------------------- - -# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h, -# this is used as a bitmask to extract future-related code flags. -PyCF_MASK = functools.reduce(operator.or_, - (getattr(__future__, fname).compiler_flag - for fname in __future__.all_feature_names)) - -#----------------------------------------------------------------------------- -# Local utilities -#----------------------------------------------------------------------------- - -def code_name(code, number=0): - """ Compute a (probably) unique name for code for caching. - - This now expects code to be unicode. - """ - hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest() - # Include the number and 12 characters of the hash in the name. It's - # pretty much impossible that in a single session we'll have collisions - # even with truncated hashes, and the full one makes tracebacks too long - return ''.format(number, hash_digest[:12]) - -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- - -class CachingCompiler(codeop.Compile): - """A compiler that caches code compiled from interactive statements. - """ - - def __init__(self): - codeop.Compile.__init__(self) - - # This is ugly, but it must be done this way to allow multiple - # simultaneous ipython instances to coexist. Since Python itself - # directly accesses the data structures in the linecache module, and - # the cache therein is global, we must work with that data structure. - # We must hold a reference to the original checkcache routine and call - # that in our own check_cache() below, but the special IPython cache - # must also be shared by all IPython instances. If we were to hold - # separate caches (one in each CachingCompiler instance), any call made - # by Python itself to linecache.checkcache() would obliterate the - # cached data from the other IPython instances. - if not hasattr(linecache, '_ipython_cache'): - linecache._ipython_cache = {} - if not hasattr(linecache, '_checkcache_ori'): - linecache._checkcache_ori = linecache.checkcache - # Now, we must monkeypatch the linecache directly so that parts of the - # stdlib that call it outside our control go through our codepath - # (otherwise we'd lose our tracebacks). - linecache.checkcache = check_linecache_ipython - - - def ast_parse(self, source, filename='', symbol='exec'): - """Parse code to an AST with the current compiler flags active. - - Arguments are exactly the same as ast.parse (in the standard library), - and are passed to the built-in compile function.""" - return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) - - def reset_compiler_flags(self): - """Reset compiler flags to default state.""" - # This value is copied from codeop.Compile.__init__, so if that ever - # changes, it will need to be updated. - self.flags = codeop.PyCF_DONT_IMPLY_DEDENT - - @property - def compiler_flags(self): - """Flags currently active in the compilation process. - """ - return self.flags - - def get_code_name(self, raw_code, transformed_code, number): - """Compute filename given the code, and the cell number. - - Parameters - ---------- - raw_code : str - The raw cell code. - transformed_code : str - The executable Python source code to cache and compile. - number : int - A number which forms part of the code's name. Used for the execution - counter. - - Returns - ------- - The computed filename. - """ - return code_name(transformed_code, number) - - def cache(self, transformed_code, number=0, raw_code=None): - """Make a name for a block of code, and cache the code. - - Parameters - ---------- - transformed_code : str - The executable Python source code to cache and compile. - number : int - A number which forms part of the code's name. Used for the execution - counter. - raw_code : str - The raw code before transformation, if None, set to `transformed_code`. - - Returns - ------- - The name of the cached code (as a string). Pass this as the filename - argument to compilation, so that tracebacks are correctly hooked up. - """ - if raw_code is None: - raw_code = transformed_code - - name = self.get_code_name(raw_code, transformed_code, number) - entry = ( - len(transformed_code), - time.time(), - [line + "\n" for line in transformed_code.splitlines()], - name, - ) - linecache.cache[name] = entry - linecache._ipython_cache[name] = entry - return name - - @contextmanager - def extra_flags(self, flags): - ## bits that we'll set to 1 - turn_on_bits = ~self.flags & flags - - - self.flags = self.flags | flags - try: - yield - finally: - # turn off only the bits we turned on so that something like - # __future__ that set flags stays. - self.flags &= ~turn_on_bits - - -def check_linecache_ipython(*args): - """Call linecache.checkcache() safely protecting our cached values. - """ - # First call the original checkcache as intended - linecache._checkcache_ori(*args) - # Then, update back the cache with our data, so that tracebacks related - # to our compiled codes can be produced. - linecache.cache.update(linecache._ipython_cache) +"""Compiler tools with improved interactive support. + +Provides compilation machinery similar to codeop, but with caching support so +we can provide interactive tracebacks. + +Authors +------- +* Robert Kern +* Fernando Perez +* Thomas Kluyver +""" + +# Note: though it might be more natural to name this module 'compiler', that +# name is in the stdlib and name collisions with the stdlib tend to produce +# weird problems (often with third-party tools). + +#----------------------------------------------------------------------------- +# Copyright (C) 2010-2011 The IPython Development Team. +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib imports +import __future__ +from ast import PyCF_ONLY_AST +import codeop +import functools +import hashlib +import linecache +import operator +import time +from contextlib import contextmanager + +#----------------------------------------------------------------------------- +# Constants +#----------------------------------------------------------------------------- + +# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h, +# this is used as a bitmask to extract future-related code flags. +PyCF_MASK = functools.reduce(operator.or_, + (getattr(__future__, fname).compiler_flag + for fname in __future__.all_feature_names)) + +#----------------------------------------------------------------------------- +# Local utilities +#----------------------------------------------------------------------------- + +def code_name(code, number=0): + """ Compute a (probably) unique name for code for caching. + + This now expects code to be unicode. + """ + hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest() + # Include the number and 12 characters of the hash in the name. It's + # pretty much impossible that in a single session we'll have collisions + # even with truncated hashes, and the full one makes tracebacks too long + return ''.format(number, hash_digest[:12]) + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +class CachingCompiler(codeop.Compile): + """A compiler that caches code compiled from interactive statements. + """ + + def __init__(self): + codeop.Compile.__init__(self) + + # This is ugly, but it must be done this way to allow multiple + # simultaneous ipython instances to coexist. Since Python itself + # directly accesses the data structures in the linecache module, and + # the cache therein is global, we must work with that data structure. + # We must hold a reference to the original checkcache routine and call + # that in our own check_cache() below, but the special IPython cache + # must also be shared by all IPython instances. If we were to hold + # separate caches (one in each CachingCompiler instance), any call made + # by Python itself to linecache.checkcache() would obliterate the + # cached data from the other IPython instances. + if not hasattr(linecache, '_ipython_cache'): + linecache._ipython_cache = {} + if not hasattr(linecache, '_checkcache_ori'): + linecache._checkcache_ori = linecache.checkcache + # Now, we must monkeypatch the linecache directly so that parts of the + # stdlib that call it outside our control go through our codepath + # (otherwise we'd lose our tracebacks). + linecache.checkcache = check_linecache_ipython + + + def ast_parse(self, source, filename='', symbol='exec'): + """Parse code to an AST with the current compiler flags active. + + Arguments are exactly the same as ast.parse (in the standard library), + and are passed to the built-in compile function.""" + return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) + + def reset_compiler_flags(self): + """Reset compiler flags to default state.""" + # This value is copied from codeop.Compile.__init__, so if that ever + # changes, it will need to be updated. + self.flags = codeop.PyCF_DONT_IMPLY_DEDENT + + @property + def compiler_flags(self): + """Flags currently active in the compilation process. + """ + return self.flags + + def get_code_name(self, raw_code, transformed_code, number): + """Compute filename given the code, and the cell number. + + Parameters + ---------- + raw_code : str + The raw cell code. + transformed_code : str + The executable Python source code to cache and compile. + number : int + A number which forms part of the code's name. Used for the execution + counter. + + Returns + ------- + The computed filename. + """ + return code_name(transformed_code, number) + + def cache(self, transformed_code, number=0, raw_code=None): + """Make a name for a block of code, and cache the code. + + Parameters + ---------- + transformed_code : str + The executable Python source code to cache and compile. + number : int + A number which forms part of the code's name. Used for the execution + counter. + raw_code : str + The raw code before transformation, if None, set to `transformed_code`. + + Returns + ------- + The name of the cached code (as a string). Pass this as the filename + argument to compilation, so that tracebacks are correctly hooked up. + """ + if raw_code is None: + raw_code = transformed_code + + name = self.get_code_name(raw_code, transformed_code, number) + entry = ( + len(transformed_code), + time.time(), + [line + "\n" for line in transformed_code.splitlines()], + name, + ) + linecache.cache[name] = entry + linecache._ipython_cache[name] = entry + return name + + @contextmanager + def extra_flags(self, flags): + ## bits that we'll set to 1 + turn_on_bits = ~self.flags & flags + + + self.flags = self.flags | flags + try: + yield + finally: + # turn off only the bits we turned on so that something like + # __future__ that set flags stays. + self.flags &= ~turn_on_bits + + +def check_linecache_ipython(*args): + """Call linecache.checkcache() safely protecting our cached values. + """ + # First call the original checkcache as intended + linecache._checkcache_ori(*args) + # Then, update back the cache with our data, so that tracebacks related + # to our compiled codes can be produced. + linecache.cache.update(linecache._ipython_cache) diff --git a/contrib/python/ipython/py3/IPython/core/completer.py b/contrib/python/ipython/py3/IPython/core/completer.py index 484d4c1bf51..776edeb52b7 100644 --- a/contrib/python/ipython/py3/IPython/core/completer.py +++ b/contrib/python/ipython/py3/IPython/core/completer.py @@ -1,2118 +1,2118 @@ -"""Completion for IPython. - -This module started as fork of the rlcompleter module in the Python standard -library. The original enhancements made to rlcompleter have been sent -upstream and were accepted as of Python 2.3, - -This module now support a wide variety of completion mechanism both available -for normal classic Python code, as well as completer for IPython specific -Syntax like magics. - -Latex and Unicode completion -============================ - -IPython and compatible frontends not only can complete your code, but can help -you to input a wide range of characters. In particular we allow you to insert -a unicode character using the tab completion mechanism. - -Forward latex/unicode completion --------------------------------- - -Forward completion allows you to easily type a unicode character using its latex -name, or unicode long description. To do so type a backslash follow by the -relevant name and press tab: - - -Using latex completion: - -.. code:: - - \\alpha - α - -or using unicode completion: - - -.. code:: - - \\greek small letter alpha - α - - -Only valid Python identifiers will complete. Combining characters (like arrow or -dots) are also available, unlike latex they need to be put after the their -counterpart that is to say, `F\\\\vec` is correct, not `\\\\vecF`. - -Some browsers are known to display combining characters incorrectly. - -Backward latex completion -------------------------- - -It is sometime challenging to know how to type a character, if you are using -IPython, or any compatible frontend you can prepend backslash to the character -and press `` to expand it to its latex form. - -.. code:: - - \\α - \\alpha - - -Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. - - -Experimental -============ - -Starting with IPython 6.0, this module can make use of the Jedi library to -generate completions both using static analysis of the code, and dynamically -inspecting multiple namespaces. Jedi is an autocompletion and static analysis -for Python. The APIs attached to this new mechanism is unstable and will -raise unless use in an :any:`provisionalcompleter` context manager. - -You will find that the following are experimental: - - - :any:`provisionalcompleter` - - :any:`IPCompleter.completions` - - :any:`Completion` - - :any:`rectify_completions` - -.. note:: - - better name for :any:`rectify_completions` ? - -We welcome any feedback on these new API, and we also encourage you to try this -module in debug mode (start IPython with ``--Completer.debug=True``) in order -to have extra logging information if :any:`jedi` is crashing, or if current -IPython completer pending deprecations are returning results not yet handled -by :any:`jedi` - -Using Jedi for tab completion allow snippets like the following to work without -having to execute any code: - - >>> myvar = ['hello', 42] - ... myvar[1].bi - -Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` -option. - -Be sure to update :any:`jedi` to the latest stable version or to try the -current development version to get better completions. -""" - - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. -# -# Some of this code originated from rlcompleter in the Python standard library -# Copyright (C) 2001 Python Software Foundation, www.python.org - - -import builtins as builtin_mod -import glob -import inspect -import itertools -import keyword -import os -import re -import string -import sys -import time -import unicodedata -import warnings -from contextlib import contextmanager -from importlib import import_module -from types import SimpleNamespace -from typing import Iterable, Iterator, List, Tuple - -from IPython.core.error import TryNext -from IPython.core.inputtransformer2 import ESC_MAGIC -from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol -from IPython.core.oinspect import InspectColors -from IPython.utils import generics -from IPython.utils.dir2 import dir2, get_real_method -from IPython.utils.process import arg_split -from traitlets import Bool, Enum, Int, observe -from traitlets.config.configurable import Configurable - -import __main__ - -# skip module docstests -skip_doctest = True - -try: - import jedi - jedi.settings.case_insensitive_completion = False - import jedi.api.helpers - import jedi.api.classes - JEDI_INSTALLED = True -except ImportError: - JEDI_INSTALLED = False -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- - -# Public API -__all__ = ['Completer','IPCompleter'] - -if sys.platform == 'win32': - PROTECTABLES = ' ' -else: - PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&' - -# Protect against returning an enormous number of completions which the frontend -# may have trouble processing. -MATCHES_LIMIT = 500 - - -class Sentinel: - def __repr__(self): - return "" - - -_deprecation_readline_sentinel = Sentinel() - - -class ProvisionalCompleterWarning(FutureWarning): - """ - Exception raise by an experimental feature in this module. - - Wrap code in :any:`provisionalcompleter` context manager if you - are certain you want to use an unstable feature. - """ - pass - -warnings.filterwarnings('error', category=ProvisionalCompleterWarning) - -@contextmanager -def provisionalcompleter(action='ignore'): - """ - - - This context manager has to be used in any place where unstable completer - behavior and API may be called. - - >>> with provisionalcompleter(): - ... completer.do_experimental_things() # works - - >>> completer.do_experimental_things() # raises. - - .. note:: - - Unstable - - By using this context manager you agree that the API in use may change - without warning, and that you won't complain if they do so. - - You also understand that, if the API is not to your liking, you should report - a bug to explain your use case upstream. - - We'll be happy to get your feedback, feature requests, and improvements on - any of the unstable APIs! - """ - with warnings.catch_warnings(): - warnings.filterwarnings(action, category=ProvisionalCompleterWarning) - yield - - -def has_open_quotes(s): - """Return whether a string has open quotes. - - This simply counts whether the number of quote characters of either type in - the string is odd. - - Returns - ------- - If there is an open quote, the quote character is returned. Else, return - False. - """ - # We check " first, then ', so complex cases with nested quotes will get - # the " to take precedence. - if s.count('"') % 2: - return '"' - elif s.count("'") % 2: - return "'" - else: - return False - - -def protect_filename(s, protectables=PROTECTABLES): - """Escape a string to protect certain characters.""" - if set(s) & set(protectables): - if sys.platform == "win32": - return '"' + s + '"' - else: - return "".join(("\\" + c if c in protectables else c) for c in s) - else: - return s - - -def expand_user(path:str) -> Tuple[str, bool, str]: - """Expand ``~``-style usernames in strings. - - This is similar to :func:`os.path.expanduser`, but it computes and returns - extra information that will be useful if the input was being used in - computing completions, and you wish to return the completions with the - original '~' instead of its expanded value. - - Parameters - ---------- - path : str - String to be expanded. If no ~ is present, the output is the same as the - input. - - Returns - ------- - newpath : str - Result of ~ expansion in the input path. - tilde_expand : bool - Whether any expansion was performed or not. - tilde_val : str - The value that ~ was replaced with. - """ - # Default values - tilde_expand = False - tilde_val = '' - newpath = path - - if path.startswith('~'): - tilde_expand = True - rest = len(path)-1 - newpath = os.path.expanduser(path) - if rest: - tilde_val = newpath[:-rest] - else: - tilde_val = newpath - - return newpath, tilde_expand, tilde_val - - -def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str: - """Does the opposite of expand_user, with its outputs. - """ - if tilde_expand: - return path.replace(tilde_val, '~') - else: - return path - - -def completions_sorting_key(word): - """key for sorting completions - - This does several things: - - - Demote any completions starting with underscores to the end - - Insert any %magic and %%cellmagic completions in the alphabetical order - by their name - """ - prio1, prio2 = 0, 0 - - if word.startswith('__'): - prio1 = 2 - elif word.startswith('_'): - prio1 = 1 - - if word.endswith('='): - prio1 = -1 - - if word.startswith('%%'): - # If there's another % in there, this is something else, so leave it alone - if not "%" in word[2:]: - word = word[2:] - prio2 = 2 - elif word.startswith('%'): - if not "%" in word[1:]: - word = word[1:] - prio2 = 1 - - return prio1, word, prio2 - - -class _FakeJediCompletion: - """ - This is a workaround to communicate to the UI that Jedi has crashed and to - report a bug. Will be used only id :any:`IPCompleter.debug` is set to true. - - Added in IPython 6.0 so should likely be removed for 7.0 - - """ - - def __init__(self, name): - - self.name = name - self.complete = name - self.type = 'crashed' - self.name_with_symbols = name - self.signature = '' - self._origin = 'fake' - - def __repr__(self): - return '' - - -class Completion: - """ - Completion object used and return by IPython completers. - - .. warning:: - - Unstable - - This function is unstable, API may change without warning. - It will also raise unless use in proper context manager. - - This act as a middle ground :any:`Completion` object between the - :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion - object. While Jedi need a lot of information about evaluator and how the - code should be ran/inspected, PromptToolkit (and other frontend) mostly - need user facing information. - - - Which range should be replaced replaced by what. - - Some metadata (like completion type), or meta information to displayed to - the use user. - - For debugging purpose we can also store the origin of the completion (``jedi``, - ``IPython.python_matches``, ``IPython.magics_matches``...). - """ - - __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] - - def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: - warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) - - self.start = start - self.end = end - self.text = text - self.type = type - self.signature = signature - self._origin = _origin - - def __repr__(self): - return '' % \ - (self.start, self.end, self.text, self.type or '?', self.signature or '?') - - def __eq__(self, other)->Bool: - """ - Equality and hash do not hash the type (as some completer may not be - able to infer the type), but are use to (partially) de-duplicate - completion. - - Completely de-duplicating completion is a bit tricker that just - comparing as it depends on surrounding text, which Completions are not - aware of. - """ - return self.start == other.start and \ - self.end == other.end and \ - self.text == other.text - - def __hash__(self): - return hash((self.start, self.end, self.text)) - - -_IC = Iterable[Completion] - - -def _deduplicate_completions(text: str, completions: _IC)-> _IC: - """ - Deduplicate a set of completions. - - .. warning:: - - Unstable - - This function is unstable, API may change without warning. - - Parameters - ---------- - text: str - text that should be completed. - completions: Iterator[Completion] - iterator over the completions to deduplicate - - Yields - ------ - `Completions` objects - - - Completions coming from multiple sources, may be different but end up having - the same effect when applied to ``text``. If this is the case, this will - consider completions as equal and only emit the first encountered. - - Not folded in `completions()` yet for debugging purpose, and to detect when - the IPython completer does return things that Jedi does not, but should be - at some point. - """ - completions = list(completions) - if not completions: - return - - new_start = min(c.start for c in completions) - new_end = max(c.end for c in completions) - - seen = set() - for c in completions: - new_text = text[new_start:c.start] + c.text + text[c.end:new_end] - if new_text not in seen: - yield c - seen.add(new_text) - - -def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: - """ - Rectify a set of completions to all have the same ``start`` and ``end`` - - .. warning:: - - Unstable - - This function is unstable, API may change without warning. - It will also raise unless use in proper context manager. - - Parameters - ---------- - text: str - text that should be completed. - completions: Iterator[Completion] - iterator over the completions to rectify - - - :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though - the Jupyter Protocol requires them to behave like so. This will readjust - the completion to have the same ``start`` and ``end`` by padding both - extremities with surrounding text. - - During stabilisation should support a ``_debug`` option to log which - completion are return by the IPython completer and not found in Jedi in - order to make upstream bug report. - """ - warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) - - completions = list(completions) - if not completions: - return - starts = (c.start for c in completions) - ends = (c.end for c in completions) - - new_start = min(starts) - new_end = max(ends) - - seen_jedi = set() - seen_python_matches = set() - for c in completions: - new_text = text[new_start:c.start] + c.text + text[c.end:new_end] - if c._origin == 'jedi': - seen_jedi.add(new_text) - elif c._origin == 'IPCompleter.python_matches': - seen_python_matches.add(new_text) - yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature) - diff = seen_python_matches.difference(seen_jedi) - if diff and _debug: - print('IPython.python matches have extras:', diff) - - -if sys.platform == 'win32': - DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?' -else: - DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?' - -GREEDY_DELIMS = ' =\r\n' - - -class CompletionSplitter(object): - """An object to split an input line in a manner similar to readline. - - By having our own implementation, we can expose readline-like completion in - a uniform manner to all frontends. This object only needs to be given the - line of text to be split and the cursor position on said line, and it - returns the 'word' to be completed on at the cursor after splitting the - entire line. - - What characters are used as splitting delimiters can be controlled by - setting the ``delims`` attribute (this is a property that internally - automatically builds the necessary regular expression)""" - - # Private interface - - # A string of delimiter characters. The default value makes sense for - # IPython's most typical usage patterns. - _delims = DELIMS - - # The expression (a normal string) to be compiled into a regular expression - # for actual splitting. We store it as an attribute mostly for ease of - # debugging, since this type of code can be so tricky to debug. - _delim_expr = None - - # The regular expression that does the actual splitting - _delim_re = None - - def __init__(self, delims=None): - delims = CompletionSplitter._delims if delims is None else delims - self.delims = delims - - @property - def delims(self): - """Return the string of delimiter characters.""" - return self._delims - - @delims.setter - def delims(self, delims): - """Set the delimiters for line splitting.""" - expr = '[' + ''.join('\\'+ c for c in delims) + ']' - self._delim_re = re.compile(expr) - self._delims = delims - self._delim_expr = expr - - def split_line(self, line, cursor_pos=None): - """Split a line of text with a cursor at the given position. - """ - l = line if cursor_pos is None else line[:cursor_pos] - return self._delim_re.split(l)[-1] - - - -class Completer(Configurable): - - greedy = Bool(False, - help="""Activate greedy completion - PENDING DEPRECTION. this is now mostly taken care of with Jedi. - - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - """ - ).tag(config=True) - - use_jedi = Bool(default_value=JEDI_INSTALLED, - help="Experimental: Use Jedi to generate autocompletions. " - "Default to True if jedi is installed.").tag(config=True) - - jedi_compute_type_timeout = Int(default_value=400, - help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types. - Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt - performance by preventing jedi to build its cache. - """).tag(config=True) - - debug = Bool(default_value=False, - help='Enable debug for the Completer. Mostly print extra ' - 'information for experimental jedi integration.')\ - .tag(config=True) - - backslash_combining_completions = Bool(True, - help="Enable unicode completions, e.g. \\alpha . " - "Includes completion of latex commands, unicode names, and expanding " - "unicode characters back to latex commands.").tag(config=True) - - - - def __init__(self, namespace=None, global_namespace=None, **kwargs): - """Create a new completer for the command line. - - Completer(namespace=ns, global_namespace=ns2) -> completer instance. - - If unspecified, the default namespace where completions are performed - is __main__ (technically, __main__.__dict__). Namespaces should be - given as dictionaries. - - An optional second namespace can be given. This allows the completer - to handle cases where both the local and global scopes need to be - distinguished. - """ - - # Don't bind to namespace quite yet, but flag whether the user wants a - # specific namespace or to use __main__.__dict__. This will allow us - # to bind to __main__.__dict__ at completion time, not now. - if namespace is None: - self.use_main_ns = True - else: - self.use_main_ns = False - self.namespace = namespace - - # The global namespace, if given, can be bound directly - if global_namespace is None: - self.global_namespace = {} - else: - self.global_namespace = global_namespace - - self.custom_matchers = [] - - super(Completer, self).__init__(**kwargs) - - def complete(self, text, state): - """Return the next possible completion for 'text'. - - This is called successively with state == 0, 1, 2, ... until it - returns None. The completion should begin with 'text'. - - """ - if self.use_main_ns: - self.namespace = __main__.__dict__ - - if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) - try: - return self.matches[state] - except IndexError: - return None - - def global_matches(self, text): - """Compute matches when text is a simple name. - - Return a list of all keywords, built-in functions and names currently - defined in self.namespace or self.global_namespace that match. - - """ - matches = [] - match_append = matches.append - n = len(text) - for lst in [keyword.kwlist, - builtin_mod.__dict__.keys(), - self.namespace.keys(), - self.global_namespace.keys()]: - for word in lst: - if word[:n] == text and word != "__builtins__": - match_append(word) - - snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z") - for lst in [self.namespace.keys(), - self.global_namespace.keys()]: - shortened = {"_".join([sub[0] for sub in word.split('_')]) : word - for word in lst if snake_case_re.match(word)} - for word in shortened.keys(): - if word[:n] == text and word != "__builtins__": - match_append(shortened[word]) - return matches - - def attr_matches(self, text): - """Compute matches when text contains a dot. - - Assuming the text is of the form NAME.NAME....[NAME], and is - evaluatable in self.namespace or self.global_namespace, it will be - evaluated and its attributes (as revealed by dir()) are used as - possible completions. (For class instances, class members are - also considered.) - - WARNING: this can still invoke arbitrary C code, if an object - with a __getattr__ hook is evaluated. - - """ - - # Another option, seems to work great. Catches things like ''. - m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) - - if m: - expr, attr = m.group(1, 3) - elif self.greedy: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return [] - expr, attr = m2.group(1,2) - else: - return [] - - try: - obj = eval(expr, self.namespace) - except: - try: - obj = eval(expr, self.global_namespace) - except: - return [] - - if self.limit_to__all__ and hasattr(obj, '__all__'): - words = get__all__entries(obj) - else: - words = dir2(obj) - - try: - words = generics.complete_object(obj, words) - except TryNext: - pass - except AssertionError: - raise - except Exception: - # Silence errors from completion function - #raise # dbg - pass - # Build match list to return - n = len(attr) - return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] - - -def get__all__entries(obj): - """returns the strings in the __all__ attribute""" - try: - words = getattr(obj, '__all__') - except: - return [] - - return [w for w in words if isinstance(w, str)] - - -def match_dict_keys(keys: List[str], prefix: str, delims: str): - """Used by dict_key_matches, matching the prefix to a list of keys - - Parameters - ========== - keys: - list of keys in dictionary currently being completed. - prefix: - Part of the text already typed by the user. e.g. `mydict[b'fo` - delims: - String of delimiters to consider when finding the current key. - - Returns - ======= - - A tuple of three elements: ``quote``, ``token_start``, ``matched``, with - ``quote`` being the quote that need to be used to close current string. - ``token_start`` the position where the replacement should start occurring, - ``matches`` a list of replacement/completion - - """ - if not prefix: - return None, 0, [repr(k) for k in keys - if isinstance(k, (str, bytes))] - quote_match = re.search('["\']', prefix) - quote = quote_match.group() - try: - prefix_str = eval(prefix + quote, {}) - except Exception: - return None, 0, [] - - pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' - token_match = re.search(pattern, prefix, re.UNICODE) - token_start = token_match.start() - token_prefix = token_match.group() - - matched = [] - for key in keys: - try: - if not key.startswith(prefix_str): - continue - except (AttributeError, TypeError, UnicodeError): - # Python 3+ TypeError on b'a'.startswith('a') or vice-versa - continue - - # reformat remainder of key to begin with prefix - rem = key[len(prefix_str):] - # force repr wrapped in ' - rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') - if rem_repr.startswith('u') and prefix[0] not in 'uU': - # Found key is unicode, but prefix is Py2 string. - # Therefore attempt to interpret key as string. - try: - rem_repr = repr(rem.encode('ascii') + '"') - except UnicodeEncodeError: - continue - - rem_repr = rem_repr[1 + rem_repr.index("'"):-2] - if quote == '"': - # The entered prefix is quoted with ", - # but the match is quoted with '. - # A contained " hence needs escaping for comparison: - rem_repr = rem_repr.replace('"', '\\"') - - # then reinsert prefix from start of token - matched.append('%s%s' % (token_prefix, rem_repr)) - return quote, token_start, matched - - -def cursor_to_position(text:str, line:int, column:int)->int: - """ - - Convert the (line,column) position of the cursor in text to an offset in a - string. - - Parameters - ---------- - - text : str - The text in which to calculate the cursor offset - line : int - Line of the cursor; 0-indexed - column : int - Column of the cursor 0-indexed - - Return - ------ - Position of the cursor in ``text``, 0-indexed. - - See Also - -------- - position_to_cursor: reciprocal of this function - - """ - lines = text.split('\n') - assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines))) - - return sum(len(l) + 1 for l in lines[:line]) + column - -def position_to_cursor(text:str, offset:int)->Tuple[int, int]: - """ - Convert the position of the cursor in text (0 indexed) to a line - number(0-indexed) and a column number (0-indexed) pair - - Position should be a valid position in ``text``. - - Parameters - ---------- - - text : str - The text in which to calculate the cursor offset - offset : int - Position of the cursor in ``text``, 0-indexed. - - Return - ------ - (line, column) : (int, int) - Line of the cursor; 0-indexed, column of the cursor 0-indexed - - - See Also - -------- - cursor_to_position : reciprocal of this function - - - """ - - assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text)) - - before = text[:offset] - blines = before.split('\n') # ! splitnes trim trailing \n - line = before.count('\n') - col = len(blines[-1]) - return line, col - - -def _safe_isinstance(obj, module, class_name): - """Checks if obj is an instance of module.class_name if loaded - """ - return (module in sys.modules and - isinstance(obj, getattr(import_module(module), class_name))) - - -def back_unicode_name_matches(text): - u"""Match unicode characters back to unicode name - - This does ``☃`` -> ``\\snowman`` - - Note that snowman is not a valid python3 combining character but will be expanded. - Though it will not recombine back to the snowman character by the completion machinery. - - This will not either back-complete standard sequences like \\n, \\b ... - - Used on Python 3 only. - """ - if len(text)<2: - return u'', () - maybe_slash = text[-2] - if maybe_slash != '\\': - return u'', () - - char = text[-1] - # no expand on quote for completion in strings. - # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () - try : - unic = unicodedata.name(char) - return '\\'+char,['\\'+unic] - except KeyError: - pass - return u'', () - -def back_latex_name_matches(text:str): - """Match latex characters back to unicode name - - This does ``\\ℵ`` -> ``\\aleph`` - - Used on Python 3 only. - """ - if len(text)<2: - return u'', () - maybe_slash = text[-2] - if maybe_slash != '\\': - return u'', () - - - char = text[-1] - # no expand on quote for completion in strings. - # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () - try : - latex = reverse_latex_symbol[char] - # '\\' replace the \ as well - return '\\'+char,[latex] - except KeyError: - pass - return u'', () - - -def _formatparamchildren(parameter) -> str: - """ - Get parameter name and value from Jedi Private API - - Jedi does not expose a simple way to get `param=value` from its API. - - Parameter - ========= - - parameter: - Jedi's function `Param` - - Returns - ======= - - A string like 'a', 'b=1', '*args', '**kwargs' - - - """ - description = parameter.description - if not description.startswith('param '): - raise ValueError('Jedi function parameter description have change format.' - 'Expected "param ...", found %r".' % description) - return description[6:] - -def _make_signature(completion)-> str: - """ - Make the signature from a jedi completion - - Parameter - ========= - - completion: jedi.Completion - object does not complete a function type - - Returns - ======= - - a string consisting of the function signature, with the parenthesis but - without the function name. example: - `(a, *args, b=1, **kwargs)` - - """ - - return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for p in completion.params) if f]) - # it looks like this might work on jedi 0.17 - if hasattr(completion, 'get_signatures'): - signatures = completion.get_signatures() - if not signatures: - return '(?)' - - c0 = completion.get_signatures()[0] - return '('+c0.to_string().split('(', maxsplit=1)[1] - - return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures() - for p in signature.defined_names()) if f]) - -class IPCompleter(Completer): - """Extension of the completer class with IPython-specific features""" - - _names = None - - @observe('greedy') - def _greedy_changed(self, change): - """update the splitter and readline delims when greedy is changed""" - if change['new']: - self.splitter.delims = GREEDY_DELIMS - else: - self.splitter.delims = DELIMS - - dict_keys_only = Bool(False, - help="""Whether to show dict key matches only""") - - merge_completions = Bool(True, - help="""Whether to merge completion results into a single list - - If False, only the completion results from the first non-empty - completer will be returned. - """ - ).tag(config=True) - omit__names = Enum((0,1,2), default_value=2, - help="""Instruct the completer to omit private method names - - Specifically, when completing on ``object.``. - - When 2 [default]: all names that start with '_' will be excluded. - - When 1: all 'magic' names (``__foo__``) will be excluded. - - When 0: nothing will be excluded. - """ - ).tag(config=True) - limit_to__all__ = Bool(False, - help=""" - DEPRECATED as of version 5.0. - - Instruct the completer to use __all__ for the completion - - Specifically, when completing on ``object.``. - - When True: only those names in obj.__all__ will be included. - - When False [default]: the __all__ attribute is ignored - """, - ).tag(config=True) - - @observe('limit_to__all__') - def _limit_to_all_changed(self, change): - warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration ' - 'value has been deprecated since IPython 5.0, will be made to have ' - 'no effects and then removed in future version of IPython.', - UserWarning) - - def __init__(self, shell=None, namespace=None, global_namespace=None, - use_readline=_deprecation_readline_sentinel, config=None, **kwargs): - """IPCompleter() -> completer - - Return a completer object. - - Parameters - ---------- - - shell - a pointer to the ipython shell itself. This is needed - because this completer knows about magic functions, and those can - only be accessed via the ipython instance. - - namespace : dict, optional - an optional dict where completions are performed. - - global_namespace : dict, optional - secondary optional dict for completions, to - handle cases (such as IPython embedded inside functions) where - both Python scopes are visible. - - use_readline : bool, optional - DEPRECATED, ignored since IPython 6.0, will have no effects - """ - - self.magic_escape = ESC_MAGIC - self.splitter = CompletionSplitter() - - if use_readline is not _deprecation_readline_sentinel: - warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.', - DeprecationWarning, stacklevel=2) - - # _greedy_changed() depends on splitter and readline being defined: - Completer.__init__(self, namespace=namespace, global_namespace=global_namespace, - config=config, **kwargs) - - # List where completion matches will be stored - self.matches = [] - self.shell = shell - # Regexp to split filenames with spaces in them - self.space_name_re = re.compile(r'([^\\] )') - # Hold a local ref. to glob.glob for speed - self.glob = glob.glob - - # Determine if we are running on 'dumb' terminals, like (X)Emacs - # buffers, to avoid completion problems. - term = os.environ.get('TERM','xterm') - self.dumb_terminal = term in ['dumb','emacs'] - - # Special handling of backslashes needed in win32 platforms - if sys.platform == "win32": - self.clean_glob = self._clean_glob_win32 - else: - self.clean_glob = self._clean_glob - - #regexp to parse docstring for function signature - self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*') - self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)') - #use this if positional argument name is also needed - #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)') - - self.magic_arg_matchers = [ - self.magic_config_matches, - self.magic_color_matches, - ] - - # This is set externally by InteractiveShell - self.custom_completers = None - - @property - def matchers(self): - """All active matcher routines for completion""" - if self.dict_keys_only: - return [self.dict_key_matches] - - if self.use_jedi: - return [ - *self.custom_matchers, - self.dict_key_matches, - self.file_matches, - self.magic_matches, - ] - else: - return [ - *self.custom_matchers, - self.dict_key_matches, - self.python_matches, - self.file_matches, - self.magic_matches, - self.python_func_kw_matches, - ] - - def all_completions(self, text) -> List[str]: - """ - Wrapper around the completion methods for the benefit of emacs. - """ - prefix = text.rpartition('.')[0] - with provisionalcompleter(): - return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text - for c in self.completions(text, len(text))] - - return self.complete(text)[1] - - def _clean_glob(self, text): - return self.glob("%s*" % text) - - def _clean_glob_win32(self,text): - return [f.replace("\\","/") - for f in self.glob("%s*" % text)] - - def file_matches(self, text): - """Match filenames, expanding ~USER type strings. - - Most of the seemingly convoluted logic in this completer is an - attempt to handle filenames with spaces in them. And yet it's not - quite perfect, because Python's readline doesn't expose all of the - GNU readline details needed for this to be done correctly. - - For a filename with a space in it, the printed completions will be - only the parts after what's already been typed (instead of the - full completions, as is normally done). I don't think with the - current (as of Python 2.3) Python readline it's possible to do - better.""" - - # chars that require escaping with backslash - i.e. chars - # that readline treats incorrectly as delimiters, but we - # don't want to treat as delimiters in filename matching - # when escaped with backslash - if text.startswith('!'): - text = text[1:] - text_prefix = u'!' - else: - text_prefix = u'' - - text_until_cursor = self.text_until_cursor - # track strings with open quotes - open_quotes = has_open_quotes(text_until_cursor) - - if '(' in text_until_cursor or '[' in text_until_cursor: - lsplit = text - else: - try: - # arg_split ~ shlex.split, but with unicode bugs fixed by us - lsplit = arg_split(text_until_cursor)[-1] - except ValueError: - # typically an unmatched ", or backslash without escaped char. - if open_quotes: - lsplit = text_until_cursor.split(open_quotes)[-1] - else: - return [] - except IndexError: - # tab pressed on empty line - lsplit = "" - - if not open_quotes and lsplit != protect_filename(lsplit): - # if protectables are found, do matching on the whole escaped name - has_protectables = True - text0,text = text,lsplit - else: - has_protectables = False - text = os.path.expanduser(text) - - if text == "": - return [text_prefix + protect_filename(f) for f in self.glob("*")] - - # Compute the matches from the filesystem - if sys.platform == 'win32': - m0 = self.clean_glob(text) - else: - m0 = self.clean_glob(text.replace('\\', '')) - - if has_protectables: - # If we had protectables, we need to revert our changes to the - # beginning of filename so that we don't double-write the part - # of the filename we have so far - len_lsplit = len(lsplit) - matches = [text_prefix + text0 + - protect_filename(f[len_lsplit:]) for f in m0] - else: - if open_quotes: - # if we have a string with an open quote, we don't need to - # protect the names beyond the quote (and we _shouldn't_, as - # it would cause bugs when the filesystem call is made). - matches = m0 if sys.platform == "win32" else\ - [protect_filename(f, open_quotes) for f in m0] - else: - matches = [text_prefix + - protect_filename(f) for f in m0] - - # Mark directories in input list by appending '/' to their names. - return [x+'/' if os.path.isdir(x) else x for x in matches] - - def magic_matches(self, text): - """Match magics""" - # Get all shell magics now rather than statically, so magics loaded at - # runtime show up too. - lsm = self.shell.magics_manager.lsmagic() - line_magics = lsm['line'] - cell_magics = lsm['cell'] - pre = self.magic_escape - pre2 = pre+pre - - explicit_magic = text.startswith(pre) - - # Completion logic: - # - user gives %%: only do cell magics - # - user gives %: do both line and cell magics - # - no prefix: do both - # In other words, line magics are skipped if the user gives %% explicitly - # - # We also exclude magics that match any currently visible names: - # https://github.com/ipython/ipython/issues/4877, unless the user has - # typed a %: - # https://github.com/ipython/ipython/issues/10754 - bare_text = text.lstrip(pre) - global_matches = self.global_matches(bare_text) - if not explicit_magic: - def matches(magic): - """ - Filter magics, in particular remove magics that match - a name present in global namespace. - """ - return ( magic.startswith(bare_text) and - magic not in global_matches ) - else: - def matches(magic): - return magic.startswith(bare_text) - - comp = [ pre2+m for m in cell_magics if matches(m)] - if not text.startswith(pre2): - comp += [ pre+m for m in line_magics if matches(m)] - - return comp - - def magic_config_matches(self, text:str) -> List[str]: - """ Match class names and attributes for %config magic """ - texts = text.strip().split() - - if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'): - # get all configuration classes - classes = sorted(set([ c for c in self.shell.configurables - if c.__class__.class_traits(config=True) - ]), key=lambda x: x.__class__.__name__) - classnames = [ c.__class__.__name__ for c in classes ] - - # return all classnames if config or %config is given - if len(texts) == 1: - return classnames - - # match classname - classname_texts = texts[1].split('.') - classname = classname_texts[0] - classname_matches = [ c for c in classnames - if c.startswith(classname) ] - - # return matched classes or the matched class with attributes - if texts[1].find('.') < 0: - return classname_matches - elif len(classname_matches) == 1 and \ - classname_matches[0] == classname: - cls = classes[classnames.index(classname)].__class__ - help = cls.class_get_help() - # strip leading '--' from cl-args: - help = re.sub(re.compile(r'^--', re.MULTILINE), '', help) - return [ attr.split('=')[0] - for attr in help.strip().splitlines() - if attr.startswith(texts[1]) ] - return [] - - def magic_color_matches(self, text:str) -> List[str] : - """ Match color schemes for %colors magic""" - texts = text.split() - if text.endswith(' '): - # .split() strips off the trailing whitespace. Add '' back - # so that: '%colors ' -> ['%colors', ''] - texts.append('') - - if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'): - prefix = texts[1] - return [ color for color in InspectColors.keys() - if color.startswith(prefix) ] - return [] - - def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): - """ - - Return a list of :any:`jedi.api.Completions` object from a ``text`` and - cursor position. - - Parameters - ---------- - cursor_column : int - column position of the cursor in ``text``, 0-indexed. - cursor_line : int - line position of the cursor in ``text``, 0-indexed - text : str - text to complete - - Debugging - --------- - - If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion` - object containing a string with the Jedi debug information attached. - """ - namespaces = [self.namespace] - if self.global_namespace is not None: - namespaces.append(self.global_namespace) - - completion_filter = lambda x:x - offset = cursor_to_position(text, cursor_line, cursor_column) - # filter output if we are completing for object members - if offset: - pre = text[offset-1] - if pre == '.': - if self.omit__names == 2: - completion_filter = lambda c:not c.name.startswith('_') - elif self.omit__names == 1: - completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__')) - elif self.omit__names == 0: - completion_filter = lambda x:x - else: - raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names)) - - interpreter = jedi.Interpreter(text[:offset], namespaces, column=cursor_column, line=cursor_line + 1) - try_jedi = True - - try: - # find the first token in the current tree -- if it is a ' or " then we are in a string - completing_string = False - try: - first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) - except StopIteration: - pass - else: - # note the value may be ', ", or it may also be ''' or """, or - # in some cases, """what/you/typed..., but all of these are - # strings. - completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - - # if we are in a string jedi is likely not the right candidate for - # now. Skip it. - try_jedi = not completing_string - except Exception as e: - # many of things can go wrong, we are using private API just don't crash. - if self.debug: - print("Error detecting if completing a non-finished string :", e, '|') - - if not try_jedi: - return [] - try: - return filter(completion_filter, interpreter.completions()) - except Exception as e: - if self.debug: - return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] - else: - return [] - - def python_matches(self, text): - """Match attributes or global python names""" - if "." in text: - try: - matches = self.attr_matches(text) - if text.endswith('.') and self.omit__names: - if self.omit__names == 1: - # true if txt is _not_ a __ name, false otherwise: - no__name = (lambda txt: - re.match(r'.*\.__.*?__',txt) is None) - else: - # true if txt is _not_ a _ name, false otherwise: - no__name = (lambda txt: - re.match(r'\._.*?',txt[txt.rindex('.'):]) is None) - matches = filter(no__name, matches) - except NameError: - # catches . - matches = [] - else: - matches = self.global_matches(text) - return matches - - def _default_arguments_from_docstring(self, doc): - """Parse the first line of docstring for call signature. - - Docstring should be of the form 'min(iterable[, key=func])\n'. - It can also parse cython docstring of the form - 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'. - """ - if doc is None: - return [] - - #care only the firstline - line = doc.lstrip().splitlines()[0] - - #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*') - #'min(iterable[, key=func])\n' -> 'iterable[, key=func]' - sig = self.docstring_sig_re.search(line) - if sig is None: - return [] - # iterable[, key=func]' -> ['iterable[' ,' key=func]'] - sig = sig.groups()[0].split(',') - ret = [] - for s in sig: - #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)') - ret += self.docstring_kwd_re.findall(s) - return ret - - def _default_arguments(self, obj): - """Return the list of default arguments of obj if it is callable, - or empty list otherwise.""" - call_obj = obj - ret = [] - if inspect.isbuiltin(obj): - pass - elif not (inspect.isfunction(obj) or inspect.ismethod(obj)): - if inspect.isclass(obj): - #for cython embedsignature=True the constructor docstring - #belongs to the object itself not __init__ - ret += self._default_arguments_from_docstring( - getattr(obj, '__doc__', '')) - # for classes, check for __init__,__new__ - call_obj = (getattr(obj, '__init__', None) or - getattr(obj, '__new__', None)) - # for all others, check if they are __call__able - elif hasattr(obj, '__call__'): - call_obj = obj.__call__ - ret += self._default_arguments_from_docstring( - getattr(call_obj, '__doc__', '')) - - _keeps = (inspect.Parameter.KEYWORD_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD) - - try: - sig = inspect.signature(obj) - ret.extend(k for k, v in sig.parameters.items() if - v.kind in _keeps) - except ValueError: - pass - - return list(set(ret)) - - def python_func_kw_matches(self,text): - """Match named parameters (kwargs) of the last open function""" - - if "." in text: # a parameter cannot be dotted - return [] - try: regexp = self.__funcParamsRegex - except AttributeError: - regexp = self.__funcParamsRegex = re.compile(r''' - '.*?(?,a=1)", the candidate is "foo" - tokens = regexp.findall(self.text_until_cursor) - iterTokens = reversed(tokens); openPar = 0 - - for token in iterTokens: - if token == ')': - openPar -= 1 - elif token == '(': - openPar += 1 - if openPar > 0: - # found the last unclosed parenthesis - break - else: - return [] - # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" ) - ids = [] - isId = re.compile(r'\w+$').match - - while True: - try: - ids.append(next(iterTokens)) - if not isId(ids[-1]): - ids.pop(); break - if not next(iterTokens) == '.': - break - except StopIteration: - break - - # Find all named arguments already assigned to, as to avoid suggesting - # them again - usedNamedArgs = set() - par_level = -1 - for token, next_token in zip(tokens, tokens[1:]): - if token == '(': - par_level += 1 - elif token == ')': - par_level -= 1 - - if par_level != 0: - continue - - if next_token != '=': - continue - - usedNamedArgs.add(token) - - argMatches = [] - try: - callableObj = '.'.join(ids[::-1]) - namedArgs = self._default_arguments(eval(callableObj, - self.namespace)) - - # Remove used named arguments from the list, no need to show twice - for namedArg in set(namedArgs) - usedNamedArgs: - if namedArg.startswith(text): - argMatches.append(u"%s=" %namedArg) - except: - pass - - return argMatches - - def dict_key_matches(self, text): - "Match string keys in a dictionary, after e.g. 'foo[' " - def get_keys(obj): - # Objects can define their own completions by defining an - # _ipy_key_completions_() method. - method = get_real_method(obj, '_ipython_key_completions_') - if method is not None: - return method() - - # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): - try: - return list(obj.keys()) - except Exception: - return [] - elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ - _safe_isinstance(obj, 'numpy', 'void'): - return obj.dtype.names or [] - return [] - - try: - regexps = self.__dict_key_regexps - except AttributeError: - dict_key_re_fmt = r'''(?x) - ( # match dict-referring expression wrt greedy setting - %s - ) - \[ # open bracket - \s* # and optional whitespace - ([uUbB]? # string prefix (r not handled) - (?: # unclosed string - '(?:[^']|(? key_start: - leading = '' - else: - leading = text[text_start:completion_start] - - # the index of the `[` character - bracket_idx = match.end(1) - - # append closing quote and bracket as appropriate - # this is *not* appropriate if the opening quote or bracket is outside - # the text given to this method - suf = '' - continuation = self.line_buffer[len(self.text_until_cursor):] - if key_start > text_start and closing_quote: - # quotes were opened inside text, maybe close them - if continuation.startswith(closing_quote): - continuation = continuation[len(closing_quote):] - else: - suf += closing_quote - if bracket_idx > text_start: - # brackets were opened inside text, maybe close them - if not continuation.startswith(']'): - suf += ']' - - return [leading + k + suf for k in matches] - - def unicode_name_matches(self, text): - u"""Match Latex-like syntax for unicode characters base - on the name of the character. - - This does ``\\GREEK SMALL LETTER ETA`` -> ``η`` - - Works only on valid python 3 identifier, or on combining characters that - will combine to form a valid identifier. - - Used on Python 3 only. - """ - slashpos = text.rfind('\\') - if slashpos > -1: - s = text[slashpos+1:] - try : - unic = unicodedata.lookup(s) - # allow combining chars - if ('a'+unic).isidentifier(): - return '\\'+s,[unic] - except KeyError: - pass - return u'', [] - - - def latex_matches(self, text): - u"""Match Latex syntax for unicode characters. - - This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - """ - slashpos = text.rfind('\\') - if slashpos > -1: - s = text[slashpos:] - if s in latex_symbols: - # Try to complete a full latex symbol to unicode - # \\alpha -> α - return s, [latex_symbols[s]] - else: - # If a user has partially typed a latex symbol, give them - # a full list of options \al -> [\aleph, \alpha] - matches = [k for k in latex_symbols if k.startswith(s)] - if matches: - return s, matches - return u'', [] - - def dispatch_custom_completer(self, text): - if not self.custom_completers: - return - - line = self.line_buffer - if not line.strip(): - return None - - # Create a little structure to pass all the relevant information about - # the current completion to any custom completer. - event = SimpleNamespace() - event.line = line - event.symbol = text - cmd = line.split(None,1)[0] - event.command = cmd - event.text_until_cursor = self.text_until_cursor - - # for foo etc, try also to find completer for %foo - if not cmd.startswith(self.magic_escape): - try_magic = self.custom_completers.s_matches( - self.magic_escape + cmd) - else: - try_magic = [] - - for c in itertools.chain(self.custom_completers.s_matches(cmd), - try_magic, - self.custom_completers.flat_matches(self.text_until_cursor)): - try: - res = c(event) - if res: - # first, try case sensitive match - withcase = [r for r in res if r.startswith(text)] - if withcase: - return withcase - # if none, then case insensitive ones are ok too - text_low = text.lower() - return [r for r in res if r.lower().startswith(text_low)] - except TryNext: - pass - except KeyboardInterrupt: - """ - If custom completer take too long, - let keyboard interrupt abort and return nothing. - """ - break - - return None - - def completions(self, text: str, offset: int)->Iterator[Completion]: - """ - Returns an iterator over the possible completions - - .. warning:: - - Unstable - - This function is unstable, API may change without warning. - It will also raise unless use in proper context manager. - - Parameters - ---------- - - text:str - Full text of the current input, multi line string. - offset:int - Integer representing the position of the cursor in ``text``. Offset - is 0-based indexed. - - Yields - ------ - :any:`Completion` object - - - The cursor on a text can either be seen as being "in between" - characters or "On" a character depending on the interface visible to - the user. For consistency the cursor being on "in between" characters X - and Y is equivalent to the cursor being "on" character Y, that is to say - the character the cursor is on is considered as being after the cursor. - - Combining characters may span more that one position in the - text. - - - .. note:: - - If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--`` - fake Completion token to distinguish completion returned by Jedi - and usual IPython completion. - - .. note:: - - Completions are not completely deduplicated yet. If identical - completions are coming from different sources this function does not - ensure that each completion object will only be present once. - """ - warnings.warn("_complete is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) - - seen = set() - try: - for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000): - if c and (c in seen): - continue - yield c - seen.add(c) - except KeyboardInterrupt: - """if completions take too long and users send keyboard interrupt, - do not crash and return ASAP. """ - pass - - def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]: - """ - Core completion module.Same signature as :any:`completions`, with the - extra `timeout` parameter (in seconds). - - - Computing jedi's completion ``.type`` can be quite expensive (it is a - lazy property) and can require some warm-up, more warm up than just - computing the ``name`` of a completion. The warm-up can be : - - - Long warm-up the first time a module is encountered after - install/update: actually build parse/inference tree. - - - first time the module is encountered in a session: load tree from - disk. - - We don't want to block completions for tens of seconds so we give the - completer a "budget" of ``_timeout`` seconds per invocation to compute - completions types, the completions that have not yet been computed will - be marked as "unknown" an will have a chance to be computed next round - are things get cached. - - Keep in mind that Jedi is not the only thing treating the completion so - keep the timeout short-ish as if we take more than 0.3 second we still - have lots of processing to do. - - """ - deadline = time.monotonic() + _timeout - - - before = full_text[:offset] - cursor_line, cursor_column = position_to_cursor(full_text, offset) - - matched_text, matches, matches_origin, jedi_matches = self._complete( - full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column) - - iter_jm = iter(jedi_matches) - if _timeout: - for jm in iter_jm: - try: - type_ = jm.type - except Exception: - if self.debug: - print("Error in Jedi getting type of ", jm) - type_ = None - delta = len(jm.name_with_symbols) - len(jm.complete) - if type_ == 'function': - signature = _make_signature(jm) - else: - signature = '' - yield Completion(start=offset - delta, - end=offset, - text=jm.name_with_symbols, - type=type_, - signature=signature, - _origin='jedi') - - if time.monotonic() > deadline: - break - - for jm in iter_jm: - delta = len(jm.name_with_symbols) - len(jm.complete) - yield Completion(start=offset - delta, - end=offset, - text=jm.name_with_symbols, - type='', # don't compute type for speed - _origin='jedi', - signature='') - - - start_offset = before.rfind(matched_text) - - # TODO: - # Suppress this, right now just for debug. - if jedi_matches and matches and self.debug: - yield Completion(start=start_offset, end=offset, text='--jedi/ipython--', - _origin='debug', type='none', signature='') - - # I'm unsure if this is always true, so let's assert and see if it - # crash - assert before.endswith(matched_text) - for m, t in zip(matches, matches_origin): - yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='') - - - def complete(self, text=None, line_buffer=None, cursor_pos=None): - """Find completions for the given text and line context. - - Note that both the text and the line_buffer are optional, but at least - one of them must be given. - - Parameters - ---------- - text : string, optional - Text to perform the completion on. If not given, the line buffer - is split using the instance's CompletionSplitter object. - - line_buffer : string, optional - If not given, the completer attempts to obtain the current line - buffer via readline. This keyword allows clients which are - requesting for text completions in non-readline contexts to inform - the completer of the entire text. - - cursor_pos : int, optional - Index of the cursor in the full line buffer. Should be provided by - remote frontends where kernel has no access to frontend state. - - Returns - ------- - text : str - Text that was actually used in the completion. - - matches : list - A list of completion matches. - - - .. note:: - - This API is likely to be deprecated and replaced by - :any:`IPCompleter.completions` in the future. - - - """ - warnings.warn('`Completer.complete` is pending deprecation since ' - 'IPython 6.0 and will be replaced by `Completer.completions`.', - PendingDeprecationWarning) - # potential todo, FOLD the 3rd throw away argument of _complete - # into the first 2 one. - return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2] - - def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, - full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]: - """ - - Like complete but can also returns raw jedi completions as well as the - origin of the completion text. This could (and should) be made much - cleaner but that will be simpler once we drop the old (and stateful) - :any:`complete` API. - - - With current provisional API, cursor_pos act both (depending on the - caller) as the offset in the ``text`` or ``line_buffer``, or as the - ``column`` when passing multiline strings this could/should be renamed - but would add extra noise. - """ - - # if the cursor position isn't given, the only sane assumption we can - # make is that it's at the end of the line (the common case) - if cursor_pos is None: - cursor_pos = len(line_buffer) if text is None else len(text) - - if self.use_main_ns: - self.namespace = __main__.__dict__ - - # if text is either None or an empty string, rely on the line buffer - if (not line_buffer) and full_text: - line_buffer = full_text.split('\n')[cursor_line] - if not text: # issue #11508: check line_buffer before calling split_line - text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' - - if self.backslash_combining_completions: - # allow deactivation of these on windows. - base_text = text if not line_buffer else line_buffer[:cursor_pos] - latex_text, latex_matches = self.latex_matches(base_text) - if latex_matches: - return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () - name_text = '' - name_matches = [] - # need to add self.fwd_unicode_match() function here when done - for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): - name_text, name_matches = meth(base_text) - if name_text: - return name_text, name_matches[:MATCHES_LIMIT], \ - [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), () - - - # If no line buffer is given, assume the input text is all there was - if line_buffer is None: - line_buffer = text - - self.line_buffer = line_buffer - self.text_until_cursor = self.line_buffer[:cursor_pos] - - # Do magic arg matches - for matcher in self.magic_arg_matchers: - matches = list(matcher(line_buffer))[:MATCHES_LIMIT] - if matches: - origins = [matcher.__qualname__] * len(matches) - return text, matches, origins, () - - # Start with a clean slate of completions - matches = [] - - # FIXME: we should extend our api to return a dict with completions for - # different types of objects. The rlcomplete() method could then - # simply collapse the dict into a list for readline, but we'd have - # richer completion semantics in other environments. - completions = () - if self.use_jedi: - if not full_text: - full_text = line_buffer - completions = self._jedi_matches( - cursor_pos, cursor_line, full_text) - - if self.merge_completions: - matches = [] - for matcher in self.matchers: - try: - matches.extend([(m, matcher.__qualname__) - for m in matcher(text)]) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - matches = [(m, matcher.__qualname__) - for m in matcher(text)] - if matches: - break - - seen = set() - filtered_matches = set() - for m in matches: - t, c = m - if t not in seen: - filtered_matches.add(m) - seen.add(t) - - _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) - - custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] - - _filtered_matches = custom_res or _filtered_matches - - _filtered_matches = _filtered_matches[:MATCHES_LIMIT] - _matches = [m[0] for m in _filtered_matches] - origins = [m[1] for m in _filtered_matches] - - self.matches = _matches - - return text, _matches, origins, completions - - def fwd_unicode_match(self, text:str) -> Tuple[str, list]: - if self._names is None: - self._names = [] - for c in range(0,0x10FFFF + 1): - try: - self._names.append(unicodedata.name(chr(c))) - except ValueError: - pass - - slashpos = text.rfind('\\') - # if text starts with slash - if slashpos > -1: - s = text[slashpos+1:] - candidates = [x for x in self._names if x.startswith(s)] - if candidates: - return s, candidates - else: - return '', () - - # if text does not start with slash - else: - return u'', () +"""Completion for IPython. + +This module started as fork of the rlcompleter module in the Python standard +library. The original enhancements made to rlcompleter have been sent +upstream and were accepted as of Python 2.3, + +This module now support a wide variety of completion mechanism both available +for normal classic Python code, as well as completer for IPython specific +Syntax like magics. + +Latex and Unicode completion +============================ + +IPython and compatible frontends not only can complete your code, but can help +you to input a wide range of characters. In particular we allow you to insert +a unicode character using the tab completion mechanism. + +Forward latex/unicode completion +-------------------------------- + +Forward completion allows you to easily type a unicode character using its latex +name, or unicode long description. To do so type a backslash follow by the +relevant name and press tab: + + +Using latex completion: + +.. code:: + + \\alpha + α + +or using unicode completion: + + +.. code:: + + \\greek small letter alpha + α + + +Only valid Python identifiers will complete. Combining characters (like arrow or +dots) are also available, unlike latex they need to be put after the their +counterpart that is to say, `F\\\\vec` is correct, not `\\\\vecF`. + +Some browsers are known to display combining characters incorrectly. + +Backward latex completion +------------------------- + +It is sometime challenging to know how to type a character, if you are using +IPython, or any compatible frontend you can prepend backslash to the character +and press `` to expand it to its latex form. + +.. code:: + + \\α + \\alpha + + +Both forward and backward completions can be deactivated by setting the +``Completer.backslash_combining_completions`` option to ``False``. + + +Experimental +============ + +Starting with IPython 6.0, this module can make use of the Jedi library to +generate completions both using static analysis of the code, and dynamically +inspecting multiple namespaces. Jedi is an autocompletion and static analysis +for Python. The APIs attached to this new mechanism is unstable and will +raise unless use in an :any:`provisionalcompleter` context manager. + +You will find that the following are experimental: + + - :any:`provisionalcompleter` + - :any:`IPCompleter.completions` + - :any:`Completion` + - :any:`rectify_completions` + +.. note:: + + better name for :any:`rectify_completions` ? + +We welcome any feedback on these new API, and we also encourage you to try this +module in debug mode (start IPython with ``--Completer.debug=True``) in order +to have extra logging information if :any:`jedi` is crashing, or if current +IPython completer pending deprecations are returning results not yet handled +by :any:`jedi` + +Using Jedi for tab completion allow snippets like the following to work without +having to execute any code: + + >>> myvar = ['hello', 42] + ... myvar[1].bi + +Tab completion will be able to infer that ``myvar[1]`` is a real number without +executing any code unlike the previously available ``IPCompleter.greedy`` +option. + +Be sure to update :any:`jedi` to the latest stable version or to try the +current development version to get better completions. +""" + + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. +# +# Some of this code originated from rlcompleter in the Python standard library +# Copyright (C) 2001 Python Software Foundation, www.python.org + + +import builtins as builtin_mod +import glob +import inspect +import itertools +import keyword +import os +import re +import string +import sys +import time +import unicodedata +import warnings +from contextlib import contextmanager +from importlib import import_module +from types import SimpleNamespace +from typing import Iterable, Iterator, List, Tuple + +from IPython.core.error import TryNext +from IPython.core.inputtransformer2 import ESC_MAGIC +from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol +from IPython.core.oinspect import InspectColors +from IPython.utils import generics +from IPython.utils.dir2 import dir2, get_real_method +from IPython.utils.process import arg_split +from traitlets import Bool, Enum, Int, observe +from traitlets.config.configurable import Configurable + +import __main__ + +# skip module docstests +skip_doctest = True + +try: + import jedi + jedi.settings.case_insensitive_completion = False + import jedi.api.helpers + import jedi.api.classes + JEDI_INSTALLED = True +except ImportError: + JEDI_INSTALLED = False +#----------------------------------------------------------------------------- +# Globals +#----------------------------------------------------------------------------- + +# Public API +__all__ = ['Completer','IPCompleter'] + +if sys.platform == 'win32': + PROTECTABLES = ' ' +else: + PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&' + +# Protect against returning an enormous number of completions which the frontend +# may have trouble processing. +MATCHES_LIMIT = 500 + + +class Sentinel: + def __repr__(self): + return "" + + +_deprecation_readline_sentinel = Sentinel() + + +class ProvisionalCompleterWarning(FutureWarning): + """ + Exception raise by an experimental feature in this module. + + Wrap code in :any:`provisionalcompleter` context manager if you + are certain you want to use an unstable feature. + """ + pass + +warnings.filterwarnings('error', category=ProvisionalCompleterWarning) + +@contextmanager +def provisionalcompleter(action='ignore'): + """ + + + This context manager has to be used in any place where unstable completer + behavior and API may be called. + + >>> with provisionalcompleter(): + ... completer.do_experimental_things() # works + + >>> completer.do_experimental_things() # raises. + + .. note:: + + Unstable + + By using this context manager you agree that the API in use may change + without warning, and that you won't complain if they do so. + + You also understand that, if the API is not to your liking, you should report + a bug to explain your use case upstream. + + We'll be happy to get your feedback, feature requests, and improvements on + any of the unstable APIs! + """ + with warnings.catch_warnings(): + warnings.filterwarnings(action, category=ProvisionalCompleterWarning) + yield + + +def has_open_quotes(s): + """Return whether a string has open quotes. + + This simply counts whether the number of quote characters of either type in + the string is odd. + + Returns + ------- + If there is an open quote, the quote character is returned. Else, return + False. + """ + # We check " first, then ', so complex cases with nested quotes will get + # the " to take precedence. + if s.count('"') % 2: + return '"' + elif s.count("'") % 2: + return "'" + else: + return False + + +def protect_filename(s, protectables=PROTECTABLES): + """Escape a string to protect certain characters.""" + if set(s) & set(protectables): + if sys.platform == "win32": + return '"' + s + '"' + else: + return "".join(("\\" + c if c in protectables else c) for c in s) + else: + return s + + +def expand_user(path:str) -> Tuple[str, bool, str]: + """Expand ``~``-style usernames in strings. + + This is similar to :func:`os.path.expanduser`, but it computes and returns + extra information that will be useful if the input was being used in + computing completions, and you wish to return the completions with the + original '~' instead of its expanded value. + + Parameters + ---------- + path : str + String to be expanded. If no ~ is present, the output is the same as the + input. + + Returns + ------- + newpath : str + Result of ~ expansion in the input path. + tilde_expand : bool + Whether any expansion was performed or not. + tilde_val : str + The value that ~ was replaced with. + """ + # Default values + tilde_expand = False + tilde_val = '' + newpath = path + + if path.startswith('~'): + tilde_expand = True + rest = len(path)-1 + newpath = os.path.expanduser(path) + if rest: + tilde_val = newpath[:-rest] + else: + tilde_val = newpath + + return newpath, tilde_expand, tilde_val + + +def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str: + """Does the opposite of expand_user, with its outputs. + """ + if tilde_expand: + return path.replace(tilde_val, '~') + else: + return path + + +def completions_sorting_key(word): + """key for sorting completions + + This does several things: + + - Demote any completions starting with underscores to the end + - Insert any %magic and %%cellmagic completions in the alphabetical order + by their name + """ + prio1, prio2 = 0, 0 + + if word.startswith('__'): + prio1 = 2 + elif word.startswith('_'): + prio1 = 1 + + if word.endswith('='): + prio1 = -1 + + if word.startswith('%%'): + # If there's another % in there, this is something else, so leave it alone + if not "%" in word[2:]: + word = word[2:] + prio2 = 2 + elif word.startswith('%'): + if not "%" in word[1:]: + word = word[1:] + prio2 = 1 + + return prio1, word, prio2 + + +class _FakeJediCompletion: + """ + This is a workaround to communicate to the UI that Jedi has crashed and to + report a bug. Will be used only id :any:`IPCompleter.debug` is set to true. + + Added in IPython 6.0 so should likely be removed for 7.0 + + """ + + def __init__(self, name): + + self.name = name + self.complete = name + self.type = 'crashed' + self.name_with_symbols = name + self.signature = '' + self._origin = 'fake' + + def __repr__(self): + return '' + + +class Completion: + """ + Completion object used and return by IPython completers. + + .. warning:: + + Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + This act as a middle ground :any:`Completion` object between the + :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion + object. While Jedi need a lot of information about evaluator and how the + code should be ran/inspected, PromptToolkit (and other frontend) mostly + need user facing information. + + - Which range should be replaced replaced by what. + - Some metadata (like completion type), or meta information to displayed to + the use user. + + For debugging purpose we can also store the origin of the completion (``jedi``, + ``IPython.python_matches``, ``IPython.magics_matches``...). + """ + + __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] + + def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: + warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + self.start = start + self.end = end + self.text = text + self.type = type + self.signature = signature + self._origin = _origin + + def __repr__(self): + return '' % \ + (self.start, self.end, self.text, self.type or '?', self.signature or '?') + + def __eq__(self, other)->Bool: + """ + Equality and hash do not hash the type (as some completer may not be + able to infer the type), but are use to (partially) de-duplicate + completion. + + Completely de-duplicating completion is a bit tricker that just + comparing as it depends on surrounding text, which Completions are not + aware of. + """ + return self.start == other.start and \ + self.end == other.end and \ + self.text == other.text + + def __hash__(self): + return hash((self.start, self.end, self.text)) + + +_IC = Iterable[Completion] + + +def _deduplicate_completions(text: str, completions: _IC)-> _IC: + """ + Deduplicate a set of completions. + + .. warning:: + + Unstable + + This function is unstable, API may change without warning. + + Parameters + ---------- + text: str + text that should be completed. + completions: Iterator[Completion] + iterator over the completions to deduplicate + + Yields + ------ + `Completions` objects + + + Completions coming from multiple sources, may be different but end up having + the same effect when applied to ``text``. If this is the case, this will + consider completions as equal and only emit the first encountered. + + Not folded in `completions()` yet for debugging purpose, and to detect when + the IPython completer does return things that Jedi does not, but should be + at some point. + """ + completions = list(completions) + if not completions: + return + + new_start = min(c.start for c in completions) + new_end = max(c.end for c in completions) + + seen = set() + for c in completions: + new_text = text[new_start:c.start] + c.text + text[c.end:new_end] + if new_text not in seen: + yield c + seen.add(new_text) + + +def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: + """ + Rectify a set of completions to all have the same ``start`` and ``end`` + + .. warning:: + + Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + Parameters + ---------- + text: str + text that should be completed. + completions: Iterator[Completion] + iterator over the completions to rectify + + + :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though + the Jupyter Protocol requires them to behave like so. This will readjust + the completion to have the same ``start`` and ``end`` by padding both + extremities with surrounding text. + + During stabilisation should support a ``_debug`` option to log which + completion are return by the IPython completer and not found in Jedi in + order to make upstream bug report. + """ + warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + completions = list(completions) + if not completions: + return + starts = (c.start for c in completions) + ends = (c.end for c in completions) + + new_start = min(starts) + new_end = max(ends) + + seen_jedi = set() + seen_python_matches = set() + for c in completions: + new_text = text[new_start:c.start] + c.text + text[c.end:new_end] + if c._origin == 'jedi': + seen_jedi.add(new_text) + elif c._origin == 'IPCompleter.python_matches': + seen_python_matches.add(new_text) + yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature) + diff = seen_python_matches.difference(seen_jedi) + if diff and _debug: + print('IPython.python matches have extras:', diff) + + +if sys.platform == 'win32': + DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?' +else: + DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?' + +GREEDY_DELIMS = ' =\r\n' + + +class CompletionSplitter(object): + """An object to split an input line in a manner similar to readline. + + By having our own implementation, we can expose readline-like completion in + a uniform manner to all frontends. This object only needs to be given the + line of text to be split and the cursor position on said line, and it + returns the 'word' to be completed on at the cursor after splitting the + entire line. + + What characters are used as splitting delimiters can be controlled by + setting the ``delims`` attribute (this is a property that internally + automatically builds the necessary regular expression)""" + + # Private interface + + # A string of delimiter characters. The default value makes sense for + # IPython's most typical usage patterns. + _delims = DELIMS + + # The expression (a normal string) to be compiled into a regular expression + # for actual splitting. We store it as an attribute mostly for ease of + # debugging, since this type of code can be so tricky to debug. + _delim_expr = None + + # The regular expression that does the actual splitting + _delim_re = None + + def __init__(self, delims=None): + delims = CompletionSplitter._delims if delims is None else delims + self.delims = delims + + @property + def delims(self): + """Return the string of delimiter characters.""" + return self._delims + + @delims.setter + def delims(self, delims): + """Set the delimiters for line splitting.""" + expr = '[' + ''.join('\\'+ c for c in delims) + ']' + self._delim_re = re.compile(expr) + self._delims = delims + self._delim_expr = expr + + def split_line(self, line, cursor_pos=None): + """Split a line of text with a cursor at the given position. + """ + l = line if cursor_pos is None else line[:cursor_pos] + return self._delim_re.split(l)[-1] + + + +class Completer(Configurable): + + greedy = Bool(False, + help="""Activate greedy completion + PENDING DEPRECTION. this is now mostly taken care of with Jedi. + + This will enable completion on elements of lists, results of function calls, etc., + but can be unsafe because the code is actually evaluated on TAB. + """ + ).tag(config=True) + + use_jedi = Bool(default_value=JEDI_INSTALLED, + help="Experimental: Use Jedi to generate autocompletions. " + "Default to True if jedi is installed.").tag(config=True) + + jedi_compute_type_timeout = Int(default_value=400, + help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types. + Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt + performance by preventing jedi to build its cache. + """).tag(config=True) + + debug = Bool(default_value=False, + help='Enable debug for the Completer. Mostly print extra ' + 'information for experimental jedi integration.')\ + .tag(config=True) + + backslash_combining_completions = Bool(True, + help="Enable unicode completions, e.g. \\alpha . " + "Includes completion of latex commands, unicode names, and expanding " + "unicode characters back to latex commands.").tag(config=True) + + + + def __init__(self, namespace=None, global_namespace=None, **kwargs): + """Create a new completer for the command line. + + Completer(namespace=ns, global_namespace=ns2) -> completer instance. + + If unspecified, the default namespace where completions are performed + is __main__ (technically, __main__.__dict__). Namespaces should be + given as dictionaries. + + An optional second namespace can be given. This allows the completer + to handle cases where both the local and global scopes need to be + distinguished. + """ + + # Don't bind to namespace quite yet, but flag whether the user wants a + # specific namespace or to use __main__.__dict__. This will allow us + # to bind to __main__.__dict__ at completion time, not now. + if namespace is None: + self.use_main_ns = True + else: + self.use_main_ns = False + self.namespace = namespace + + # The global namespace, if given, can be bound directly + if global_namespace is None: + self.global_namespace = {} + else: + self.global_namespace = global_namespace + + self.custom_matchers = [] + + super(Completer, self).__init__(**kwargs) + + def complete(self, text, state): + """Return the next possible completion for 'text'. + + This is called successively with state == 0, 1, 2, ... until it + returns None. The completion should begin with 'text'. + + """ + if self.use_main_ns: + self.namespace = __main__.__dict__ + + if state == 0: + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) + try: + return self.matches[state] + except IndexError: + return None + + def global_matches(self, text): + """Compute matches when text is a simple name. + + Return a list of all keywords, built-in functions and names currently + defined in self.namespace or self.global_namespace that match. + + """ + matches = [] + match_append = matches.append + n = len(text) + for lst in [keyword.kwlist, + builtin_mod.__dict__.keys(), + self.namespace.keys(), + self.global_namespace.keys()]: + for word in lst: + if word[:n] == text and word != "__builtins__": + match_append(word) + + snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z") + for lst in [self.namespace.keys(), + self.global_namespace.keys()]: + shortened = {"_".join([sub[0] for sub in word.split('_')]) : word + for word in lst if snake_case_re.match(word)} + for word in shortened.keys(): + if word[:n] == text and word != "__builtins__": + match_append(shortened[word]) + return matches + + def attr_matches(self, text): + """Compute matches when text contains a dot. + + Assuming the text is of the form NAME.NAME....[NAME], and is + evaluatable in self.namespace or self.global_namespace, it will be + evaluated and its attributes (as revealed by dir()) are used as + possible completions. (For class instances, class members are + also considered.) + + WARNING: this can still invoke arbitrary C code, if an object + with a __getattr__ hook is evaluated. + + """ + + # Another option, seems to work great. Catches things like ''. + m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) + + if m: + expr, attr = m.group(1, 3) + elif self.greedy: + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return [] + expr, attr = m2.group(1,2) + else: + return [] + + try: + obj = eval(expr, self.namespace) + except: + try: + obj = eval(expr, self.global_namespace) + except: + return [] + + if self.limit_to__all__ and hasattr(obj, '__all__'): + words = get__all__entries(obj) + else: + words = dir2(obj) + + try: + words = generics.complete_object(obj, words) + except TryNext: + pass + except AssertionError: + raise + except Exception: + # Silence errors from completion function + #raise # dbg + pass + # Build match list to return + n = len(attr) + return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] + + +def get__all__entries(obj): + """returns the strings in the __all__ attribute""" + try: + words = getattr(obj, '__all__') + except: + return [] + + return [w for w in words if isinstance(w, str)] + + +def match_dict_keys(keys: List[str], prefix: str, delims: str): + """Used by dict_key_matches, matching the prefix to a list of keys + + Parameters + ========== + keys: + list of keys in dictionary currently being completed. + prefix: + Part of the text already typed by the user. e.g. `mydict[b'fo` + delims: + String of delimiters to consider when finding the current key. + + Returns + ======= + + A tuple of three elements: ``quote``, ``token_start``, ``matched``, with + ``quote`` being the quote that need to be used to close current string. + ``token_start`` the position where the replacement should start occurring, + ``matches`` a list of replacement/completion + + """ + if not prefix: + return None, 0, [repr(k) for k in keys + if isinstance(k, (str, bytes))] + quote_match = re.search('["\']', prefix) + quote = quote_match.group() + try: + prefix_str = eval(prefix + quote, {}) + except Exception: + return None, 0, [] + + pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' + token_match = re.search(pattern, prefix, re.UNICODE) + token_start = token_match.start() + token_prefix = token_match.group() + + matched = [] + for key in keys: + try: + if not key.startswith(prefix_str): + continue + except (AttributeError, TypeError, UnicodeError): + # Python 3+ TypeError on b'a'.startswith('a') or vice-versa + continue + + # reformat remainder of key to begin with prefix + rem = key[len(prefix_str):] + # force repr wrapped in ' + rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') + if rem_repr.startswith('u') and prefix[0] not in 'uU': + # Found key is unicode, but prefix is Py2 string. + # Therefore attempt to interpret key as string. + try: + rem_repr = repr(rem.encode('ascii') + '"') + except UnicodeEncodeError: + continue + + rem_repr = rem_repr[1 + rem_repr.index("'"):-2] + if quote == '"': + # The entered prefix is quoted with ", + # but the match is quoted with '. + # A contained " hence needs escaping for comparison: + rem_repr = rem_repr.replace('"', '\\"') + + # then reinsert prefix from start of token + matched.append('%s%s' % (token_prefix, rem_repr)) + return quote, token_start, matched + + +def cursor_to_position(text:str, line:int, column:int)->int: + """ + + Convert the (line,column) position of the cursor in text to an offset in a + string. + + Parameters + ---------- + + text : str + The text in which to calculate the cursor offset + line : int + Line of the cursor; 0-indexed + column : int + Column of the cursor 0-indexed + + Return + ------ + Position of the cursor in ``text``, 0-indexed. + + See Also + -------- + position_to_cursor: reciprocal of this function + + """ + lines = text.split('\n') + assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines))) + + return sum(len(l) + 1 for l in lines[:line]) + column + +def position_to_cursor(text:str, offset:int)->Tuple[int, int]: + """ + Convert the position of the cursor in text (0 indexed) to a line + number(0-indexed) and a column number (0-indexed) pair + + Position should be a valid position in ``text``. + + Parameters + ---------- + + text : str + The text in which to calculate the cursor offset + offset : int + Position of the cursor in ``text``, 0-indexed. + + Return + ------ + (line, column) : (int, int) + Line of the cursor; 0-indexed, column of the cursor 0-indexed + + + See Also + -------- + cursor_to_position : reciprocal of this function + + + """ + + assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text)) + + before = text[:offset] + blines = before.split('\n') # ! splitnes trim trailing \n + line = before.count('\n') + col = len(blines[-1]) + return line, col + + +def _safe_isinstance(obj, module, class_name): + """Checks if obj is an instance of module.class_name if loaded + """ + return (module in sys.modules and + isinstance(obj, getattr(import_module(module), class_name))) + + +def back_unicode_name_matches(text): + u"""Match unicode characters back to unicode name + + This does ``☃`` -> ``\\snowman`` + + Note that snowman is not a valid python3 combining character but will be expanded. + Though it will not recombine back to the snowman character by the completion machinery. + + This will not either back-complete standard sequences like \\n, \\b ... + + Used on Python 3 only. + """ + if len(text)<2: + return u'', () + maybe_slash = text[-2] + if maybe_slash != '\\': + return u'', () + + char = text[-1] + # no expand on quote for completion in strings. + # nor backcomplete standard ascii keys + if char in string.ascii_letters or char in ['"',"'"]: + return u'', () + try : + unic = unicodedata.name(char) + return '\\'+char,['\\'+unic] + except KeyError: + pass + return u'', () + +def back_latex_name_matches(text:str): + """Match latex characters back to unicode name + + This does ``\\ℵ`` -> ``\\aleph`` + + Used on Python 3 only. + """ + if len(text)<2: + return u'', () + maybe_slash = text[-2] + if maybe_slash != '\\': + return u'', () + + + char = text[-1] + # no expand on quote for completion in strings. + # nor backcomplete standard ascii keys + if char in string.ascii_letters or char in ['"',"'"]: + return u'', () + try : + latex = reverse_latex_symbol[char] + # '\\' replace the \ as well + return '\\'+char,[latex] + except KeyError: + pass + return u'', () + + +def _formatparamchildren(parameter) -> str: + """ + Get parameter name and value from Jedi Private API + + Jedi does not expose a simple way to get `param=value` from its API. + + Parameter + ========= + + parameter: + Jedi's function `Param` + + Returns + ======= + + A string like 'a', 'b=1', '*args', '**kwargs' + + + """ + description = parameter.description + if not description.startswith('param '): + raise ValueError('Jedi function parameter description have change format.' + 'Expected "param ...", found %r".' % description) + return description[6:] + +def _make_signature(completion)-> str: + """ + Make the signature from a jedi completion + + Parameter + ========= + + completion: jedi.Completion + object does not complete a function type + + Returns + ======= + + a string consisting of the function signature, with the parenthesis but + without the function name. example: + `(a, *args, b=1, **kwargs)` + + """ + + return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for p in completion.params) if f]) + # it looks like this might work on jedi 0.17 + if hasattr(completion, 'get_signatures'): + signatures = completion.get_signatures() + if not signatures: + return '(?)' + + c0 = completion.get_signatures()[0] + return '('+c0.to_string().split('(', maxsplit=1)[1] + + return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures() + for p in signature.defined_names()) if f]) + +class IPCompleter(Completer): + """Extension of the completer class with IPython-specific features""" + + _names = None + + @observe('greedy') + def _greedy_changed(self, change): + """update the splitter and readline delims when greedy is changed""" + if change['new']: + self.splitter.delims = GREEDY_DELIMS + else: + self.splitter.delims = DELIMS + + dict_keys_only = Bool(False, + help="""Whether to show dict key matches only""") + + merge_completions = Bool(True, + help="""Whether to merge completion results into a single list + + If False, only the completion results from the first non-empty + completer will be returned. + """ + ).tag(config=True) + omit__names = Enum((0,1,2), default_value=2, + help="""Instruct the completer to omit private method names + + Specifically, when completing on ``object.``. + + When 2 [default]: all names that start with '_' will be excluded. + + When 1: all 'magic' names (``__foo__``) will be excluded. + + When 0: nothing will be excluded. + """ + ).tag(config=True) + limit_to__all__ = Bool(False, + help=""" + DEPRECATED as of version 5.0. + + Instruct the completer to use __all__ for the completion + + Specifically, when completing on ``object.``. + + When True: only those names in obj.__all__ will be included. + + When False [default]: the __all__ attribute is ignored + """, + ).tag(config=True) + + @observe('limit_to__all__') + def _limit_to_all_changed(self, change): + warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration ' + 'value has been deprecated since IPython 5.0, will be made to have ' + 'no effects and then removed in future version of IPython.', + UserWarning) + + def __init__(self, shell=None, namespace=None, global_namespace=None, + use_readline=_deprecation_readline_sentinel, config=None, **kwargs): + """IPCompleter() -> completer + + Return a completer object. + + Parameters + ---------- + + shell + a pointer to the ipython shell itself. This is needed + because this completer knows about magic functions, and those can + only be accessed via the ipython instance. + + namespace : dict, optional + an optional dict where completions are performed. + + global_namespace : dict, optional + secondary optional dict for completions, to + handle cases (such as IPython embedded inside functions) where + both Python scopes are visible. + + use_readline : bool, optional + DEPRECATED, ignored since IPython 6.0, will have no effects + """ + + self.magic_escape = ESC_MAGIC + self.splitter = CompletionSplitter() + + if use_readline is not _deprecation_readline_sentinel: + warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.', + DeprecationWarning, stacklevel=2) + + # _greedy_changed() depends on splitter and readline being defined: + Completer.__init__(self, namespace=namespace, global_namespace=global_namespace, + config=config, **kwargs) + + # List where completion matches will be stored + self.matches = [] + self.shell = shell + # Regexp to split filenames with spaces in them + self.space_name_re = re.compile(r'([^\\] )') + # Hold a local ref. to glob.glob for speed + self.glob = glob.glob + + # Determine if we are running on 'dumb' terminals, like (X)Emacs + # buffers, to avoid completion problems. + term = os.environ.get('TERM','xterm') + self.dumb_terminal = term in ['dumb','emacs'] + + # Special handling of backslashes needed in win32 platforms + if sys.platform == "win32": + self.clean_glob = self._clean_glob_win32 + else: + self.clean_glob = self._clean_glob + + #regexp to parse docstring for function signature + self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*') + self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)') + #use this if positional argument name is also needed + #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)') + + self.magic_arg_matchers = [ + self.magic_config_matches, + self.magic_color_matches, + ] + + # This is set externally by InteractiveShell + self.custom_completers = None + + @property + def matchers(self): + """All active matcher routines for completion""" + if self.dict_keys_only: + return [self.dict_key_matches] + + if self.use_jedi: + return [ + *self.custom_matchers, + self.dict_key_matches, + self.file_matches, + self.magic_matches, + ] + else: + return [ + *self.custom_matchers, + self.dict_key_matches, + self.python_matches, + self.file_matches, + self.magic_matches, + self.python_func_kw_matches, + ] + + def all_completions(self, text) -> List[str]: + """ + Wrapper around the completion methods for the benefit of emacs. + """ + prefix = text.rpartition('.')[0] + with provisionalcompleter(): + return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text + for c in self.completions(text, len(text))] + + return self.complete(text)[1] + + def _clean_glob(self, text): + return self.glob("%s*" % text) + + def _clean_glob_win32(self,text): + return [f.replace("\\","/") + for f in self.glob("%s*" % text)] + + def file_matches(self, text): + """Match filenames, expanding ~USER type strings. + + Most of the seemingly convoluted logic in this completer is an + attempt to handle filenames with spaces in them. And yet it's not + quite perfect, because Python's readline doesn't expose all of the + GNU readline details needed for this to be done correctly. + + For a filename with a space in it, the printed completions will be + only the parts after what's already been typed (instead of the + full completions, as is normally done). I don't think with the + current (as of Python 2.3) Python readline it's possible to do + better.""" + + # chars that require escaping with backslash - i.e. chars + # that readline treats incorrectly as delimiters, but we + # don't want to treat as delimiters in filename matching + # when escaped with backslash + if text.startswith('!'): + text = text[1:] + text_prefix = u'!' + else: + text_prefix = u'' + + text_until_cursor = self.text_until_cursor + # track strings with open quotes + open_quotes = has_open_quotes(text_until_cursor) + + if '(' in text_until_cursor or '[' in text_until_cursor: + lsplit = text + else: + try: + # arg_split ~ shlex.split, but with unicode bugs fixed by us + lsplit = arg_split(text_until_cursor)[-1] + except ValueError: + # typically an unmatched ", or backslash without escaped char. + if open_quotes: + lsplit = text_until_cursor.split(open_quotes)[-1] + else: + return [] + except IndexError: + # tab pressed on empty line + lsplit = "" + + if not open_quotes and lsplit != protect_filename(lsplit): + # if protectables are found, do matching on the whole escaped name + has_protectables = True + text0,text = text,lsplit + else: + has_protectables = False + text = os.path.expanduser(text) + + if text == "": + return [text_prefix + protect_filename(f) for f in self.glob("*")] + + # Compute the matches from the filesystem + if sys.platform == 'win32': + m0 = self.clean_glob(text) + else: + m0 = self.clean_glob(text.replace('\\', '')) + + if has_protectables: + # If we had protectables, we need to revert our changes to the + # beginning of filename so that we don't double-write the part + # of the filename we have so far + len_lsplit = len(lsplit) + matches = [text_prefix + text0 + + protect_filename(f[len_lsplit:]) for f in m0] + else: + if open_quotes: + # if we have a string with an open quote, we don't need to + # protect the names beyond the quote (and we _shouldn't_, as + # it would cause bugs when the filesystem call is made). + matches = m0 if sys.platform == "win32" else\ + [protect_filename(f, open_quotes) for f in m0] + else: + matches = [text_prefix + + protect_filename(f) for f in m0] + + # Mark directories in input list by appending '/' to their names. + return [x+'/' if os.path.isdir(x) else x for x in matches] + + def magic_matches(self, text): + """Match magics""" + # Get all shell magics now rather than statically, so magics loaded at + # runtime show up too. + lsm = self.shell.magics_manager.lsmagic() + line_magics = lsm['line'] + cell_magics = lsm['cell'] + pre = self.magic_escape + pre2 = pre+pre + + explicit_magic = text.startswith(pre) + + # Completion logic: + # - user gives %%: only do cell magics + # - user gives %: do both line and cell magics + # - no prefix: do both + # In other words, line magics are skipped if the user gives %% explicitly + # + # We also exclude magics that match any currently visible names: + # https://github.com/ipython/ipython/issues/4877, unless the user has + # typed a %: + # https://github.com/ipython/ipython/issues/10754 + bare_text = text.lstrip(pre) + global_matches = self.global_matches(bare_text) + if not explicit_magic: + def matches(magic): + """ + Filter magics, in particular remove magics that match + a name present in global namespace. + """ + return ( magic.startswith(bare_text) and + magic not in global_matches ) + else: + def matches(magic): + return magic.startswith(bare_text) + + comp = [ pre2+m for m in cell_magics if matches(m)] + if not text.startswith(pre2): + comp += [ pre+m for m in line_magics if matches(m)] + + return comp + + def magic_config_matches(self, text:str) -> List[str]: + """ Match class names and attributes for %config magic """ + texts = text.strip().split() + + if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'): + # get all configuration classes + classes = sorted(set([ c for c in self.shell.configurables + if c.__class__.class_traits(config=True) + ]), key=lambda x: x.__class__.__name__) + classnames = [ c.__class__.__name__ for c in classes ] + + # return all classnames if config or %config is given + if len(texts) == 1: + return classnames + + # match classname + classname_texts = texts[1].split('.') + classname = classname_texts[0] + classname_matches = [ c for c in classnames + if c.startswith(classname) ] + + # return matched classes or the matched class with attributes + if texts[1].find('.') < 0: + return classname_matches + elif len(classname_matches) == 1 and \ + classname_matches[0] == classname: + cls = classes[classnames.index(classname)].__class__ + help = cls.class_get_help() + # strip leading '--' from cl-args: + help = re.sub(re.compile(r'^--', re.MULTILINE), '', help) + return [ attr.split('=')[0] + for attr in help.strip().splitlines() + if attr.startswith(texts[1]) ] + return [] + + def magic_color_matches(self, text:str) -> List[str] : + """ Match color schemes for %colors magic""" + texts = text.split() + if text.endswith(' '): + # .split() strips off the trailing whitespace. Add '' back + # so that: '%colors ' -> ['%colors', ''] + texts.append('') + + if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'): + prefix = texts[1] + return [ color for color in InspectColors.keys() + if color.startswith(prefix) ] + return [] + + def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): + """ + + Return a list of :any:`jedi.api.Completions` object from a ``text`` and + cursor position. + + Parameters + ---------- + cursor_column : int + column position of the cursor in ``text``, 0-indexed. + cursor_line : int + line position of the cursor in ``text``, 0-indexed + text : str + text to complete + + Debugging + --------- + + If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion` + object containing a string with the Jedi debug information attached. + """ + namespaces = [self.namespace] + if self.global_namespace is not None: + namespaces.append(self.global_namespace) + + completion_filter = lambda x:x + offset = cursor_to_position(text, cursor_line, cursor_column) + # filter output if we are completing for object members + if offset: + pre = text[offset-1] + if pre == '.': + if self.omit__names == 2: + completion_filter = lambda c:not c.name.startswith('_') + elif self.omit__names == 1: + completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__')) + elif self.omit__names == 0: + completion_filter = lambda x:x + else: + raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names)) + + interpreter = jedi.Interpreter(text[:offset], namespaces, column=cursor_column, line=cursor_line + 1) + try_jedi = True + + try: + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False + try: + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} + + # if we are in a string jedi is likely not the right candidate for + # now. Skip it. + try_jedi = not completing_string + except Exception as e: + # many of things can go wrong, we are using private API just don't crash. + if self.debug: + print("Error detecting if completing a non-finished string :", e, '|') + + if not try_jedi: + return [] + try: + return filter(completion_filter, interpreter.completions()) + except Exception as e: + if self.debug: + return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + else: + return [] + + def python_matches(self, text): + """Match attributes or global python names""" + if "." in text: + try: + matches = self.attr_matches(text) + if text.endswith('.') and self.omit__names: + if self.omit__names == 1: + # true if txt is _not_ a __ name, false otherwise: + no__name = (lambda txt: + re.match(r'.*\.__.*?__',txt) is None) + else: + # true if txt is _not_ a _ name, false otherwise: + no__name = (lambda txt: + re.match(r'\._.*?',txt[txt.rindex('.'):]) is None) + matches = filter(no__name, matches) + except NameError: + # catches . + matches = [] + else: + matches = self.global_matches(text) + return matches + + def _default_arguments_from_docstring(self, doc): + """Parse the first line of docstring for call signature. + + Docstring should be of the form 'min(iterable[, key=func])\n'. + It can also parse cython docstring of the form + 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'. + """ + if doc is None: + return [] + + #care only the firstline + line = doc.lstrip().splitlines()[0] + + #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*') + #'min(iterable[, key=func])\n' -> 'iterable[, key=func]' + sig = self.docstring_sig_re.search(line) + if sig is None: + return [] + # iterable[, key=func]' -> ['iterable[' ,' key=func]'] + sig = sig.groups()[0].split(',') + ret = [] + for s in sig: + #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)') + ret += self.docstring_kwd_re.findall(s) + return ret + + def _default_arguments(self, obj): + """Return the list of default arguments of obj if it is callable, + or empty list otherwise.""" + call_obj = obj + ret = [] + if inspect.isbuiltin(obj): + pass + elif not (inspect.isfunction(obj) or inspect.ismethod(obj)): + if inspect.isclass(obj): + #for cython embedsignature=True the constructor docstring + #belongs to the object itself not __init__ + ret += self._default_arguments_from_docstring( + getattr(obj, '__doc__', '')) + # for classes, check for __init__,__new__ + call_obj = (getattr(obj, '__init__', None) or + getattr(obj, '__new__', None)) + # for all others, check if they are __call__able + elif hasattr(obj, '__call__'): + call_obj = obj.__call__ + ret += self._default_arguments_from_docstring( + getattr(call_obj, '__doc__', '')) + + _keeps = (inspect.Parameter.KEYWORD_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD) + + try: + sig = inspect.signature(obj) + ret.extend(k for k, v in sig.parameters.items() if + v.kind in _keeps) + except ValueError: + pass + + return list(set(ret)) + + def python_func_kw_matches(self,text): + """Match named parameters (kwargs) of the last open function""" + + if "." in text: # a parameter cannot be dotted + return [] + try: regexp = self.__funcParamsRegex + except AttributeError: + regexp = self.__funcParamsRegex = re.compile(r''' + '.*?(?,a=1)", the candidate is "foo" + tokens = regexp.findall(self.text_until_cursor) + iterTokens = reversed(tokens); openPar = 0 + + for token in iterTokens: + if token == ')': + openPar -= 1 + elif token == '(': + openPar += 1 + if openPar > 0: + # found the last unclosed parenthesis + break + else: + return [] + # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" ) + ids = [] + isId = re.compile(r'\w+$').match + + while True: + try: + ids.append(next(iterTokens)) + if not isId(ids[-1]): + ids.pop(); break + if not next(iterTokens) == '.': + break + except StopIteration: + break + + # Find all named arguments already assigned to, as to avoid suggesting + # them again + usedNamedArgs = set() + par_level = -1 + for token, next_token in zip(tokens, tokens[1:]): + if token == '(': + par_level += 1 + elif token == ')': + par_level -= 1 + + if par_level != 0: + continue + + if next_token != '=': + continue + + usedNamedArgs.add(token) + + argMatches = [] + try: + callableObj = '.'.join(ids[::-1]) + namedArgs = self._default_arguments(eval(callableObj, + self.namespace)) + + # Remove used named arguments from the list, no need to show twice + for namedArg in set(namedArgs) - usedNamedArgs: + if namedArg.startswith(text): + argMatches.append(u"%s=" %namedArg) + except: + pass + + return argMatches + + def dict_key_matches(self, text): + "Match string keys in a dictionary, after e.g. 'foo[' " + def get_keys(obj): + # Objects can define their own completions by defining an + # _ipy_key_completions_() method. + method = get_real_method(obj, '_ipython_key_completions_') + if method is not None: + return method() + + # Special case some common in-memory dict-like types + if isinstance(obj, dict) or\ + _safe_isinstance(obj, 'pandas', 'DataFrame'): + try: + return list(obj.keys()) + except Exception: + return [] + elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ + _safe_isinstance(obj, 'numpy', 'void'): + return obj.dtype.names or [] + return [] + + try: + regexps = self.__dict_key_regexps + except AttributeError: + dict_key_re_fmt = r'''(?x) + ( # match dict-referring expression wrt greedy setting + %s + ) + \[ # open bracket + \s* # and optional whitespace + ([uUbB]? # string prefix (r not handled) + (?: # unclosed string + '(?:[^']|(? key_start: + leading = '' + else: + leading = text[text_start:completion_start] + + # the index of the `[` character + bracket_idx = match.end(1) + + # append closing quote and bracket as appropriate + # this is *not* appropriate if the opening quote or bracket is outside + # the text given to this method + suf = '' + continuation = self.line_buffer[len(self.text_until_cursor):] + if key_start > text_start and closing_quote: + # quotes were opened inside text, maybe close them + if continuation.startswith(closing_quote): + continuation = continuation[len(closing_quote):] + else: + suf += closing_quote + if bracket_idx > text_start: + # brackets were opened inside text, maybe close them + if not continuation.startswith(']'): + suf += ']' + + return [leading + k + suf for k in matches] + + def unicode_name_matches(self, text): + u"""Match Latex-like syntax for unicode characters base + on the name of the character. + + This does ``\\GREEK SMALL LETTER ETA`` -> ``η`` + + Works only on valid python 3 identifier, or on combining characters that + will combine to form a valid identifier. + + Used on Python 3 only. + """ + slashpos = text.rfind('\\') + if slashpos > -1: + s = text[slashpos+1:] + try : + unic = unicodedata.lookup(s) + # allow combining chars + if ('a'+unic).isidentifier(): + return '\\'+s,[unic] + except KeyError: + pass + return u'', [] + + + def latex_matches(self, text): + u"""Match Latex syntax for unicode characters. + + This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` + """ + slashpos = text.rfind('\\') + if slashpos > -1: + s = text[slashpos:] + if s in latex_symbols: + # Try to complete a full latex symbol to unicode + # \\alpha -> α + return s, [latex_symbols[s]] + else: + # If a user has partially typed a latex symbol, give them + # a full list of options \al -> [\aleph, \alpha] + matches = [k for k in latex_symbols if k.startswith(s)] + if matches: + return s, matches + return u'', [] + + def dispatch_custom_completer(self, text): + if not self.custom_completers: + return + + line = self.line_buffer + if not line.strip(): + return None + + # Create a little structure to pass all the relevant information about + # the current completion to any custom completer. + event = SimpleNamespace() + event.line = line + event.symbol = text + cmd = line.split(None,1)[0] + event.command = cmd + event.text_until_cursor = self.text_until_cursor + + # for foo etc, try also to find completer for %foo + if not cmd.startswith(self.magic_escape): + try_magic = self.custom_completers.s_matches( + self.magic_escape + cmd) + else: + try_magic = [] + + for c in itertools.chain(self.custom_completers.s_matches(cmd), + try_magic, + self.custom_completers.flat_matches(self.text_until_cursor)): + try: + res = c(event) + if res: + # first, try case sensitive match + withcase = [r for r in res if r.startswith(text)] + if withcase: + return withcase + # if none, then case insensitive ones are ok too + text_low = text.lower() + return [r for r in res if r.lower().startswith(text_low)] + except TryNext: + pass + except KeyboardInterrupt: + """ + If custom completer take too long, + let keyboard interrupt abort and return nothing. + """ + break + + return None + + def completions(self, text: str, offset: int)->Iterator[Completion]: + """ + Returns an iterator over the possible completions + + .. warning:: + + Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + Parameters + ---------- + + text:str + Full text of the current input, multi line string. + offset:int + Integer representing the position of the cursor in ``text``. Offset + is 0-based indexed. + + Yields + ------ + :any:`Completion` object + + + The cursor on a text can either be seen as being "in between" + characters or "On" a character depending on the interface visible to + the user. For consistency the cursor being on "in between" characters X + and Y is equivalent to the cursor being "on" character Y, that is to say + the character the cursor is on is considered as being after the cursor. + + Combining characters may span more that one position in the + text. + + + .. note:: + + If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--`` + fake Completion token to distinguish completion returned by Jedi + and usual IPython completion. + + .. note:: + + Completions are not completely deduplicated yet. If identical + completions are coming from different sources this function does not + ensure that each completion object will only be present once. + """ + warnings.warn("_complete is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + seen = set() + try: + for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000): + if c and (c in seen): + continue + yield c + seen.add(c) + except KeyboardInterrupt: + """if completions take too long and users send keyboard interrupt, + do not crash and return ASAP. """ + pass + + def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]: + """ + Core completion module.Same signature as :any:`completions`, with the + extra `timeout` parameter (in seconds). + + + Computing jedi's completion ``.type`` can be quite expensive (it is a + lazy property) and can require some warm-up, more warm up than just + computing the ``name`` of a completion. The warm-up can be : + + - Long warm-up the first time a module is encountered after + install/update: actually build parse/inference tree. + + - first time the module is encountered in a session: load tree from + disk. + + We don't want to block completions for tens of seconds so we give the + completer a "budget" of ``_timeout`` seconds per invocation to compute + completions types, the completions that have not yet been computed will + be marked as "unknown" an will have a chance to be computed next round + are things get cached. + + Keep in mind that Jedi is not the only thing treating the completion so + keep the timeout short-ish as if we take more than 0.3 second we still + have lots of processing to do. + + """ + deadline = time.monotonic() + _timeout + + + before = full_text[:offset] + cursor_line, cursor_column = position_to_cursor(full_text, offset) + + matched_text, matches, matches_origin, jedi_matches = self._complete( + full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column) + + iter_jm = iter(jedi_matches) + if _timeout: + for jm in iter_jm: + try: + type_ = jm.type + except Exception: + if self.debug: + print("Error in Jedi getting type of ", jm) + type_ = None + delta = len(jm.name_with_symbols) - len(jm.complete) + if type_ == 'function': + signature = _make_signature(jm) + else: + signature = '' + yield Completion(start=offset - delta, + end=offset, + text=jm.name_with_symbols, + type=type_, + signature=signature, + _origin='jedi') + + if time.monotonic() > deadline: + break + + for jm in iter_jm: + delta = len(jm.name_with_symbols) - len(jm.complete) + yield Completion(start=offset - delta, + end=offset, + text=jm.name_with_symbols, + type='', # don't compute type for speed + _origin='jedi', + signature='') + + + start_offset = before.rfind(matched_text) + + # TODO: + # Suppress this, right now just for debug. + if jedi_matches and matches and self.debug: + yield Completion(start=start_offset, end=offset, text='--jedi/ipython--', + _origin='debug', type='none', signature='') + + # I'm unsure if this is always true, so let's assert and see if it + # crash + assert before.endswith(matched_text) + for m, t in zip(matches, matches_origin): + yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='') + + + def complete(self, text=None, line_buffer=None, cursor_pos=None): + """Find completions for the given text and line context. + + Note that both the text and the line_buffer are optional, but at least + one of them must be given. + + Parameters + ---------- + text : string, optional + Text to perform the completion on. If not given, the line buffer + is split using the instance's CompletionSplitter object. + + line_buffer : string, optional + If not given, the completer attempts to obtain the current line + buffer via readline. This keyword allows clients which are + requesting for text completions in non-readline contexts to inform + the completer of the entire text. + + cursor_pos : int, optional + Index of the cursor in the full line buffer. Should be provided by + remote frontends where kernel has no access to frontend state. + + Returns + ------- + text : str + Text that was actually used in the completion. + + matches : list + A list of completion matches. + + + .. note:: + + This API is likely to be deprecated and replaced by + :any:`IPCompleter.completions` in the future. + + + """ + warnings.warn('`Completer.complete` is pending deprecation since ' + 'IPython 6.0 and will be replaced by `Completer.completions`.', + PendingDeprecationWarning) + # potential todo, FOLD the 3rd throw away argument of _complete + # into the first 2 one. + return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2] + + def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, + full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]: + """ + + Like complete but can also returns raw jedi completions as well as the + origin of the completion text. This could (and should) be made much + cleaner but that will be simpler once we drop the old (and stateful) + :any:`complete` API. + + + With current provisional API, cursor_pos act both (depending on the + caller) as the offset in the ``text`` or ``line_buffer``, or as the + ``column`` when passing multiline strings this could/should be renamed + but would add extra noise. + """ + + # if the cursor position isn't given, the only sane assumption we can + # make is that it's at the end of the line (the common case) + if cursor_pos is None: + cursor_pos = len(line_buffer) if text is None else len(text) + + if self.use_main_ns: + self.namespace = __main__.__dict__ + + # if text is either None or an empty string, rely on the line buffer + if (not line_buffer) and full_text: + line_buffer = full_text.split('\n')[cursor_line] + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' + + if self.backslash_combining_completions: + # allow deactivation of these on windows. + base_text = text if not line_buffer else line_buffer[:cursor_pos] + latex_text, latex_matches = self.latex_matches(base_text) + if latex_matches: + return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () + name_text = '' + name_matches = [] + # need to add self.fwd_unicode_match() function here when done + for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): + name_text, name_matches = meth(base_text) + if name_text: + return name_text, name_matches[:MATCHES_LIMIT], \ + [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), () + + + # If no line buffer is given, assume the input text is all there was + if line_buffer is None: + line_buffer = text + + self.line_buffer = line_buffer + self.text_until_cursor = self.line_buffer[:cursor_pos] + + # Do magic arg matches + for matcher in self.magic_arg_matchers: + matches = list(matcher(line_buffer))[:MATCHES_LIMIT] + if matches: + origins = [matcher.__qualname__] * len(matches) + return text, matches, origins, () + + # Start with a clean slate of completions + matches = [] + + # FIXME: we should extend our api to return a dict with completions for + # different types of objects. The rlcomplete() method could then + # simply collapse the dict into a list for readline, but we'd have + # richer completion semantics in other environments. + completions = () + if self.use_jedi: + if not full_text: + full_text = line_buffer + completions = self._jedi_matches( + cursor_pos, cursor_line, full_text) + + if self.merge_completions: + matches = [] + for matcher in self.matchers: + try: + matches.extend([(m, matcher.__qualname__) + for m in matcher(text)]) + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) + else: + for matcher in self.matchers: + matches = [(m, matcher.__qualname__) + for m in matcher(text)] + if matches: + break + + seen = set() + filtered_matches = set() + for m in matches: + t, c = m + if t not in seen: + filtered_matches.add(m) + seen.add(t) + + _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) + + custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] + + _filtered_matches = custom_res or _filtered_matches + + _filtered_matches = _filtered_matches[:MATCHES_LIMIT] + _matches = [m[0] for m in _filtered_matches] + origins = [m[1] for m in _filtered_matches] + + self.matches = _matches + + return text, _matches, origins, completions + + def fwd_unicode_match(self, text:str) -> Tuple[str, list]: + if self._names is None: + self._names = [] + for c in range(0,0x10FFFF + 1): + try: + self._names.append(unicodedata.name(chr(c))) + except ValueError: + pass + + slashpos = text.rfind('\\') + # if text starts with slash + if slashpos > -1: + s = text[slashpos+1:] + candidates = [x for x in self._names if x.startswith(s)] + if candidates: + return s, candidates + else: + return '', () + + # if text does not start with slash + else: + return u'', () diff --git a/contrib/python/ipython/py3/IPython/core/completerlib.py b/contrib/python/ipython/py3/IPython/core/completerlib.py index 6f9ea485136..bda665d8a2b 100644 --- a/contrib/python/ipython/py3/IPython/core/completerlib.py +++ b/contrib/python/ipython/py3/IPython/core/completerlib.py @@ -1,402 +1,402 @@ -# encoding: utf-8 -"""Implementations for various useful completers. - -These are all loaded by default by IPython. -""" -#----------------------------------------------------------------------------- -# Copyright (C) 2010-2011 The IPython Development Team. -# -# Distributed under the terms of the BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -# Stdlib imports -import glob -import inspect -import itertools -import os -import re -import sys -from importlib import import_module -from importlib.machinery import all_suffixes - - -# Third-party imports -from time import time -from zipimport import zipimporter - -# Our own imports -from .completer import expand_user, compress_user -from .error import TryNext -from ..utils._process_common import arg_split - -# FIXME: this should be pulled in with the right call via the component system -from IPython import get_ipython - -from typing import List - -from __res import importer - -#----------------------------------------------------------------------------- -# Globals and constants -#----------------------------------------------------------------------------- -_suffixes = all_suffixes() - -# Time in seconds after which the rootmodules will be stored permanently in the -# ipython ip.db database (kept in the user's .ipython dir). -TIMEOUT_STORAGE = 2 - -# Time in seconds after which we give up -TIMEOUT_GIVEUP = 20 - -# Regular expression for the python import statement -import_re = re.compile(r'(?P[^\W\d]\w*?)' - r'(?P[/\\]__init__)?' - r'(?P%s)$' % - r'|'.join(re.escape(s) for s in _suffixes)) - -# RE for the ipython %run command (python + ipython scripts) -magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$') - -#----------------------------------------------------------------------------- -# Local utilities -#----------------------------------------------------------------------------- - -arcadia_rootmodules_cache = None -arcadia_modules_cache = None - - -def arcadia_init_cache(): - global arcadia_rootmodules_cache, arcadia_modules_cache - arcadia_rootmodules_cache = set() - arcadia_modules_cache = {} - - all_modules = itertools.chain( - sys.builtin_module_names, - importer.memory - ) - - for name in all_modules: - path = name.split('.') - arcadia_rootmodules_cache.add(path[0]) - - prefix = path[0] - for element in path[1:]: - if element == '__init__': - continue - - arcadia_modules_cache.setdefault(prefix, set()).add(element) - prefix += '.' + element - - arcadia_rootmodules_cache = sorted(arcadia_rootmodules_cache) - arcadia_modules_cache = {k: sorted(v) for k, v in arcadia_modules_cache.items()} - - -def arcadia_module_list(mod): - if arcadia_modules_cache is None: - arcadia_init_cache() - - return arcadia_modules_cache.get(mod, ()) - - -def arcadia_get_root_modules(): - if arcadia_rootmodules_cache is None: - arcadia_init_cache() - - return arcadia_rootmodules_cache - - -def module_list(path): - """ - Return the list containing the names of the modules available in the given - folder. - """ - # sys.path has the cwd as an empty string, but isdir/listdir need it as '.' - if path == '': - path = '.' - - # A few local constants to be used in loops below - pjoin = os.path.join - - if os.path.isdir(path): - # Build a list of all files in the directory and all files - # in its subdirectories. For performance reasons, do not - # recurse more than one level into subdirectories. - files = [] - for root, dirs, nondirs in os.walk(path, followlinks=True): - subdir = root[len(path)+1:] - if subdir: - files.extend(pjoin(subdir, f) for f in nondirs) - dirs[:] = [] # Do not recurse into additional subdirectories. - else: - files.extend(nondirs) - - else: - try: - files = list(zipimporter(path)._files.keys()) - except: - files = [] - - # Build a list of modules which match the import_re regex. - modules = [] - for f in files: - m = import_re.match(f) - if m: - modules.append(m.group('name')) - return list(set(modules)) - - -def get_root_modules(): - """ - Returns a list containing the names of all the modules available in the - folders of the pythonpath. - - ip.db['rootmodules_cache'] maps sys.path entries to list of modules. - """ - ip = get_ipython() - if ip is None: - # No global shell instance to store cached list of modules. - # Don't try to scan for modules every time. - return list(sys.builtin_module_names) - - rootmodules_cache = ip.db.get('rootmodules_cache', {}) - rootmodules = list(sys.builtin_module_names) - start_time = time() - store = False - for path in sys.path: - try: - modules = rootmodules_cache[path] - except KeyError: - modules = module_list(path) - try: - modules.remove('__init__') - except ValueError: - pass - if path not in ('', '.'): # cwd modules should not be cached - rootmodules_cache[path] = modules - if time() - start_time > TIMEOUT_STORAGE and not store: - store = True - print("\nCaching the list of root modules, please wait!") - print("(This will only be done once - type '%rehashx' to " - "reset cache!)\n") - sys.stdout.flush() - if time() - start_time > TIMEOUT_GIVEUP: - print("This is taking too long, we give up.\n") - return [] - rootmodules.extend(modules) - if store: - ip.db['rootmodules_cache'] = rootmodules_cache - rootmodules = list(set(rootmodules)) - return rootmodules - - -def is_importable(module, attr, only_modules): - if only_modules: - return inspect.ismodule(getattr(module, attr)) - else: - return not(attr[:2] == '__' and attr[-2:] == '__') - - -def try_import(mod: str, only_modules=False) -> List[str]: - """ - Try to import given module and return list of potential completions. - """ - mod = mod.rstrip('.') - try: - m = import_module(mod) - except: - return [] - - filename = getattr(m, '__file__', '') - m_is_init = '__init__' in (filename or '') or filename == mod - - completions = [] - if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init: - completions.extend( [attr for attr in dir(m) if - is_importable(m, attr, only_modules)]) - - completions.extend(getattr(m, '__all__', [])) - if m_is_init: - completions.extend(arcadia_module_list(mod)) - completions_set = {c for c in completions if isinstance(c, str)} - completions_set.discard('__init__') - return sorted(completions_set) - - -#----------------------------------------------------------------------------- -# Completion-related functions. -#----------------------------------------------------------------------------- - -def quick_completer(cmd, completions): - r""" Easily create a trivial completer for a command. - - Takes either a list of completions, or all completions in string (that will - be split on whitespace). - - Example:: - - [d:\ipython]|1> import ipy_completers - [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz']) - [d:\ipython]|3> foo b - bar baz - [d:\ipython]|3> foo ba - """ - - if isinstance(completions, str): - completions = completions.split() - - def do_complete(self, event): - return completions - - get_ipython().set_hook('complete_command',do_complete, str_key = cmd) - -def module_completion(line): - """ - Returns a list containing the completion possibilities for an import line. - - The line looks like this : - 'import xml.d' - 'from xml.dom import' - """ - - words = line.split(' ') - nwords = len(words) - - # from whatever -> 'import ' - if nwords == 3 and words[0] == 'from': - return ['import '] - - # 'from xy' or 'import xy' - if nwords < 3 and (words[0] in {'%aimport', 'import', 'from'}) : - if nwords == 1: - return arcadia_get_root_modules() - mod = words[1].split('.') - if len(mod) < 2: - return arcadia_get_root_modules() - completion_list = try_import('.'.join(mod[:-1]), True) - return ['.'.join(mod[:-1] + [el]) for el in completion_list] - - # 'from xyz import abc' - if nwords >= 3 and words[0] == 'from': - mod = words[1] - return try_import(mod) - -#----------------------------------------------------------------------------- -# Completers -#----------------------------------------------------------------------------- -# These all have the func(self, event) signature to be used as custom -# completers - -def module_completer(self,event): - """Give completions after user has typed 'import ...' or 'from ...'""" - - # This works in all versions of python. While 2.5 has - # pkgutil.walk_packages(), that particular routine is fairly dangerous, - # since it imports *EVERYTHING* on sys.path. That is: a) very slow b) full - # of possibly problematic side effects. - # This search the folders in the sys.path for available modules. - - return module_completion(event.line) - -# FIXME: there's a lot of logic common to the run, cd and builtin file -# completers, that is currently reimplemented in each. - -def magic_run_completer(self, event): - """Complete files that end in .py or .ipy or .ipynb for the %run command. - """ - comps = arg_split(event.line, strict=False) - # relpath should be the current token that we need to complete. - if (len(comps) > 1) and (not event.line.endswith(' ')): - relpath = comps[-1].strip("'\"") - else: - relpath = '' - - #print("\nev=", event) # dbg - #print("rp=", relpath) # dbg - #print('comps=', comps) # dbg - - lglob = glob.glob - isdir = os.path.isdir - relpath, tilde_expand, tilde_val = expand_user(relpath) - - # Find if the user has already typed the first filename, after which we - # should complete on all files, since after the first one other files may - # be arguments to the input script. - - if any(magic_run_re.match(c) for c in comps): - matches = [f.replace('\\','/') + ('/' if isdir(f) else '') - for f in lglob(relpath+'*')] - else: - dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)] - pys = [f.replace('\\','/') - for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') + - lglob(relpath+'*.ipynb') + lglob(relpath + '*.pyw')] - - matches = dirs + pys - - #print('run comp:', dirs+pys) # dbg - return [compress_user(p, tilde_expand, tilde_val) for p in matches] - - -def cd_completer(self, event): - """Completer function for cd, which only returns directories.""" - ip = get_ipython() - relpath = event.symbol - - #print(event) # dbg - if event.line.endswith('-b') or ' -b ' in event.line: - # return only bookmark completions - bkms = self.db.get('bookmarks', None) - if bkms: - return bkms.keys() - else: - return [] - - if event.symbol == '-': - width_dh = str(len(str(len(ip.user_ns['_dh']) + 1))) - # jump in directory history by number - fmt = '-%0' + width_dh +'d [%s]' - ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])] - if len(ents) > 1: - return ents - return [] - - if event.symbol.startswith('--'): - return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']] - - # Expand ~ in path and normalize directory separators. - relpath, tilde_expand, tilde_val = expand_user(relpath) - relpath = relpath.replace('\\','/') - - found = [] - for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*') - if os.path.isdir(f)]: - if ' ' in d: - # we don't want to deal with any of that, complex code - # for this is elsewhere - raise TryNext - - found.append(d) - - if not found: - if os.path.isdir(relpath): - return [compress_user(relpath, tilde_expand, tilde_val)] - - # if no completions so far, try bookmarks - bks = self.db.get('bookmarks',{}) - bkmatches = [s for s in bks if s.startswith(event.symbol)] - if bkmatches: - return bkmatches - - raise TryNext - - return [compress_user(p, tilde_expand, tilde_val) for p in found] - -def reset_completer(self, event): - "A completer for %reset magic" - return '-f -s in out array dhist'.split() +# encoding: utf-8 +"""Implementations for various useful completers. + +These are all loaded by default by IPython. +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010-2011 The IPython Development Team. +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib imports +import glob +import inspect +import itertools +import os +import re +import sys +from importlib import import_module +from importlib.machinery import all_suffixes + + +# Third-party imports +from time import time +from zipimport import zipimporter + +# Our own imports +from .completer import expand_user, compress_user +from .error import TryNext +from ..utils._process_common import arg_split + +# FIXME: this should be pulled in with the right call via the component system +from IPython import get_ipython + +from typing import List + +from __res import importer + +#----------------------------------------------------------------------------- +# Globals and constants +#----------------------------------------------------------------------------- +_suffixes = all_suffixes() + +# Time in seconds after which the rootmodules will be stored permanently in the +# ipython ip.db database (kept in the user's .ipython dir). +TIMEOUT_STORAGE = 2 + +# Time in seconds after which we give up +TIMEOUT_GIVEUP = 20 + +# Regular expression for the python import statement +import_re = re.compile(r'(?P[^\W\d]\w*?)' + r'(?P[/\\]__init__)?' + r'(?P%s)$' % + r'|'.join(re.escape(s) for s in _suffixes)) + +# RE for the ipython %run command (python + ipython scripts) +magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$') + +#----------------------------------------------------------------------------- +# Local utilities +#----------------------------------------------------------------------------- + +arcadia_rootmodules_cache = None +arcadia_modules_cache = None + + +def arcadia_init_cache(): + global arcadia_rootmodules_cache, arcadia_modules_cache + arcadia_rootmodules_cache = set() + arcadia_modules_cache = {} + + all_modules = itertools.chain( + sys.builtin_module_names, + importer.memory + ) + + for name in all_modules: + path = name.split('.') + arcadia_rootmodules_cache.add(path[0]) + + prefix = path[0] + for element in path[1:]: + if element == '__init__': + continue + + arcadia_modules_cache.setdefault(prefix, set()).add(element) + prefix += '.' + element + + arcadia_rootmodules_cache = sorted(arcadia_rootmodules_cache) + arcadia_modules_cache = {k: sorted(v) for k, v in arcadia_modules_cache.items()} + + +def arcadia_module_list(mod): + if arcadia_modules_cache is None: + arcadia_init_cache() + + return arcadia_modules_cache.get(mod, ()) + + +def arcadia_get_root_modules(): + if arcadia_rootmodules_cache is None: + arcadia_init_cache() + + return arcadia_rootmodules_cache + + +def module_list(path): + """ + Return the list containing the names of the modules available in the given + folder. + """ + # sys.path has the cwd as an empty string, but isdir/listdir need it as '.' + if path == '': + path = '.' + + # A few local constants to be used in loops below + pjoin = os.path.join + + if os.path.isdir(path): + # Build a list of all files in the directory and all files + # in its subdirectories. For performance reasons, do not + # recurse more than one level into subdirectories. + files = [] + for root, dirs, nondirs in os.walk(path, followlinks=True): + subdir = root[len(path)+1:] + if subdir: + files.extend(pjoin(subdir, f) for f in nondirs) + dirs[:] = [] # Do not recurse into additional subdirectories. + else: + files.extend(nondirs) + + else: + try: + files = list(zipimporter(path)._files.keys()) + except: + files = [] + + # Build a list of modules which match the import_re regex. + modules = [] + for f in files: + m = import_re.match(f) + if m: + modules.append(m.group('name')) + return list(set(modules)) + + +def get_root_modules(): + """ + Returns a list containing the names of all the modules available in the + folders of the pythonpath. + + ip.db['rootmodules_cache'] maps sys.path entries to list of modules. + """ + ip = get_ipython() + if ip is None: + # No global shell instance to store cached list of modules. + # Don't try to scan for modules every time. + return list(sys.builtin_module_names) + + rootmodules_cache = ip.db.get('rootmodules_cache', {}) + rootmodules = list(sys.builtin_module_names) + start_time = time() + store = False + for path in sys.path: + try: + modules = rootmodules_cache[path] + except KeyError: + modules = module_list(path) + try: + modules.remove('__init__') + except ValueError: + pass + if path not in ('', '.'): # cwd modules should not be cached + rootmodules_cache[path] = modules + if time() - start_time > TIMEOUT_STORAGE and not store: + store = True + print("\nCaching the list of root modules, please wait!") + print("(This will only be done once - type '%rehashx' to " + "reset cache!)\n") + sys.stdout.flush() + if time() - start_time > TIMEOUT_GIVEUP: + print("This is taking too long, we give up.\n") + return [] + rootmodules.extend(modules) + if store: + ip.db['rootmodules_cache'] = rootmodules_cache + rootmodules = list(set(rootmodules)) + return rootmodules + + +def is_importable(module, attr, only_modules): + if only_modules: + return inspect.ismodule(getattr(module, attr)) + else: + return not(attr[:2] == '__' and attr[-2:] == '__') + + +def try_import(mod: str, only_modules=False) -> List[str]: + """ + Try to import given module and return list of potential completions. + """ + mod = mod.rstrip('.') + try: + m = import_module(mod) + except: + return [] + + filename = getattr(m, '__file__', '') + m_is_init = '__init__' in (filename or '') or filename == mod + + completions = [] + if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init: + completions.extend( [attr for attr in dir(m) if + is_importable(m, attr, only_modules)]) + + completions.extend(getattr(m, '__all__', [])) + if m_is_init: + completions.extend(arcadia_module_list(mod)) + completions_set = {c for c in completions if isinstance(c, str)} + completions_set.discard('__init__') + return sorted(completions_set) + + +#----------------------------------------------------------------------------- +# Completion-related functions. +#----------------------------------------------------------------------------- + +def quick_completer(cmd, completions): + r""" Easily create a trivial completer for a command. + + Takes either a list of completions, or all completions in string (that will + be split on whitespace). + + Example:: + + [d:\ipython]|1> import ipy_completers + [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz']) + [d:\ipython]|3> foo b + bar baz + [d:\ipython]|3> foo ba + """ + + if isinstance(completions, str): + completions = completions.split() + + def do_complete(self, event): + return completions + + get_ipython().set_hook('complete_command',do_complete, str_key = cmd) + +def module_completion(line): + """ + Returns a list containing the completion possibilities for an import line. + + The line looks like this : + 'import xml.d' + 'from xml.dom import' + """ + + words = line.split(' ') + nwords = len(words) + + # from whatever -> 'import ' + if nwords == 3 and words[0] == 'from': + return ['import '] + + # 'from xy' or 'import xy' + if nwords < 3 and (words[0] in {'%aimport', 'import', 'from'}) : + if nwords == 1: + return arcadia_get_root_modules() + mod = words[1].split('.') + if len(mod) < 2: + return arcadia_get_root_modules() + completion_list = try_import('.'.join(mod[:-1]), True) + return ['.'.join(mod[:-1] + [el]) for el in completion_list] + + # 'from xyz import abc' + if nwords >= 3 and words[0] == 'from': + mod = words[1] + return try_import(mod) + +#----------------------------------------------------------------------------- +# Completers +#----------------------------------------------------------------------------- +# These all have the func(self, event) signature to be used as custom +# completers + +def module_completer(self,event): + """Give completions after user has typed 'import ...' or 'from ...'""" + + # This works in all versions of python. While 2.5 has + # pkgutil.walk_packages(), that particular routine is fairly dangerous, + # since it imports *EVERYTHING* on sys.path. That is: a) very slow b) full + # of possibly problematic side effects. + # This search the folders in the sys.path for available modules. + + return module_completion(event.line) + +# FIXME: there's a lot of logic common to the run, cd and builtin file +# completers, that is currently reimplemented in each. + +def magic_run_completer(self, event): + """Complete files that end in .py or .ipy or .ipynb for the %run command. + """ + comps = arg_split(event.line, strict=False) + # relpath should be the current token that we need to complete. + if (len(comps) > 1) and (not event.line.endswith(' ')): + relpath = comps[-1].strip("'\"") + else: + relpath = '' + + #print("\nev=", event) # dbg + #print("rp=", relpath) # dbg + #print('comps=', comps) # dbg + + lglob = glob.glob + isdir = os.path.isdir + relpath, tilde_expand, tilde_val = expand_user(relpath) + + # Find if the user has already typed the first filename, after which we + # should complete on all files, since after the first one other files may + # be arguments to the input script. + + if any(magic_run_re.match(c) for c in comps): + matches = [f.replace('\\','/') + ('/' if isdir(f) else '') + for f in lglob(relpath+'*')] + else: + dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)] + pys = [f.replace('\\','/') + for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') + + lglob(relpath+'*.ipynb') + lglob(relpath + '*.pyw')] + + matches = dirs + pys + + #print('run comp:', dirs+pys) # dbg + return [compress_user(p, tilde_expand, tilde_val) for p in matches] + + +def cd_completer(self, event): + """Completer function for cd, which only returns directories.""" + ip = get_ipython() + relpath = event.symbol + + #print(event) # dbg + if event.line.endswith('-b') or ' -b ' in event.line: + # return only bookmark completions + bkms = self.db.get('bookmarks', None) + if bkms: + return bkms.keys() + else: + return [] + + if event.symbol == '-': + width_dh = str(len(str(len(ip.user_ns['_dh']) + 1))) + # jump in directory history by number + fmt = '-%0' + width_dh +'d [%s]' + ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])] + if len(ents) > 1: + return ents + return [] + + if event.symbol.startswith('--'): + return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']] + + # Expand ~ in path and normalize directory separators. + relpath, tilde_expand, tilde_val = expand_user(relpath) + relpath = relpath.replace('\\','/') + + found = [] + for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*') + if os.path.isdir(f)]: + if ' ' in d: + # we don't want to deal with any of that, complex code + # for this is elsewhere + raise TryNext + + found.append(d) + + if not found: + if os.path.isdir(relpath): + return [compress_user(relpath, tilde_expand, tilde_val)] + + # if no completions so far, try bookmarks + bks = self.db.get('bookmarks',{}) + bkmatches = [s for s in bks if s.startswith(event.symbol)] + if bkmatches: + return bkmatches + + raise TryNext + + return [compress_user(p, tilde_expand, tilde_val) for p in found] + +def reset_completer(self, event): + "A completer for %reset magic" + return '-f -s in out array dhist'.split() diff --git a/contrib/python/ipython/py3/IPython/core/crashhandler.py b/contrib/python/ipython/py3/IPython/core/crashhandler.py index b39aac4c1f2..1e0b429d09a 100644 --- a/contrib/python/ipython/py3/IPython/core/crashhandler.py +++ b/contrib/python/ipython/py3/IPython/core/crashhandler.py @@ -1,228 +1,228 @@ -# encoding: utf-8 -"""sys.excepthook for IPython itself, leaves a detailed report on disk. - -Authors: - -* Fernando Perez -* Brian E. Granger -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2001-2007 Fernando Perez. -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import os -import sys -import traceback -from pprint import pformat - -from IPython.core import ultratb -from IPython.core.release import author_email -from IPython.utils.sysinfo import sys_info -from IPython.utils.py3compat import input - -from IPython.core.release import __version__ as version - -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - -# Template for the user message. -_default_message_template = """\ -Oops, {app_name} crashed. We do our best to make it stable, but... - -A crash report was automatically generated with the following information: - - A verbatim copy of the crash traceback. - - A copy of your input history during this session. - - Data on your current {app_name} configuration. - -It was left in the file named: -\t'{crash_report_fname}' -If you can email this file to the developers, the information in it will help -them in understanding and correcting the problem. - -You can mail it to: {contact_name} at {contact_email} -with the subject '{app_name} Crash Report'. - -If you want to do it now, the following command will work (under Unix): -mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname} - -In your email, please also include information about: -- The operating system under which the crash happened: Linux, macOS, Windows, - other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2, - Windows 10 Pro), and whether it is 32-bit or 64-bit; -- How {app_name} was installed: using pip or conda, from GitHub, as part of - a Docker container, or other, providing more detail if possible; -- How to reproduce the crash: what exact sequence of instructions can one - input to get the same crash? Ideally, find a minimal yet complete sequence - of instructions that yields the crash. - -To ensure accurate tracking of this issue, please file a report about it at: -{bug_tracker} -""" - -_lite_message_template = """ -If you suspect this is an IPython {version} bug, please report it at: - https://github.com/ipython/ipython/issues -or send an email to the mailing list at {email} - -You can print a more detailed traceback right now with "%tb", or use "%debug" -to interactively debug it. - -Extra-detailed tracebacks for bug-reporting purposes can be enabled via: - {config}Application.verbose_crash=True -""" - - -class CrashHandler(object): - """Customizable crash handlers for IPython applications. - - Instances of this class provide a :meth:`__call__` method which can be - used as a ``sys.excepthook``. The :meth:`__call__` signature is:: - - def __call__(self, etype, evalue, etb) - """ - - message_template = _default_message_template - section_sep = '\n\n'+'*'*75+'\n\n' - - def __init__(self, app, contact_name=None, contact_email=None, - bug_tracker=None, show_crash_traceback=True, call_pdb=False): - """Create a new crash handler - - Parameters - ---------- - app : Application - A running :class:`Application` instance, which will be queried at - crash time for internal information. - - contact_name : str - A string with the name of the person to contact. - - contact_email : str - A string with the email address of the contact. - - bug_tracker : str - A string with the URL for your project's bug tracker. - - show_crash_traceback : bool - If false, don't print the crash traceback on stderr, only generate - the on-disk report - - Non-argument instance attributes: - - These instances contain some non-argument attributes which allow for - further customization of the crash handler's behavior. Please see the - source for further details. - """ - self.crash_report_fname = "Crash_report_%s.txt" % app.name - self.app = app - self.call_pdb = call_pdb - #self.call_pdb = True # dbg - self.show_crash_traceback = show_crash_traceback - self.info = dict(app_name = app.name, - contact_name = contact_name, - contact_email = contact_email, - bug_tracker = bug_tracker, - crash_report_fname = self.crash_report_fname) - - - def __call__(self, etype, evalue, etb): - """Handle an exception, call for compatible with sys.excepthook""" - - # do not allow the crash handler to be called twice without reinstalling it - # this prevents unlikely errors in the crash handling from entering an - # infinite loop. - sys.excepthook = sys.__excepthook__ - - # Report tracebacks shouldn't use color in general (safer for users) - color_scheme = 'NoColor' - - # Use this ONLY for developer debugging (keep commented out for release) - #color_scheme = 'Linux' # dbg - try: - rptdir = self.app.ipython_dir - except: - rptdir = os.getcwd() - if rptdir is None or not os.path.isdir(rptdir): - rptdir = os.getcwd() - report_name = os.path.join(rptdir,self.crash_report_fname) - # write the report filename into the instance dict so it can get - # properly expanded out in the user message template - self.crash_report_fname = report_name - self.info['crash_report_fname'] = report_name - TBhandler = ultratb.VerboseTB( - color_scheme=color_scheme, - long_header=1, - call_pdb=self.call_pdb, - ) - if self.call_pdb: - TBhandler(etype,evalue,etb) - return - else: - traceback = TBhandler.text(etype,evalue,etb,context=31) - - # print traceback to screen - if self.show_crash_traceback: - print(traceback, file=sys.stderr) - - # and generate a complete report on disk - try: - report = open(report_name,'w') - except: - print('Could not create crash report on disk.', file=sys.stderr) - return - - with report: - # Inform user on stderr of what happened - print('\n'+'*'*70+'\n', file=sys.stderr) - print(self.message_template.format(**self.info), file=sys.stderr) - - # Construct report on disk - report.write(self.make_report(traceback)) - - input("Hit to quit (your terminal may close):") - - def make_report(self,traceback): - """Return a string containing a crash report.""" - - sec_sep = self.section_sep - - report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n'] - rpt_add = report.append - rpt_add(sys_info()) - - try: - config = pformat(self.app.config) - rpt_add(sec_sep) - rpt_add('Application name: %s\n\n' % self.app_name) - rpt_add('Current user configuration structure:\n\n') - rpt_add(config) - except: - pass - rpt_add(sec_sep+'Crash traceback:\n\n' + traceback) - - return ''.join(report) - - -def crash_handler_lite(etype, evalue, tb): - """a light excepthook, adding a small message to the usual traceback""" - traceback.print_exception(etype, evalue, tb) - - from IPython.core.interactiveshell import InteractiveShell - if InteractiveShell.initialized(): - # we are in a Shell environment, give %magic example - config = "%config " - else: - # we are not in a shell, show generic config - config = "c." - print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) - +# encoding: utf-8 +"""sys.excepthook for IPython itself, leaves a detailed report on disk. + +Authors: + +* Fernando Perez +* Brian E. Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2001-2007 Fernando Perez. +# Copyright (C) 2008-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import sys +import traceback +from pprint import pformat + +from IPython.core import ultratb +from IPython.core.release import author_email +from IPython.utils.sysinfo import sys_info +from IPython.utils.py3compat import input + +from IPython.core.release import __version__ as version + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + +# Template for the user message. +_default_message_template = """\ +Oops, {app_name} crashed. We do our best to make it stable, but... + +A crash report was automatically generated with the following information: + - A verbatim copy of the crash traceback. + - A copy of your input history during this session. + - Data on your current {app_name} configuration. + +It was left in the file named: +\t'{crash_report_fname}' +If you can email this file to the developers, the information in it will help +them in understanding and correcting the problem. + +You can mail it to: {contact_name} at {contact_email} +with the subject '{app_name} Crash Report'. + +If you want to do it now, the following command will work (under Unix): +mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname} + +In your email, please also include information about: +- The operating system under which the crash happened: Linux, macOS, Windows, + other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2, + Windows 10 Pro), and whether it is 32-bit or 64-bit; +- How {app_name} was installed: using pip or conda, from GitHub, as part of + a Docker container, or other, providing more detail if possible; +- How to reproduce the crash: what exact sequence of instructions can one + input to get the same crash? Ideally, find a minimal yet complete sequence + of instructions that yields the crash. + +To ensure accurate tracking of this issue, please file a report about it at: +{bug_tracker} +""" + +_lite_message_template = """ +If you suspect this is an IPython {version} bug, please report it at: + https://github.com/ipython/ipython/issues +or send an email to the mailing list at {email} + +You can print a more detailed traceback right now with "%tb", or use "%debug" +to interactively debug it. + +Extra-detailed tracebacks for bug-reporting purposes can be enabled via: + {config}Application.verbose_crash=True +""" + + +class CrashHandler(object): + """Customizable crash handlers for IPython applications. + + Instances of this class provide a :meth:`__call__` method which can be + used as a ``sys.excepthook``. The :meth:`__call__` signature is:: + + def __call__(self, etype, evalue, etb) + """ + + message_template = _default_message_template + section_sep = '\n\n'+'*'*75+'\n\n' + + def __init__(self, app, contact_name=None, contact_email=None, + bug_tracker=None, show_crash_traceback=True, call_pdb=False): + """Create a new crash handler + + Parameters + ---------- + app : Application + A running :class:`Application` instance, which will be queried at + crash time for internal information. + + contact_name : str + A string with the name of the person to contact. + + contact_email : str + A string with the email address of the contact. + + bug_tracker : str + A string with the URL for your project's bug tracker. + + show_crash_traceback : bool + If false, don't print the crash traceback on stderr, only generate + the on-disk report + + Non-argument instance attributes: + + These instances contain some non-argument attributes which allow for + further customization of the crash handler's behavior. Please see the + source for further details. + """ + self.crash_report_fname = "Crash_report_%s.txt" % app.name + self.app = app + self.call_pdb = call_pdb + #self.call_pdb = True # dbg + self.show_crash_traceback = show_crash_traceback + self.info = dict(app_name = app.name, + contact_name = contact_name, + contact_email = contact_email, + bug_tracker = bug_tracker, + crash_report_fname = self.crash_report_fname) + + + def __call__(self, etype, evalue, etb): + """Handle an exception, call for compatible with sys.excepthook""" + + # do not allow the crash handler to be called twice without reinstalling it + # this prevents unlikely errors in the crash handling from entering an + # infinite loop. + sys.excepthook = sys.__excepthook__ + + # Report tracebacks shouldn't use color in general (safer for users) + color_scheme = 'NoColor' + + # Use this ONLY for developer debugging (keep commented out for release) + #color_scheme = 'Linux' # dbg + try: + rptdir = self.app.ipython_dir + except: + rptdir = os.getcwd() + if rptdir is None or not os.path.isdir(rptdir): + rptdir = os.getcwd() + report_name = os.path.join(rptdir,self.crash_report_fname) + # write the report filename into the instance dict so it can get + # properly expanded out in the user message template + self.crash_report_fname = report_name + self.info['crash_report_fname'] = report_name + TBhandler = ultratb.VerboseTB( + color_scheme=color_scheme, + long_header=1, + call_pdb=self.call_pdb, + ) + if self.call_pdb: + TBhandler(etype,evalue,etb) + return + else: + traceback = TBhandler.text(etype,evalue,etb,context=31) + + # print traceback to screen + if self.show_crash_traceback: + print(traceback, file=sys.stderr) + + # and generate a complete report on disk + try: + report = open(report_name,'w') + except: + print('Could not create crash report on disk.', file=sys.stderr) + return + + with report: + # Inform user on stderr of what happened + print('\n'+'*'*70+'\n', file=sys.stderr) + print(self.message_template.format(**self.info), file=sys.stderr) + + # Construct report on disk + report.write(self.make_report(traceback)) + + input("Hit to quit (your terminal may close):") + + def make_report(self,traceback): + """Return a string containing a crash report.""" + + sec_sep = self.section_sep + + report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n'] + rpt_add = report.append + rpt_add(sys_info()) + + try: + config = pformat(self.app.config) + rpt_add(sec_sep) + rpt_add('Application name: %s\n\n' % self.app_name) + rpt_add('Current user configuration structure:\n\n') + rpt_add(config) + except: + pass + rpt_add(sec_sep+'Crash traceback:\n\n' + traceback) + + return ''.join(report) + + +def crash_handler_lite(etype, evalue, tb): + """a light excepthook, adding a small message to the usual traceback""" + traceback.print_exception(etype, evalue, tb) + + from IPython.core.interactiveshell import InteractiveShell + if InteractiveShell.initialized(): + # we are in a Shell environment, give %magic example + config = "%config " + else: + # we are not in a shell, show generic config + config = "c." + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) + diff --git a/contrib/python/ipython/py3/IPython/core/debugger.py b/contrib/python/ipython/py3/IPython/core/debugger.py index 0622342239f..1744bdb8a8e 100644 --- a/contrib/python/ipython/py3/IPython/core/debugger.py +++ b/contrib/python/ipython/py3/IPython/core/debugger.py @@ -1,1099 +1,1099 @@ -# -*- coding: utf-8 -*- -""" -Pdb debugger class. - - -This is an extension to PDB which adds a number of new features. -Note that there is also the `IPython.terminal.debugger` class which provides UI -improvements. - -We also strongly recommend to use this via the `ipdb` package, which provides -extra configuration options. - -Among other things, this subclass of PDB: - - supports many IPython magics like pdef/psource - - hide frames in tracebacks based on `__tracebackhide__` - - allows to skip frames based on `__debuggerskip__` - -The skipping and hiding frames are configurable via the `skip_predicates` -command. - -By default, frames from readonly files will be hidden, frames containing -``__tracebackhide__=True`` will be hidden. - -Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent -frames value of ``__debuggerskip__`` is ``True`` will be skipped. - - >>> def helpers_helper(): - ... pass - ... - ... def helper_1(): - ... print("don't step in me") - ... helpers_helpers() # will be stepped over unless breakpoint set. - ... - ... - ... def helper_2(): - ... print("in me neither") - ... - -One can define a decorator that wraps a function between the two helpers: - - >>> def pdb_skipped_decorator(function): - ... - ... - ... def wrapped_fn(*args, **kwargs): - ... __debuggerskip__ = True - ... helper_1() - ... __debuggerskip__ = False - ... result = function(*args, **kwargs) - ... __debuggerskip__ = True - ... helper_2() - ... # setting __debuggerskip__ to False again is not necessary - ... return result - ... - ... return wrapped_fn - -When decorating a function, ipdb will directly step into ``bar()`` by -default: - - >>> @foo_decorator - ... def bar(x, y): - ... return x * y - - -You can toggle the behavior with - - ipdb> skip_predicates debuggerskip false - -or configure it in your ``.pdbrc`` - - - -Licencse --------- - -Modified from the standard pdb.Pdb class to avoid including readline, so that -the command line completion of other programs which include this isn't -damaged. - -In the future, this class will be expanded with improvements over the standard -pdb. - -The original code in this file is mainly lifted out of cmd.py in Python 2.2, -with minor changes. Licensing should therefore be under the standard Python -terms. For details on the PSF (Python Software Foundation) standard license, -see: - -https://docs.python.org/2/license.html - - -All the changes since then are under the same license as IPython. - -""" - -#***************************************************************************** -# -# This file is licensed under the PSF license. -# -# Copyright (C) 2001 Python Software Foundation, www.python.org -# Copyright (C) 2005-2006 Fernando Perez. -# -# -#***************************************************************************** - -import bdb -import functools -import inspect -import linecache -import sys -import warnings -import re -import os - -from IPython import get_ipython -from IPython.utils import PyColorize -from IPython.utils import coloransi, py3compat -from IPython.core.excolors import exception_colors -from IPython.testing.skipdoctest import skip_doctest - - -prompt = 'ipdb> ' - -#We have to check this directly from sys.argv, config struct not yet available -from pdb import Pdb as OldPdb - -# Allow the set_trace code to operate outside of an ipython instance, even if -# it does so with some limitations. The rest of this support is implemented in -# the Tracer constructor. - -DEBUGGERSKIP = "__debuggerskip__" - - -def make_arrow(pad): - """generate the leading arrow in front of traceback or debugger""" - if pad >= 2: - return '-'*(pad-2) + '> ' - elif pad == 1: - return '>' - return '' - - -def BdbQuit_excepthook(et, ev, tb, excepthook=None): - """Exception hook which handles `BdbQuit` exceptions. - - All other exceptions are processed using the `excepthook` - parameter. - """ - warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - if et==bdb.BdbQuit: - print('Exiting Debugger.') - elif excepthook is not None: - excepthook(et, ev, tb) - else: - # Backwards compatibility. Raise deprecation warning? - BdbQuit_excepthook.excepthook_ori(et,ev,tb) - - -def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None): - warnings.warn( - "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - print('Exiting Debugger.') - - -class Tracer(object): - """ - DEPRECATED - - Class for local debugging, similar to pdb.set_trace. - - Instances of this class, when called, behave like pdb.set_trace, but - providing IPython's enhanced capabilities. - - This is implemented as a class which must be initialized in your own code - and not as a standalone function because we need to detect at runtime - whether IPython is already active or not. That detection is done in the - constructor, ensuring that this code plays nicely with a running IPython, - while functioning acceptably (though with limitations) if outside of it. - """ - - @skip_doctest - def __init__(self, colors=None): - """ - DEPRECATED - - Create a local debugger instance. - - Parameters - ---------- - - colors : str, optional - The name of the color scheme to use, it must be one of IPython's - valid color schemes. If not given, the function will default to - the current IPython scheme when running inside IPython, and to - 'NoColor' otherwise. - - Examples - -------- - :: - - from IPython.core.debugger import Tracer; debug_here = Tracer() - - Later in your code:: - - debug_here() # -> will open up the debugger at that point. - - Once the debugger activates, you can use all of its regular commands to - step through code, set breakpoints, etc. See the pdb documentation - from the Python standard library for usage details. - """ - warnings.warn("`Tracer` is deprecated since version 5.1, directly use " - "`IPython.core.debugger.Pdb.set_trace()`", - DeprecationWarning, stacklevel=2) - - ip = get_ipython() - if ip is None: - # Outside of ipython, we set our own exception hook manually - sys.excepthook = functools.partial(BdbQuit_excepthook, - excepthook=sys.excepthook) - def_colors = 'NoColor' - else: - # In ipython, we use its custom exception handler mechanism - def_colors = ip.colors - ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook) - - if colors is None: - colors = def_colors - - # The stdlib debugger internally uses a modified repr from the `repr` - # module, that limits the length of printed strings to a hardcoded - # limit of 30 characters. That much trimming is too aggressive, let's - # at least raise that limit to 80 chars, which should be enough for - # most interactive uses. - try: - from reprlib import aRepr - aRepr.maxstring = 80 - except: - # This is only a user-facing convenience, so any error we encounter - # here can be warned about but can be otherwise ignored. These - # printouts will tell us about problems if this API changes - import traceback - traceback.print_exc() - - self.debugger = Pdb(colors) - - def __call__(self): - """Starts an interactive debugger at the point where called. - - This is similar to the pdb.set_trace() function from the std lib, but - using IPython's enhanced debugger.""" - - self.debugger.set_trace(sys._getframe().f_back) - - -RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+') - - -def strip_indentation(multiline_string): - return RGX_EXTRA_INDENT.sub('', multiline_string) - - -def decorate_fn_with_doc(new_fn, old_fn, additional_text=""): - """Make new_fn have old_fn's doc string. This is particularly useful - for the ``do_...`` commands that hook into the help system. - Adapted from from a comp.lang.python posting - by Duncan Booth.""" - def wrapper(*args, **kw): - return new_fn(*args, **kw) - if old_fn.__doc__: - wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text - return wrapper - - -class Pdb(OldPdb): - """Modified Pdb class, does not load readline. - - for a standalone version that uses prompt_toolkit, see - `IPython.terminal.debugger.TerminalPdb` and - `IPython.terminal.debugger.set_trace()` - - - This debugger can hide and skip frames that are tagged according to some predicates. - See the `skip_predicates` commands. - - """ - - default_predicates = { - "tbhide": True, - "readonly": False, - "ipython_internal": True, - "debuggerskip": True, - } - - def __init__(self, color_scheme=None, completekey=None, - stdin=None, stdout=None, context=5, **kwargs): - """Create a new IPython debugger. - - Parameters - ---------- - color_scheme : default None - Deprecated, do not use. - completekey : default None - Passed to pdb.Pdb. - stdin : default None - Passed to pdb.Pdb. - stdout : default None - Passed to pdb.Pdb. - context : int - Number of lines of source code context to show when - displaying stacktrace information. - **kwargs - Passed to pdb.Pdb. - - Notes - ----- - The possibilities are python version dependent, see the python - docs for more info. - """ - - # Parent constructor: - try: - self.context = int(context) - if self.context <= 0: - raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") - - # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. - OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) - - # IPython changes... - self.shell = get_ipython() - - if self.shell is None: - save_main = sys.modules['__main__'] - # No IPython instance running, we must create one - from IPython.terminal.interactiveshell import \ - TerminalInteractiveShell - self.shell = TerminalInteractiveShell.instance() - # needed by any code which calls __import__("__main__") after - # the debugger was entered. See also #9941. - sys.modules['__main__'] = save_main - - if color_scheme is not None: - warnings.warn( - "The `color_scheme` argument is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - else: - color_scheme = self.shell.colors - - self.aliases = {} - - # Create color table: we copy the default one from the traceback - # module and add a few attributes needed for debugging - self.color_scheme_table = exception_colors() - - # shorthands - C = coloransi.TermColors - cst = self.color_scheme_table - - cst['NoColor'].colors.prompt = C.NoColor - cst['NoColor'].colors.breakpoint_enabled = C.NoColor - cst['NoColor'].colors.breakpoint_disabled = C.NoColor - - cst['Linux'].colors.prompt = C.Green - cst['Linux'].colors.breakpoint_enabled = C.LightRed - cst['Linux'].colors.breakpoint_disabled = C.Red - - cst['LightBG'].colors.prompt = C.Blue - cst['LightBG'].colors.breakpoint_enabled = C.LightRed - cst['LightBG'].colors.breakpoint_disabled = C.Red - - cst['Neutral'].colors.prompt = C.Blue - cst['Neutral'].colors.breakpoint_enabled = C.LightRed - cst['Neutral'].colors.breakpoint_disabled = C.Red - - - # Add a python parser so we can syntax highlight source while - # debugging. - self.parser = PyColorize.Parser(style=color_scheme) - self.set_colors(color_scheme) - - # Set the prompt - the default prompt is '(Pdb)' - self.prompt = prompt - self.skip_hidden = True - self.report_skipped = True - - # list of predicates we use to skip frames - self._predicates = self.default_predicates - - # - def set_colors(self, scheme): - """Shorthand access to the color table scheme selector method.""" - self.color_scheme_table.set_active_scheme(scheme) - self.parser.style = scheme - - def set_trace(self, frame=None): - if frame is None: - frame = sys._getframe().f_back - self.initial_frame = frame - return super().set_trace(frame) - - def _hidden_predicate(self, frame): - """ - Given a frame return whether it it should be hidden or not by IPython. - """ - - if self._predicates["readonly"]: - fname = frame.f_code.co_filename - # we need to check for file existence and interactively define - # function would otherwise appear as RO. - if os.path.isfile(fname) and not os.access(fname, os.W_OK): - return True - - if self._predicates["tbhide"]: - if frame in (self.curframe, getattr(self, "initial_frame", None)): - return False - frame_locals = self._get_frame_locals(frame) - if "__tracebackhide__" not in frame_locals: - return False - return frame_locals["__tracebackhide__"] - return False - - def hidden_frames(self, stack): - """ - Given an index in the stack return wether it should be skipped. - - This is used in up/down and where to skip frames. - """ - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # avoid calling it here to preserve self.curframe_locals. - # Futhermore, there is no good reason to hide the current frame. - ip_hide = [self._hidden_predicate(s[0]) for s in stack] - ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] - if ip_start and self._predicates["ipython_internal"]: - ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] - return ip_hide - - def interaction(self, frame, traceback): - try: - OldPdb.interaction(self, frame, traceback) - except KeyboardInterrupt: - self.stdout.write("\n" + self.shell.get_exception_only()) - - def new_do_frame(self, arg): - OldPdb.do_frame(self, arg) - - def new_do_quit(self, arg): - - if hasattr(self, 'old_all_completions'): - self.shell.Completer.all_completions=self.old_all_completions - - return OldPdb.do_quit(self, arg) - - do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit) - - def new_do_restart(self, arg): - """Restart command. In the context of ipython this is exactly the same - thing as 'quit'.""" - self.msg("Restart doesn't make sense here. Using 'quit' instead.") - return self.do_quit(arg) - - def print_stack_trace(self, context=None): - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - if context is None: - context = self.context - try: - context=int(context) - if context <= 0: - raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") - try: - skipped = 0 - for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): - if hidden and self.skip_hidden: - skipped += 1 - continue - if skipped: - print( - f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" - ) - skipped = 0 - self.print_stack_entry(frame_lineno, context=context) - if skipped: - print( - f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" - ) - except KeyboardInterrupt: - pass - - def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', - context=None): - if context is None: - context = self.context - try: - context=int(context) - if context <= 0: - raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") - print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) - - # vds: >> - frame, lineno = frame_lineno - filename = frame.f_code.co_filename - self.shell.hooks.synchronize_with_editor(filename, lineno, 0) - # vds: << - - def _get_frame_locals(self, frame): - """ " - Acessing f_local of current frame reset the namespace, so we want to avoid - that or the following can happend - - ipdb> foo - "old" - ipdb> foo = "new" - ipdb> foo - "new" - ipdb> where - ipdb> foo - "old" - - So if frame is self.current_frame we instead return self.curframe_locals - - """ - if frame is self.curframe: - return self.curframe_locals - else: - return frame.f_locals - - def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): - if context is None: - context = self.context - try: - context=int(context) - if context <= 0: - print("Context must be a positive integer", file=self.stdout) - except (TypeError, ValueError): - print("Context must be a positive integer", file=self.stdout) - try: - import reprlib # Py 3 - except ImportError: - import repr as reprlib # Py 2 - - ret = [] - - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal) - tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal) - tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, - ColorsNormal) - - frame, lineno = frame_lineno - - return_value = '' - loc_frame = self._get_frame_locals(frame) - if "__return__" in loc_frame: - rv = loc_frame["__return__"] - # return_value += '->' - return_value += reprlib.repr(rv) + "\n" - ret.append(return_value) - - #s = filename + '(' + `lineno` + ')' - filename = self.canonic(frame.f_code.co_filename) - link = tpl_link % py3compat.cast_unicode(filename) - - if frame.f_code.co_name: - func = frame.f_code.co_name - else: - func = "" - - call = "" - if func != "?": - if "__args__" in loc_frame: - args = reprlib.repr(loc_frame["__args__"]) - else: - args = '()' - call = tpl_call % (func, args) - - # The level info should be generated in the same format pdb uses, to - # avoid breaking the pdbtrack functionality of python-mode in *emacs. - if frame is self.curframe: - ret.append('> ') - else: - ret.append(' ') - ret.append(u'%s(%s)%s\n' % (link,lineno,call)) - - start = lineno - 1 - context//2 - lines = linecache.getlines(filename) - start = min(start, len(lines) - context) - start = max(start, 0) - lines = lines[start : start + context] - - for i,line in enumerate(lines): - show_arrow = (start + 1 + i == lineno) - linetpl = (frame is self.curframe or show_arrow) \ - and tpl_line_em \ - or tpl_line - ret.append(self.__format_line(linetpl, filename, - start + 1 + i, line, - arrow = show_arrow) ) - return ''.join(ret) - - def __format_line(self, tpl_line, filename, lineno, line, arrow = False): - bp_mark = "" - bp_mark_color = "" - - new_line, err = self.parser.format2(line, 'str') - if not err: - line = new_line - - bp = None - if lineno in self.get_file_breaks(filename): - bps = self.get_breaks(filename, lineno) - bp = bps[-1] - - if bp: - Colors = self.color_scheme_table.active_colors - bp_mark = str(bp.number) - bp_mark_color = Colors.breakpoint_enabled - if not bp.enabled: - bp_mark_color = Colors.breakpoint_disabled - - numbers_width = 7 - if arrow: - # This is the line with the error - pad = numbers_width - len(str(lineno)) - len(bp_mark) - num = '%s%s' % (make_arrow(pad), str(lineno)) - else: - num = '%*s' % (numbers_width - len(bp_mark), str(lineno)) - - return tpl_line % (bp_mark_color + bp_mark, num, line) - - - def print_list_lines(self, filename, first, last): - """The printing (as opposed to the parsing part of a 'list' - command.""" - try: - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal) - src = [] - if filename == "" and hasattr(self, "_exec_filename"): - filename = self._exec_filename - - for lineno in range(first, last+1): - line = linecache.getline(filename, lineno) - if not line: - break - - if lineno == self.curframe.f_lineno: - line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True) - else: - line = self.__format_line(tpl_line, filename, lineno, line, arrow = False) - - src.append(line) - self.lineno = lineno - - print(''.join(src), file=self.stdout) - - except KeyboardInterrupt: - pass - - def do_skip_predicates(self, args): - """ - Turn on/off individual predicates as to whether a frame should be hidden/skip. - - The global option to skip (or not) hidden frames is set with skip_hidden - - To change the value of a predicate - - skip_predicates key [true|false] - - Call without arguments to see the current values. - - To permanently change the value of an option add the corresponding - command to your ``~/.pdbrc`` file. If you are programmatically using the - Pdb instance you can also change the ``default_predicates`` class - attribute. - """ - if not args.strip(): - print("current predicates:") - for (p, v) in self._predicates.items(): - print(" ", p, ":", v) - return - type_value = args.strip().split(" ") - if len(type_value) != 2: - print( - f"Usage: skip_predicates , with one of {set(self._predicates.keys())}" - ) - return - - type_, value = type_value - if type_ not in self._predicates: - print(f"{type_!r} not in {set(self._predicates.keys())}") - return - if value.lower() not in ("true", "yes", "1", "no", "false", "0"): - print( - f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')" - ) - return - - self._predicates[type_] = value.lower() in ("true", "yes", "1") - if not any(self._predicates.values()): - print( - "Warning, all predicates set to False, skip_hidden may not have any effects." - ) - - def do_skip_hidden(self, arg): - """ - Change whether or not we should skip frames with the - __tracebackhide__ attribute. - """ - if not arg.strip(): - print( - f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change." - ) - elif arg.strip().lower() in ("true", "yes"): - self.skip_hidden = True - elif arg.strip().lower() in ("false", "no"): - self.skip_hidden = False - if not any(self._predicates.values()): - print( - "Warning, all predicates set to False, skip_hidden may not have any effects." - ) - - def do_list(self, arg): - """Print lines of code from the current stack frame - """ - self.lastcmd = 'list' - last = None - if arg: - try: - x = eval(arg, {}, {}) - if type(x) == type(()): - first, last = x - first = int(first) - last = int(last) - if last < first: - # Assume it's a count - last = first + last - else: - first = max(1, int(x) - 5) - except: - print('*** Error in argument:', repr(arg), file=self.stdout) - return - elif self.lineno is None: - first = max(1, self.curframe.f_lineno - 5) - else: - first = self.lineno + 1 - if last is None: - last = first + 10 - self.print_list_lines(self.curframe.f_code.co_filename, first, last) - - # vds: >> - lineno = first - filename = self.curframe.f_code.co_filename - self.shell.hooks.synchronize_with_editor(filename, lineno, 0) - # vds: << - - do_l = do_list - - def getsourcelines(self, obj): - lines, lineno = inspect.findsource(obj) - if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj): - # must be a module frame: do not try to cut a block out of it - return lines, 1 - elif inspect.ismodule(obj): - return lines, 1 - return inspect.getblock(lines[lineno:]), lineno+1 - - def do_longlist(self, arg): - """Print lines of code from the current stack frame. - - Shows more lines than 'list' does. - """ - self.lastcmd = 'longlist' - try: - lines, lineno = self.getsourcelines(self.curframe) - except OSError as err: - self.error(err) - return - last = lineno + len(lines) - self.print_list_lines(self.curframe.f_code.co_filename, lineno, last) - do_ll = do_longlist - - def do_debug(self, arg): - """debug code - Enter a recursive debugger that steps through the code - argument (which is an arbitrary expression or statement to be - executed in the current environment). - """ - trace_function = sys.gettrace() - sys.settrace(None) - globals = self.curframe.f_globals - locals = self.curframe_locals - p = self.__class__(completekey=self.completekey, - stdin=self.stdin, stdout=self.stdout) - p.use_rawinput = self.use_rawinput - p.prompt = "(%s) " % self.prompt.strip() - self.message("ENTERING RECURSIVE DEBUGGER") - sys.call_tracing(p.run, (arg, globals, locals)) - self.message("LEAVING RECURSIVE DEBUGGER") - sys.settrace(trace_function) - self.lastcmd = p.lastcmd - - def do_pdef(self, arg): - """Print the call signature for any callable object. - - The debugger interface to %pdef""" - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("pdef")(arg, namespaces=namespaces) - - def do_pdoc(self, arg): - """Print the docstring for an object. - - The debugger interface to %pdoc.""" - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces) - - def do_pfile(self, arg): - """Print (or run through pager) the file where an object is defined. - - The debugger interface to %pfile. - """ - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("pfile")(arg, namespaces=namespaces) - - def do_pinfo(self, arg): - """Provide detailed information about an object. - - The debugger interface to %pinfo, i.e., obj?.""" - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces) - - def do_pinfo2(self, arg): - """Provide extra detailed information about an object. - - The debugger interface to %pinfo2, i.e., obj??.""" - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces) - - def do_psource(self, arg): - """Print (or run through pager) the source code for an object.""" - namespaces = [ - ("Locals", self.curframe_locals), - ("Globals", self.curframe.f_globals), - ] - self.shell.find_line_magic("psource")(arg, namespaces=namespaces) - - def do_where(self, arg): - """w(here) - Print a stack trace, with the most recent frame at the bottom. - An arrow indicates the "current frame", which determines the - context of most commands. 'bt' is an alias for this command. - - Take a number as argument as an (optional) number of context line to - print""" - if arg: - try: - context = int(arg) - except ValueError as err: - self.error(err) - return - self.print_stack_trace(context) - else: - self.print_stack_trace() - - do_w = do_where - - def break_anywhere(self, frame): - """ - - _stop_in_decorator_internals is overly restrictive, as we may still want - to trace function calls, so we need to also update break_anywhere so - that is we don't `stop_here`, because of debugger skip, we may still - stop at any point inside the function - - """ - - sup = super().break_anywhere(frame) - if sup: - return sup - if self._predicates["debuggerskip"]: - if DEBUGGERSKIP in frame.f_code.co_varnames: - return True - if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP): - return True - return False - - @skip_doctest - def _is_in_decorator_internal_and_should_skip(self, frame): - """ - Utility to tell us whether we are in a decorator internal and should stop. - - - - """ - - # if we are disabled don't skip - if not self._predicates["debuggerskip"]: - return False - - # if frame is tagged, skip by default. - if DEBUGGERSKIP in frame.f_code.co_varnames: - return True - - # if one of the parent frame value set to True skip as well. - - cframe = frame - while getattr(cframe, "f_back", None): - cframe = cframe.f_back - if self._get_frame_locals(cframe).get(DEBUGGERSKIP): - return True - - return False - - def stop_here(self, frame): - """Check if pdb should stop here""" - if not super().stop_here(frame): - return False - - if self._is_in_decorator_internal_and_should_skip(frame) is True: - return False - - hidden = False - if self.skip_hidden: - hidden = self._hidden_predicate(frame) - if hidden: - if self.report_skipped: - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") - return False - return True - - def do_up(self, arg): - """u(p) [count] - Move the current frame count (default one) levels up in the - stack trace (to an older frame). - - Will skip hidden frames. - """ - # modified version of upstream that skips - # frames with __tracebackhide__ - if self.curindex == 0: - self.error("Oldest frame") - return - try: - count = int(arg or 1) - except ValueError: - self.error("Invalid frame count (%s)" % arg) - return - skipped = 0 - if count < 0: - _newframe = 0 - else: - _newindex = self.curindex - counter = 0 - hidden_frames = self.hidden_frames(self.stack) - for i in range(self.curindex - 1, -1, -1): - frame = self.stack[i][0] - if hidden_frames[i] and self.skip_hidden: - skipped += 1 - continue - counter += 1 - if counter >= count: - break - else: - # if no break occured. - self.error("all frames above hidden") - return - - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - _newframe = i - self._select_frame(_newframe) - if skipped: - print( - f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" - ) - - def do_down(self, arg): - """d(own) [count] - Move the current frame count (default one) levels down in the - stack trace (to a newer frame). - - Will skip hidden frames. - """ - if self.curindex + 1 == len(self.stack): - self.error("Newest frame") - return - try: - count = int(arg or 1) - except ValueError: - self.error("Invalid frame count (%s)" % arg) - return - if count < 0: - _newframe = len(self.stack) - 1 - else: - _newindex = self.curindex - counter = 0 - skipped = 0 - hidden_frames = self.hidden_frames(self.stack) - for i in range(self.curindex + 1, len(self.stack)): - frame = self.stack[i][0] - if hidden_frames[i] and self.skip_hidden: - skipped += 1 - continue - counter += 1 - if counter >= count: - break - else: - self.error("all frames bellow hidden") - return - - Colors = self.color_scheme_table.active_colors - ColorsNormal = Colors.Normal - if skipped: - print( - f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" - ) - _newframe = i - - self._select_frame(_newframe) - - do_d = do_down - do_u = do_up - - def do_context(self, context): - """context number_of_lines - Set the number of lines of source code to show when displaying - stacktrace information. - """ - try: - new_context = int(context) - if new_context <= 0: - raise ValueError() - self.context = new_context - except ValueError: - self.error("The 'context' command requires a positive integer argument.") - - -class InterruptiblePdb(Pdb): - """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" - - def cmdloop(self, intro=None): - """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" - try: - return OldPdb.cmdloop(self, intro=intro) - except KeyboardInterrupt: - self.stop_here = lambda frame: False - self.do_quit("") - sys.settrace(None) - self.quitting = False - raise - - def _cmdloop(self): - while True: - try: - # keyboard interrupts allow for an easy way to cancel - # the current command, so allow them during interactive input - self.allow_kbdint = True - self.cmdloop() - self.allow_kbdint = False - break - except KeyboardInterrupt: - self.message('--KeyboardInterrupt--') - raise - - -def set_trace(frame=None): - """ - Start debugging from `frame`. - - If frame is not specified, debugging starts from caller's frame. - """ - Pdb().set_trace(frame or sys._getframe().f_back) +# -*- coding: utf-8 -*- +""" +Pdb debugger class. + + +This is an extension to PDB which adds a number of new features. +Note that there is also the `IPython.terminal.debugger` class which provides UI +improvements. + +We also strongly recommend to use this via the `ipdb` package, which provides +extra configuration options. + +Among other things, this subclass of PDB: + - supports many IPython magics like pdef/psource + - hide frames in tracebacks based on `__tracebackhide__` + - allows to skip frames based on `__debuggerskip__` + +The skipping and hiding frames are configurable via the `skip_predicates` +command. + +By default, frames from readonly files will be hidden, frames containing +``__tracebackhide__=True`` will be hidden. + +Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent +frames value of ``__debuggerskip__`` is ``True`` will be skipped. + + >>> def helpers_helper(): + ... pass + ... + ... def helper_1(): + ... print("don't step in me") + ... helpers_helpers() # will be stepped over unless breakpoint set. + ... + ... + ... def helper_2(): + ... print("in me neither") + ... + +One can define a decorator that wraps a function between the two helpers: + + >>> def pdb_skipped_decorator(function): + ... + ... + ... def wrapped_fn(*args, **kwargs): + ... __debuggerskip__ = True + ... helper_1() + ... __debuggerskip__ = False + ... result = function(*args, **kwargs) + ... __debuggerskip__ = True + ... helper_2() + ... # setting __debuggerskip__ to False again is not necessary + ... return result + ... + ... return wrapped_fn + +When decorating a function, ipdb will directly step into ``bar()`` by +default: + + >>> @foo_decorator + ... def bar(x, y): + ... return x * y + + +You can toggle the behavior with + + ipdb> skip_predicates debuggerskip false + +or configure it in your ``.pdbrc`` + + + +Licencse +-------- + +Modified from the standard pdb.Pdb class to avoid including readline, so that +the command line completion of other programs which include this isn't +damaged. + +In the future, this class will be expanded with improvements over the standard +pdb. + +The original code in this file is mainly lifted out of cmd.py in Python 2.2, +with minor changes. Licensing should therefore be under the standard Python +terms. For details on the PSF (Python Software Foundation) standard license, +see: + +https://docs.python.org/2/license.html + + +All the changes since then are under the same license as IPython. + +""" + +#***************************************************************************** +# +# This file is licensed under the PSF license. +# +# Copyright (C) 2001 Python Software Foundation, www.python.org +# Copyright (C) 2005-2006 Fernando Perez. +# +# +#***************************************************************************** + +import bdb +import functools +import inspect +import linecache +import sys +import warnings +import re +import os + +from IPython import get_ipython +from IPython.utils import PyColorize +from IPython.utils import coloransi, py3compat +from IPython.core.excolors import exception_colors +from IPython.testing.skipdoctest import skip_doctest + + +prompt = 'ipdb> ' + +#We have to check this directly from sys.argv, config struct not yet available +from pdb import Pdb as OldPdb + +# Allow the set_trace code to operate outside of an ipython instance, even if +# it does so with some limitations. The rest of this support is implemented in +# the Tracer constructor. + +DEBUGGERSKIP = "__debuggerskip__" + + +def make_arrow(pad): + """generate the leading arrow in front of traceback or debugger""" + if pad >= 2: + return '-'*(pad-2) + '> ' + elif pad == 1: + return '>' + return '' + + +def BdbQuit_excepthook(et, ev, tb, excepthook=None): + """Exception hook which handles `BdbQuit` exceptions. + + All other exceptions are processed using the `excepthook` + parameter. + """ + warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1", + DeprecationWarning, stacklevel=2) + if et==bdb.BdbQuit: + print('Exiting Debugger.') + elif excepthook is not None: + excepthook(et, ev, tb) + else: + # Backwards compatibility. Raise deprecation warning? + BdbQuit_excepthook.excepthook_ori(et,ev,tb) + + +def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None): + warnings.warn( + "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", + DeprecationWarning, stacklevel=2) + print('Exiting Debugger.') + + +class Tracer(object): + """ + DEPRECATED + + Class for local debugging, similar to pdb.set_trace. + + Instances of this class, when called, behave like pdb.set_trace, but + providing IPython's enhanced capabilities. + + This is implemented as a class which must be initialized in your own code + and not as a standalone function because we need to detect at runtime + whether IPython is already active or not. That detection is done in the + constructor, ensuring that this code plays nicely with a running IPython, + while functioning acceptably (though with limitations) if outside of it. + """ + + @skip_doctest + def __init__(self, colors=None): + """ + DEPRECATED + + Create a local debugger instance. + + Parameters + ---------- + + colors : str, optional + The name of the color scheme to use, it must be one of IPython's + valid color schemes. If not given, the function will default to + the current IPython scheme when running inside IPython, and to + 'NoColor' otherwise. + + Examples + -------- + :: + + from IPython.core.debugger import Tracer; debug_here = Tracer() + + Later in your code:: + + debug_here() # -> will open up the debugger at that point. + + Once the debugger activates, you can use all of its regular commands to + step through code, set breakpoints, etc. See the pdb documentation + from the Python standard library for usage details. + """ + warnings.warn("`Tracer` is deprecated since version 5.1, directly use " + "`IPython.core.debugger.Pdb.set_trace()`", + DeprecationWarning, stacklevel=2) + + ip = get_ipython() + if ip is None: + # Outside of ipython, we set our own exception hook manually + sys.excepthook = functools.partial(BdbQuit_excepthook, + excepthook=sys.excepthook) + def_colors = 'NoColor' + else: + # In ipython, we use its custom exception handler mechanism + def_colors = ip.colors + ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook) + + if colors is None: + colors = def_colors + + # The stdlib debugger internally uses a modified repr from the `repr` + # module, that limits the length of printed strings to a hardcoded + # limit of 30 characters. That much trimming is too aggressive, let's + # at least raise that limit to 80 chars, which should be enough for + # most interactive uses. + try: + from reprlib import aRepr + aRepr.maxstring = 80 + except: + # This is only a user-facing convenience, so any error we encounter + # here can be warned about but can be otherwise ignored. These + # printouts will tell us about problems if this API changes + import traceback + traceback.print_exc() + + self.debugger = Pdb(colors) + + def __call__(self): + """Starts an interactive debugger at the point where called. + + This is similar to the pdb.set_trace() function from the std lib, but + using IPython's enhanced debugger.""" + + self.debugger.set_trace(sys._getframe().f_back) + + +RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+') + + +def strip_indentation(multiline_string): + return RGX_EXTRA_INDENT.sub('', multiline_string) + + +def decorate_fn_with_doc(new_fn, old_fn, additional_text=""): + """Make new_fn have old_fn's doc string. This is particularly useful + for the ``do_...`` commands that hook into the help system. + Adapted from from a comp.lang.python posting + by Duncan Booth.""" + def wrapper(*args, **kw): + return new_fn(*args, **kw) + if old_fn.__doc__: + wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text + return wrapper + + +class Pdb(OldPdb): + """Modified Pdb class, does not load readline. + + for a standalone version that uses prompt_toolkit, see + `IPython.terminal.debugger.TerminalPdb` and + `IPython.terminal.debugger.set_trace()` + + + This debugger can hide and skip frames that are tagged according to some predicates. + See the `skip_predicates` commands. + + """ + + default_predicates = { + "tbhide": True, + "readonly": False, + "ipython_internal": True, + "debuggerskip": True, + } + + def __init__(self, color_scheme=None, completekey=None, + stdin=None, stdout=None, context=5, **kwargs): + """Create a new IPython debugger. + + Parameters + ---------- + color_scheme : default None + Deprecated, do not use. + completekey : default None + Passed to pdb.Pdb. + stdin : default None + Passed to pdb.Pdb. + stdout : default None + Passed to pdb.Pdb. + context : int + Number of lines of source code context to show when + displaying stacktrace information. + **kwargs + Passed to pdb.Pdb. + + Notes + ----- + The possibilities are python version dependent, see the python + docs for more info. + """ + + # Parent constructor: + try: + self.context = int(context) + if self.context <= 0: + raise ValueError("Context must be a positive integer") + except (TypeError, ValueError): + raise ValueError("Context must be a positive integer") + + # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. + OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) + + # IPython changes... + self.shell = get_ipython() + + if self.shell is None: + save_main = sys.modules['__main__'] + # No IPython instance running, we must create one + from IPython.terminal.interactiveshell import \ + TerminalInteractiveShell + self.shell = TerminalInteractiveShell.instance() + # needed by any code which calls __import__("__main__") after + # the debugger was entered. See also #9941. + sys.modules['__main__'] = save_main + + if color_scheme is not None: + warnings.warn( + "The `color_scheme` argument is deprecated since version 5.1", + DeprecationWarning, stacklevel=2) + else: + color_scheme = self.shell.colors + + self.aliases = {} + + # Create color table: we copy the default one from the traceback + # module and add a few attributes needed for debugging + self.color_scheme_table = exception_colors() + + # shorthands + C = coloransi.TermColors + cst = self.color_scheme_table + + cst['NoColor'].colors.prompt = C.NoColor + cst['NoColor'].colors.breakpoint_enabled = C.NoColor + cst['NoColor'].colors.breakpoint_disabled = C.NoColor + + cst['Linux'].colors.prompt = C.Green + cst['Linux'].colors.breakpoint_enabled = C.LightRed + cst['Linux'].colors.breakpoint_disabled = C.Red + + cst['LightBG'].colors.prompt = C.Blue + cst['LightBG'].colors.breakpoint_enabled = C.LightRed + cst['LightBG'].colors.breakpoint_disabled = C.Red + + cst['Neutral'].colors.prompt = C.Blue + cst['Neutral'].colors.breakpoint_enabled = C.LightRed + cst['Neutral'].colors.breakpoint_disabled = C.Red + + + # Add a python parser so we can syntax highlight source while + # debugging. + self.parser = PyColorize.Parser(style=color_scheme) + self.set_colors(color_scheme) + + # Set the prompt - the default prompt is '(Pdb)' + self.prompt = prompt + self.skip_hidden = True + self.report_skipped = True + + # list of predicates we use to skip frames + self._predicates = self.default_predicates + + # + def set_colors(self, scheme): + """Shorthand access to the color table scheme selector method.""" + self.color_scheme_table.set_active_scheme(scheme) + self.parser.style = scheme + + def set_trace(self, frame=None): + if frame is None: + frame = sys._getframe().f_back + self.initial_frame = frame + return super().set_trace(frame) + + def _hidden_predicate(self, frame): + """ + Given a frame return whether it it should be hidden or not by IPython. + """ + + if self._predicates["readonly"]: + fname = frame.f_code.co_filename + # we need to check for file existence and interactively define + # function would otherwise appear as RO. + if os.path.isfile(fname) and not os.access(fname, os.W_OK): + return True + + if self._predicates["tbhide"]: + if frame in (self.curframe, getattr(self, "initial_frame", None)): + return False + frame_locals = self._get_frame_locals(frame) + if "__tracebackhide__" not in frame_locals: + return False + return frame_locals["__tracebackhide__"] + return False + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + # The f_locals dictionary is updated from the actual frame + # locals whenever the .f_locals accessor is called, so we + # avoid calling it here to preserve self.curframe_locals. + # Futhermore, there is no good reason to hide the current frame. + ip_hide = [self._hidden_predicate(s[0]) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start and self._predicates["ipython_internal"]: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + + def interaction(self, frame, traceback): + try: + OldPdb.interaction(self, frame, traceback) + except KeyboardInterrupt: + self.stdout.write("\n" + self.shell.get_exception_only()) + + def new_do_frame(self, arg): + OldPdb.do_frame(self, arg) + + def new_do_quit(self, arg): + + if hasattr(self, 'old_all_completions'): + self.shell.Completer.all_completions=self.old_all_completions + + return OldPdb.do_quit(self, arg) + + do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit) + + def new_do_restart(self, arg): + """Restart command. In the context of ipython this is exactly the same + thing as 'quit'.""" + self.msg("Restart doesn't make sense here. Using 'quit' instead.") + return self.do_quit(arg) + + def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if context is None: + context = self.context + try: + context=int(context) + if context <= 0: + raise ValueError("Context must be a positive integer") + except (TypeError, ValueError): + raise ValueError("Context must be a positive integer") + try: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 + self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + except KeyboardInterrupt: + pass + + def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', + context=None): + if context is None: + context = self.context + try: + context=int(context) + if context <= 0: + raise ValueError("Context must be a positive integer") + except (TypeError, ValueError): + raise ValueError("Context must be a positive integer") + print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) + + # vds: >> + frame, lineno = frame_lineno + filename = frame.f_code.co_filename + self.shell.hooks.synchronize_with_editor(filename, lineno, 0) + # vds: << + + def _get_frame_locals(self, frame): + """ " + Acessing f_local of current frame reset the namespace, so we want to avoid + that or the following can happend + + ipdb> foo + "old" + ipdb> foo = "new" + ipdb> foo + "new" + ipdb> where + ipdb> foo + "old" + + So if frame is self.current_frame we instead return self.curframe_locals + + """ + if frame is self.curframe: + return self.curframe_locals + else: + return frame.f_locals + + def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): + if context is None: + context = self.context + try: + context=int(context) + if context <= 0: + print("Context must be a positive integer", file=self.stdout) + except (TypeError, ValueError): + print("Context must be a positive integer", file=self.stdout) + try: + import reprlib # Py 3 + except ImportError: + import repr as reprlib # Py 2 + + ret = [] + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal) + tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal) + tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) + tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, + ColorsNormal) + + frame, lineno = frame_lineno + + return_value = '' + loc_frame = self._get_frame_locals(frame) + if "__return__" in loc_frame: + rv = loc_frame["__return__"] + # return_value += '->' + return_value += reprlib.repr(rv) + "\n" + ret.append(return_value) + + #s = filename + '(' + `lineno` + ')' + filename = self.canonic(frame.f_code.co_filename) + link = tpl_link % py3compat.cast_unicode(filename) + + if frame.f_code.co_name: + func = frame.f_code.co_name + else: + func = "" + + call = "" + if func != "?": + if "__args__" in loc_frame: + args = reprlib.repr(loc_frame["__args__"]) + else: + args = '()' + call = tpl_call % (func, args) + + # The level info should be generated in the same format pdb uses, to + # avoid breaking the pdbtrack functionality of python-mode in *emacs. + if frame is self.curframe: + ret.append('> ') + else: + ret.append(' ') + ret.append(u'%s(%s)%s\n' % (link,lineno,call)) + + start = lineno - 1 - context//2 + lines = linecache.getlines(filename) + start = min(start, len(lines) - context) + start = max(start, 0) + lines = lines[start : start + context] + + for i,line in enumerate(lines): + show_arrow = (start + 1 + i == lineno) + linetpl = (frame is self.curframe or show_arrow) \ + and tpl_line_em \ + or tpl_line + ret.append(self.__format_line(linetpl, filename, + start + 1 + i, line, + arrow = show_arrow) ) + return ''.join(ret) + + def __format_line(self, tpl_line, filename, lineno, line, arrow = False): + bp_mark = "" + bp_mark_color = "" + + new_line, err = self.parser.format2(line, 'str') + if not err: + line = new_line + + bp = None + if lineno in self.get_file_breaks(filename): + bps = self.get_breaks(filename, lineno) + bp = bps[-1] + + if bp: + Colors = self.color_scheme_table.active_colors + bp_mark = str(bp.number) + bp_mark_color = Colors.breakpoint_enabled + if not bp.enabled: + bp_mark_color = Colors.breakpoint_disabled + + numbers_width = 7 + if arrow: + # This is the line with the error + pad = numbers_width - len(str(lineno)) - len(bp_mark) + num = '%s%s' % (make_arrow(pad), str(lineno)) + else: + num = '%*s' % (numbers_width - len(bp_mark), str(lineno)) + + return tpl_line % (bp_mark_color + bp_mark, num, line) + + + def print_list_lines(self, filename, first, last): + """The printing (as opposed to the parsing part of a 'list' + command.""" + try: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) + tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal) + src = [] + if filename == "" and hasattr(self, "_exec_filename"): + filename = self._exec_filename + + for lineno in range(first, last+1): + line = linecache.getline(filename, lineno) + if not line: + break + + if lineno == self.curframe.f_lineno: + line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True) + else: + line = self.__format_line(tpl_line, filename, lineno, line, arrow = False) + + src.append(line) + self.lineno = lineno + + print(''.join(src), file=self.stdout) + + except KeyboardInterrupt: + pass + + def do_skip_predicates(self, args): + """ + Turn on/off individual predicates as to whether a frame should be hidden/skip. + + The global option to skip (or not) hidden frames is set with skip_hidden + + To change the value of a predicate + + skip_predicates key [true|false] + + Call without arguments to see the current values. + + To permanently change the value of an option add the corresponding + command to your ``~/.pdbrc`` file. If you are programmatically using the + Pdb instance you can also change the ``default_predicates`` class + attribute. + """ + if not args.strip(): + print("current predicates:") + for (p, v) in self._predicates.items(): + print(" ", p, ":", v) + return + type_value = args.strip().split(" ") + if len(type_value) != 2: + print( + f"Usage: skip_predicates , with one of {set(self._predicates.keys())}" + ) + return + + type_, value = type_value + if type_ not in self._predicates: + print(f"{type_!r} not in {set(self._predicates.keys())}") + return + if value.lower() not in ("true", "yes", "1", "no", "false", "0"): + print( + f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')" + ) + return + + self._predicates[type_] = value.lower() in ("true", "yes", "1") + if not any(self._predicates.values()): + print( + "Warning, all predicates set to False, skip_hidden may not have any effects." + ) + + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if not arg.strip(): + print( + f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change." + ) + elif arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + if not any(self._predicates.values()): + print( + "Warning, all predicates set to False, skip_hidden may not have any effects." + ) + + def do_list(self, arg): + """Print lines of code from the current stack frame + """ + self.lastcmd = 'list' + last = None + if arg: + try: + x = eval(arg, {}, {}) + if type(x) == type(()): + first, last = x + first = int(first) + last = int(last) + if last < first: + # Assume it's a count + last = first + last + else: + first = max(1, int(x) - 5) + except: + print('*** Error in argument:', repr(arg), file=self.stdout) + return + elif self.lineno is None: + first = max(1, self.curframe.f_lineno - 5) + else: + first = self.lineno + 1 + if last is None: + last = first + 10 + self.print_list_lines(self.curframe.f_code.co_filename, first, last) + + # vds: >> + lineno = first + filename = self.curframe.f_code.co_filename + self.shell.hooks.synchronize_with_editor(filename, lineno, 0) + # vds: << + + do_l = do_list + + def getsourcelines(self, obj): + lines, lineno = inspect.findsource(obj) + if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj): + # must be a module frame: do not try to cut a block out of it + return lines, 1 + elif inspect.ismodule(obj): + return lines, 1 + return inspect.getblock(lines[lineno:]), lineno+1 + + def do_longlist(self, arg): + """Print lines of code from the current stack frame. + + Shows more lines than 'list' does. + """ + self.lastcmd = 'longlist' + try: + lines, lineno = self.getsourcelines(self.curframe) + except OSError as err: + self.error(err) + return + last = lineno + len(lines) + self.print_list_lines(self.curframe.f_code.co_filename, lineno, last) + do_ll = do_longlist + + def do_debug(self, arg): + """debug code + Enter a recursive debugger that steps through the code + argument (which is an arbitrary expression or statement to be + executed in the current environment). + """ + trace_function = sys.gettrace() + sys.settrace(None) + globals = self.curframe.f_globals + locals = self.curframe_locals + p = self.__class__(completekey=self.completekey, + stdin=self.stdin, stdout=self.stdout) + p.use_rawinput = self.use_rawinput + p.prompt = "(%s) " % self.prompt.strip() + self.message("ENTERING RECURSIVE DEBUGGER") + sys.call_tracing(p.run, (arg, globals, locals)) + self.message("LEAVING RECURSIVE DEBUGGER") + sys.settrace(trace_function) + self.lastcmd = p.lastcmd + + def do_pdef(self, arg): + """Print the call signature for any callable object. + + The debugger interface to %pdef""" + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pdef")(arg, namespaces=namespaces) + + def do_pdoc(self, arg): + """Print the docstring for an object. + + The debugger interface to %pdoc.""" + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces) + + def do_pfile(self, arg): + """Print (or run through pager) the file where an object is defined. + + The debugger interface to %pfile. + """ + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pfile")(arg, namespaces=namespaces) + + def do_pinfo(self, arg): + """Provide detailed information about an object. + + The debugger interface to %pinfo, i.e., obj?.""" + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces) + + def do_pinfo2(self, arg): + """Provide extra detailed information about an object. + + The debugger interface to %pinfo2, i.e., obj??.""" + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces) + + def do_psource(self, arg): + """Print (or run through pager) the source code for an object.""" + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("psource")(arg, namespaces=namespaces) + + def do_where(self, arg): + """w(here) + Print a stack trace, with the most recent frame at the bottom. + An arrow indicates the "current frame", which determines the + context of most commands. 'bt' is an alias for this command. + + Take a number as argument as an (optional) number of context line to + print""" + if arg: + try: + context = int(arg) + except ValueError as err: + self.error(err) + return + self.print_stack_trace(context) + else: + self.print_stack_trace() + + do_w = do_where + + def break_anywhere(self, frame): + """ + + _stop_in_decorator_internals is overly restrictive, as we may still want + to trace function calls, so we need to also update break_anywhere so + that is we don't `stop_here`, because of debugger skip, we may still + stop at any point inside the function + + """ + + sup = super().break_anywhere(frame) + if sup: + return sup + if self._predicates["debuggerskip"]: + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP): + return True + return False + + @skip_doctest + def _is_in_decorator_internal_and_should_skip(self, frame): + """ + Utility to tell us whether we are in a decorator internal and should stop. + + + + """ + + # if we are disabled don't skip + if not self._predicates["debuggerskip"]: + return False + + # if frame is tagged, skip by default. + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + + # if one of the parent frame value set to True skip as well. + + cframe = frame + while getattr(cframe, "f_back", None): + cframe = cframe.f_back + if self._get_frame_locals(cframe).get(DEBUGGERSKIP): + return True + + return False + + def stop_here(self, frame): + """Check if pdb should stop here""" + if not super().stop_here(frame): + return False + + if self._is_in_decorator_internal_and_should_skip(frame) is True: + return False + + hidden = False + if self.skip_hidden: + hidden = self._hidden_predicate(frame) + if hidden: + if self.report_skipped: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + return False + return True + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + # modified version of upstream that skips + # frames with __tracebackhide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + + def do_context(self, context): + """context number_of_lines + Set the number of lines of source code to show when displaying + stacktrace information. + """ + try: + new_context = int(context) + if new_context <= 0: + raise ValueError() + self.context = new_context + except ValueError: + self.error("The 'context' command requires a positive integer argument.") + + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self, intro=None): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self, intro=intro) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + + +def set_trace(frame=None): + """ + Start debugging from `frame`. + + If frame is not specified, debugging starts from caller's frame. + """ + Pdb().set_trace(frame or sys._getframe().f_back) diff --git a/contrib/python/ipython/py3/IPython/core/display.py b/contrib/python/ipython/py3/IPython/core/display.py index f5a5a90e4a9..f45e7599c9e 100644 --- a/contrib/python/ipython/py3/IPython/core/display.py +++ b/contrib/python/ipython/py3/IPython/core/display.py @@ -1,1556 +1,1556 @@ -# -*- coding: utf-8 -*- -"""Top-level display functions for displaying object in different formats.""" - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - - -from binascii import b2a_hex, b2a_base64, hexlify -import json -import mimetypes -import os -import struct -import sys -import warnings -from copy import deepcopy -from os.path import splitext -from pathlib import Path, PurePath - -from IPython.utils.py3compat import cast_unicode -from IPython.testing.skipdoctest import skip_doctest - -__all__ = ['display', 'display_pretty', 'display_html', 'display_markdown', -'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', -'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', -'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', -'GeoJSON', 'Javascript', 'Image', 'clear_output', 'set_matplotlib_formats', -'set_matplotlib_close', 'publish_display_data', 'update_display', 'DisplayHandle', -'Video'] - -#----------------------------------------------------------------------------- -# utility functions -#----------------------------------------------------------------------------- - -def _safe_exists(path): - """Check path, but don't let exceptions raise""" - try: - return os.path.exists(path) - except Exception: - return False - -def _merge(d1, d2): - """Like update, but merges sub-dicts instead of clobbering at the top level. - - Updates d1 in-place - """ - - if not isinstance(d2, dict) or not isinstance(d1, dict): - return d2 - for key, value in d2.items(): - d1[key] = _merge(d1.get(key), value) - return d1 - -def _display_mimetype(mimetype, objs, raw=False, metadata=None): - """internal implementation of all display_foo methods - - Parameters - ---------- - mimetype : str - The mimetype to be published (e.g. 'image/png') - *objs : object - The Python objects to display, or if raw=True raw text data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - if metadata: - metadata = {mimetype: metadata} - if raw: - # turn list of pngdata into list of { 'image/png': pngdata } - objs = [ {mimetype: obj} for obj in objs ] - display(*objs, raw=raw, metadata=metadata, include=[mimetype]) - -#----------------------------------------------------------------------------- -# Main functions -#----------------------------------------------------------------------------- - -# use * to indicate transient is keyword-only -def publish_display_data(data, metadata=None, source=None, *, transient=None, **kwargs): - """Publish data and metadata to all frontends. - - See the ``display_data`` message in the messaging documentation for - more details about this message type. - - Keys of data and metadata can be any mime-type. - - Parameters - ---------- - data : dict - A dictionary having keys that are valid MIME types (like - 'text/plain' or 'image/svg+xml') and values that are the data for - that MIME type. The data itself must be a JSON'able data - structure. Minimally all data should have the 'text/plain' data, - which can be displayed by all frontends. If more than the plain - text is given, it is up to the frontend to decide which - representation to use. - metadata : dict - A dictionary for metadata related to the data. This can contain - arbitrary key, value pairs that frontends can use to interpret - the data. mime-type keys matching those in data can be used - to specify metadata about particular representations. - source : str, deprecated - Unused. - transient : dict, keyword-only - A dictionary of transient data, such as display_id. - """ - from IPython.core.interactiveshell import InteractiveShell - - display_pub = InteractiveShell.instance().display_pub - - # only pass transient if supplied, - # to avoid errors with older ipykernel. - # TODO: We could check for ipykernel version and provide a detailed upgrade message. - if transient: - kwargs['transient'] = transient - - display_pub.publish( - data=data, - metadata=metadata, - **kwargs - ) - - -def _new_id(): - """Generate a new random text id with urandom""" - return b2a_hex(os.urandom(16)).decode('ascii') - - -def display(*objs, include=None, exclude=None, metadata=None, transient=None, display_id=None, **kwargs): - """Display a Python object in all frontends. - - By default all representations will be computed and sent to the frontends. - Frontends can decide which representation is used and how. - - In terminal IPython this will be similar to using :func:`print`, for use in richer - frontends see Jupyter notebook examples with rich display logic. - - Parameters - ---------- - *objs : object - The Python objects to display. - raw : bool, optional - Are the objects to be displayed already mimetype-keyed dicts of raw display data, - or Python objects that need to be formatted before display? [default: False] - include : list, tuple or set, optional - A list of format type strings (MIME types) to include in the - format data dict. If this is set *only* the format types included - in this list will be computed. - exclude : list, tuple or set, optional - A list of format type strings (MIME types) to exclude in the format - data dict. If this is set all format types will be computed, - except for those included in this argument. - metadata : dict, optional - A dictionary of metadata to associate with the output. - mime-type keys in this dictionary will be associated with the individual - representation formats, if they exist. - transient : dict, optional - A dictionary of transient data to associate with the output. - Data in this dict should not be persisted to files (e.g. notebooks). - display_id : str, bool optional - Set an id for the display. - This id can be used for updating this display area later via update_display. - If given as `True`, generate a new `display_id` - clear : bool, optional - Should the output area be cleared before displaying anything? If True, - this will wait for additional output before clearing. [default: False] - kwargs: additional keyword-args, optional - Additional keyword-arguments are passed through to the display publisher. - - Returns - ------- - - handle: DisplayHandle - Returns a handle on updatable displays for use with :func:`update_display`, - if `display_id` is given. Returns :any:`None` if no `display_id` is given - (default). - - Examples - -------- - - >>> class Json(object): - ... def __init__(self, json): - ... self.json = json - ... def _repr_pretty_(self, pp, cycle): - ... import json - ... pp.text(json.dumps(self.json, indent=2)) - ... def __repr__(self): - ... return str(self.json) - ... - - >>> d = Json({1:2, 3: {4:5}}) - - >>> print(d) - {1: 2, 3: {4: 5}} - - >>> display(d) - { - "1": 2, - "3": { - "4": 5 - } - } - - >>> def int_formatter(integer, pp, cycle): - ... pp.text('I'*integer) - - >>> plain = get_ipython().display_formatter.formatters['text/plain'] - >>> plain.for_type(int, int_formatter) - - >>> display(7-5) - II - - >>> del plain.type_printers[int] - >>> display(7-5) - 2 - - See Also - -------- - - :func:`update_display` - - Notes - ----- - - In Python, objects can declare their textual representation using the - `__repr__` method. IPython expands on this idea and allows objects to declare - other, rich representations including: - - - HTML - - JSON - - PNG - - JPEG - - SVG - - LaTeX - - A single object can declare some or all of these representations; all are - handled by IPython's display system. - - The main idea of the first approach is that you have to implement special - display methods when you define your class, one for each representation you - want to use. Here is a list of the names of the special methods and the - values they must return: - - - `_repr_html_`: return raw HTML as a string, or a tuple (see below). - - `_repr_json_`: return a JSONable dict, or a tuple (see below). - - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). - - `_repr_png_`: return raw PNG data, or a tuple (see below). - - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). - - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", - or a tuple (see below). - - `_repr_mimebundle_`: return a full mimebundle containing the mapping - from all mimetypes to data. - Use this for any mime-type not listed above. - - The above functions may also return the object's metadata alonside the - data. If the metadata is available, the functions will return a tuple - containing the data and metadata, in that order. If there is no metadata - available, then the functions will return the data only. - - When you are directly writing your own classes, you can adapt them for - display in IPython by following the above approach. But in practice, you - often need to work with existing classes that you can't easily modify. - - You can refer to the documentation on integrating with the display system in - order to register custom formatters for already existing types - (:ref:`integrating_rich_display`). - - .. versionadded:: 5.4 display available without import - .. versionadded:: 6.1 display available without import - - Since IPython 5.4 and 6.1 :func:`display` is automatically made available to - the user without import. If you are using display in a document that might - be used in a pure python context or with older version of IPython, use the - following import at the top of your file:: - - from IPython.display import display - - """ - from IPython.core.interactiveshell import InteractiveShell - - if not InteractiveShell.initialized(): - # Directly print objects. - print(*objs) - return - - raw = kwargs.pop("raw", False) - clear = kwargs.pop("clear", False) - if transient is None: - transient = {} - if metadata is None: - metadata={} - if display_id: - if display_id is True: - display_id = _new_id() - transient['display_id'] = display_id - if kwargs.get('update') and 'display_id' not in transient: - raise TypeError('display_id required for update_display') - if transient: - kwargs['transient'] = transient - - if not objs and display_id: - # if given no objects, but still a request for a display_id, - # we assume the user wants to insert an empty output that - # can be updated later - objs = [{}] - raw = True - - if not raw: - format = InteractiveShell.instance().display_formatter.format - - if clear: - clear_output(wait=True) - - for obj in objs: - if raw: - publish_display_data(data=obj, metadata=metadata, **kwargs) - else: - format_dict, md_dict = format(obj, include=include, exclude=exclude) - if not format_dict: - # nothing to display (e.g. _ipython_display_ took over) - continue - if metadata: - # kwarg-specified metadata gets precedence - _merge(md_dict, metadata) - publish_display_data(data=format_dict, metadata=md_dict, **kwargs) - if display_id: - return DisplayHandle(display_id) - - -# use * for keyword-only display_id arg -def update_display(obj, *, display_id, **kwargs): - """Update an existing display by id - - Parameters - ---------- - - obj: - The object with which to update the display - display_id: keyword-only - The id of the display to update - - See Also - -------- - - :func:`display` - """ - kwargs['update'] = True - display(obj, display_id=display_id, **kwargs) - - -class DisplayHandle(object): - """A handle on an updatable display - - Call `.update(obj)` to display a new object. - - Call `.display(obj`) to add a new instance of this display, - and update existing instances. - - See Also - -------- - - :func:`display`, :func:`update_display` - - """ - - def __init__(self, display_id=None): - if display_id is None: - display_id = _new_id() - self.display_id = display_id - - def __repr__(self): - return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) - - def display(self, obj, **kwargs): - """Make a new display with my id, updating existing instances. - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to display - """ - display(obj, display_id=self.display_id, **kwargs) - - def update(self, obj, **kwargs): - """Update existing displays with my id - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to update_display - """ - update_display(obj, display_id=self.display_id, **kwargs) - - -def display_pretty(*objs, **kwargs): - """Display the pretty (default) representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw text data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/plain', objs, **kwargs) - - -def display_html(*objs, **kwargs): - """Display the HTML representation of an object. - - Note: If raw=False and the object does not have a HTML - representation, no HTML will be shown. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw HTML data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/html', objs, **kwargs) - - -def display_markdown(*objs, **kwargs): - """Displays the Markdown representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw markdown data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - - _display_mimetype('text/markdown', objs, **kwargs) - - -def display_svg(*objs, **kwargs): - """Display the SVG representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw svg data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/svg+xml', objs, **kwargs) - - -def display_png(*objs, **kwargs): - """Display the PNG representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw png data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/png', objs, **kwargs) - - -def display_jpeg(*objs, **kwargs): - """Display the JPEG representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw JPEG data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/jpeg', objs, **kwargs) - - -def display_latex(*objs, **kwargs): - """Display the LaTeX representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw latex data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/latex', objs, **kwargs) - - -def display_json(*objs, **kwargs): - """Display the JSON representation of an object. - - Note that not many frontends support displaying JSON. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw json data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/json', objs, **kwargs) - - -def display_javascript(*objs, **kwargs): - """Display the Javascript representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw javascript data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/javascript', objs, **kwargs) - - -def display_pdf(*objs, **kwargs): - """Display the PDF representation of an object. - - Parameters - ---------- - *objs : object - The Python objects to display, or if raw=True raw javascript data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/pdf', objs, **kwargs) - - -#----------------------------------------------------------------------------- -# Smart classes -#----------------------------------------------------------------------------- - - -class DisplayObject(object): - """An object that wraps data to be displayed.""" - - _read_flags = 'r' - _show_mem_addr = False - metadata = None - - def __init__(self, data=None, url=None, filename=None, metadata=None): - """Create a display object given raw data. - - When this object is returned by an expression or passed to the - display function, it will result in the data being displayed - in the frontend. The MIME type of the data should match the - subclasses used, so the Png subclass should be used for 'image/png' - data. If the data is a URL, the data will first be downloaded - and then displayed. If - - Parameters - ---------- - data : unicode, str or bytes - The raw data or a URL or file to load the data from - url : unicode - A URL to download the data from. - filename : unicode - Path to a local file to load the data from. - metadata : dict - Dict of metadata associated to be the object when displayed - """ - if isinstance(data, (Path, PurePath)): - data = str(data) - - if data is not None and isinstance(data, str): - if data.startswith('http') and url is None: - url = data - filename = None - data = None - elif _safe_exists(data) and filename is None: - url = None - filename = data - data = None - - self.url = url - self.filename = filename - # because of @data.setter methods in - # subclasses ensure url and filename are set - # before assigning to self.data - self.data = data - - if metadata is not None: - self.metadata = metadata - elif self.metadata is None: - self.metadata = {} - - self.reload() - self._check_data() - - def __repr__(self): - if not self._show_mem_addr: - cls = self.__class__ - r = "<%s.%s object>" % (cls.__module__, cls.__name__) - else: - r = super(DisplayObject, self).__repr__() - return r - - def _check_data(self): - """Override in subclasses if there's something to check.""" - pass - - def _data_and_metadata(self): - """shortcut for returning metadata with shape information, if defined""" - if self.metadata: - return self.data, deepcopy(self.metadata) - else: - return self.data - - def reload(self): - """Reload the raw data from file or URL.""" - if self.filename is not None: - with open(self.filename, self._read_flags) as f: - self.data = f.read() - elif self.url is not None: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - data = response.read() - # extract encoding from header, if there is one: - encoding = None - if 'content-type' in response.headers: - for sub in response.headers['content-type'].split(';'): - sub = sub.strip() - if sub.startswith('charset'): - encoding = sub.split('=')[-1].strip() - break - if 'content-encoding' in response.headers: - # TODO: do deflate? - if 'gzip' in response.headers['content-encoding']: - import gzip - from io import BytesIO - with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: - encoding = None - data = fp.read() - - # decode data, if an encoding was specified - # We only touch self.data once since - # subclasses such as SVG have @data.setter methods - # that transform self.data into ... well svg. - if encoding: - self.data = data.decode(encoding, 'replace') - else: - self.data = data - - -class TextDisplayObject(DisplayObject): - """Validate that display data is text""" - def _check_data(self): - if self.data is not None and not isinstance(self.data, str): - raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data)) - -class Pretty(TextDisplayObject): - - def _repr_pretty_(self, pp, cycle): - return pp.text(self.data) - - -class HTML(TextDisplayObject): - - def __init__(self, data=None, url=None, filename=None, metadata=None): - def warn(): - if not data: - return False - - # - # Avoid calling lower() on the entire data, because it could be a - # long string and we're only interested in its beginning and end. - # - prefix = data[:10].lower() - suffix = data[-10:].lower() - return prefix.startswith("") + + if warn(): + warnings.warn("Consider using IPython.display.IFrame instead") + super(HTML, self).__init__(data=data, url=url, filename=filename, metadata=metadata) + + def _repr_html_(self): + return self._data_and_metadata() + + def __html__(self): + """ + This method exists to inform other HTML-using modules (e.g. Markupsafe, + htmltag, etc) that this object is HTML and does not need things like + special characters (<>&) escaped. + """ + return self._repr_html_() + + +class Markdown(TextDisplayObject): + + def _repr_markdown_(self): + return self._data_and_metadata() + + +class Math(TextDisplayObject): + + def _repr_latex_(self): + s = r"$\displaystyle %s$" % self.data.strip('$') + if self.metadata: + return s, deepcopy(self.metadata) + else: + return s + + +class Latex(TextDisplayObject): + + def _repr_latex_(self): + return self._data_and_metadata() + + +class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ + + _read_flags = 'rb' + # wrap data in a property, which extracts the tag, discarding + # document headers + _data = None + + @property + def data(self): + return self._data + + @data.setter + def data(self, svg): + if svg is None: + self._data = None + return + # parse into dom object + from xml.dom import minidom + x = minidom.parseString(svg) + # get svg tag (should be 1) + found_svg = x.getElementsByTagName('svg') + if found_svg: + svg = found_svg[0].toxml() + else: + # fallback on the input, trust the user + # but this is probably an error. + pass + svg = cast_unicode(svg) + self._data = svg + + def _repr_svg_(self): + return self._data_and_metadata() + +class ProgressBar(DisplayObject): + """Progressbar supports displaying a progressbar like element + """ + def __init__(self, total): + """Creates a new progressbar + + Parameters + ---------- + total : int + maximum size of the progressbar + """ + self.total = total + self._progress = 0 + self.html_width = '60ex' + self.text_width = 60 + self._display_id = hexlify(os.urandom(8)).decode('ascii') + + def __repr__(self): + fraction = self.progress / self.total + filled = '=' * int(fraction * self.text_width) + rest = ' ' * (self.text_width - len(filled)) + return '[{}{}] {}/{}'.format( + filled, rest, + self.progress, self.total, + ) + + def _repr_html_(self): + return "".format( + self.html_width, self.total, self.progress) + + def display(self): + display(self, display_id=self._display_id) + + def update(self): + display(self, display_id=self._display_id, update=True) + + @property + def progress(self): + return self._progress + + @progress.setter + def progress(self, value): + self._progress = value + self.update() + + def __iter__(self): + self.display() + self._progress = -1 # First iteration is 0 + return self + + def __next__(self): + """Returns current value and increments display by one.""" + self.progress += 1 + if self.progress < self.total: + return self.progress + else: + raise StopIteration() + +class JSON(DisplayObject): + """JSON expects a JSON-able dict or list + + not an already-serialized JSON string. + + Scalar types (None, number, string) are not allowed, only dict or list containers. + """ + # wrap data in a property, which warns about passing already-serialized JSON + _data = None + def __init__(self, data=None, url=None, filename=None, expanded=False, metadata=None, root='root', **kwargs): + """Create a JSON display object given raw data. + + Parameters + ---------- + data : dict or list + JSON data to display. Not an already-serialized JSON string. + Scalar types (None, number, string) are not allowed, only dict + or list containers. + url : unicode + A URL to download the data from. + filename : unicode + Path to a local file to load the data from. + expanded : boolean + Metadata to control whether a JSON display component is expanded. + metadata: dict + Specify extra metadata to attach to the json display object. + root : str + The name of the root element of the JSON tree + """ + self.metadata = { + 'expanded': expanded, + 'root': root, + } + if metadata: + self.metadata.update(metadata) + if kwargs: + self.metadata.update(kwargs) + super(JSON, self).__init__(data=data, url=url, filename=filename) + + def _check_data(self): + if self.data is not None and not isinstance(self.data, (dict, list)): + raise TypeError("%s expects JSONable dict or list, not %r" % (self.__class__.__name__, self.data)) + + @property + def data(self): + return self._data + + @data.setter + def data(self, data): + if isinstance(data, (Path, PurePath)): + data = str(data) + + if isinstance(data, str): + if self.filename is None and self.url is None: + warnings.warn("JSON expects JSONable dict or list, not JSON strings") + data = json.loads(data) + self._data = data + + def _data_and_metadata(self): + return self.data, self.metadata + + def _repr_json_(self): + return self._data_and_metadata() + +_css_t = """var link = document.createElement("link"); + link.ref = "stylesheet"; + link.type = "text/css"; + link.href = "%s"; + document.head.appendChild(link); +""" + +_lib_t1 = """new Promise(function(resolve, reject) { + var script = document.createElement("script"); + script.onload = resolve; + script.onerror = reject; + script.src = "%s"; + document.head.appendChild(script); +}).then(() => { +""" + +_lib_t2 = """ +});""" + +class GeoJSON(JSON): + """GeoJSON expects JSON-able dict + + not an already-serialized JSON string. + + Scalar types (None, number, string) are not allowed, only dict containers. + """ + + def __init__(self, *args, **kwargs): + """Create a GeoJSON display object given raw data. + + Parameters + ---------- + data : dict or list + VegaLite data. Not an already-serialized JSON string. + Scalar types (None, number, string) are not allowed, only dict + or list containers. + url_template : string + Leaflet TileLayer URL template: http://leafletjs.com/reference.html#url-template + layer_options : dict + Leaflet TileLayer options: http://leafletjs.com/reference.html#tilelayer-options + url : unicode + A URL to download the data from. + filename : unicode + Path to a local file to load the data from. + metadata: dict + Specify extra metadata to attach to the json display object. + + Examples + -------- + + The following will display an interactive map of Mars with a point of + interest on frontend that do support GeoJSON display. + + >>> from IPython.display import GeoJSON + + >>> GeoJSON(data={ + ... "type": "Feature", + ... "geometry": { + ... "type": "Point", + ... "coordinates": [-81.327, 296.038] + ... } + ... }, + ... url_template="http://s3-eu-west-1.amazonaws.com/whereonmars.cartodb.net/{basemap_id}/{z}/{x}/{y}.png", + ... layer_options={ + ... "basemap_id": "celestia_mars-shaded-16k_global", + ... "attribution" : "Celestia/praesepe", + ... "minZoom" : 0, + ... "maxZoom" : 18, + ... }) + + + In the terminal IPython, you will only see the text representation of + the GeoJSON object. + + """ + + super(GeoJSON, self).__init__(*args, **kwargs) + + + def _ipython_display_(self): + bundle = { + 'application/geo+json': self.data, + 'text/plain': '' + } + metadata = { + 'application/geo+json': self.metadata + } + display(bundle, metadata=metadata, raw=True) + +class Javascript(TextDisplayObject): + + def __init__(self, data=None, url=None, filename=None, lib=None, css=None): + """Create a Javascript display object given raw data. + + When this object is returned by an expression or passed to the + display function, it will result in the data being displayed + in the frontend. If the data is a URL, the data will first be + downloaded and then displayed. + + In the Notebook, the containing element will be available as `element`, + and jQuery will be available. Content appended to `element` will be + visible in the output area. + + Parameters + ---------- + data : unicode, str or bytes + The Javascript source code or a URL to download it from. + url : unicode + A URL to download the data from. + filename : unicode + Path to a local file to load the data from. + lib : list or str + A sequence of Javascript library URLs to load asynchronously before + running the source code. The full URLs of the libraries should + be given. A single Javascript library URL can also be given as a + string. + css: : list or str + A sequence of css files to load before running the source code. + The full URLs of the css files should be given. A single css URL + can also be given as a string. + """ + if isinstance(lib, str): + lib = [lib] + elif lib is None: + lib = [] + if isinstance(css, str): + css = [css] + elif css is None: + css = [] + if not isinstance(lib, (list,tuple)): + raise TypeError('expected sequence, got: %r' % lib) + if not isinstance(css, (list,tuple)): + raise TypeError('expected sequence, got: %r' % css) + self.lib = lib + self.css = css + super(Javascript, self).__init__(data=data, url=url, filename=filename) + + def _repr_javascript_(self): + r = '' + for c in self.css: + r += _css_t % c + for l in self.lib: + r += _lib_t1 % l + r += self.data + r += _lib_t2*len(self.lib) + return r + +# constants for identifying png/jpeg data +_PNG = b'\x89PNG\r\n\x1a\n' +_JPEG = b'\xff\xd8' + +def _pngxy(data): + """read the (width, height) from a PNG header""" + ihdr = data.index(b'IHDR') + # next 8 bytes are width/height + return struct.unpack('>ii', data[ihdr+4:ihdr+12]) + +def _jpegxy(data): + """read the (width, height) from a JPEG header""" + # adapted from http://www.64lines.com/jpeg-width-height + + idx = 4 + while True: + block_size = struct.unpack('>H', data[idx:idx+2])[0] + idx = idx + block_size + if data[idx:idx+2] == b'\xFF\xC0': + # found Start of Frame + iSOF = idx + break + else: + # read another block + idx += 2 + + h, w = struct.unpack('>HH', data[iSOF+5:iSOF+9]) + return w, h + +def _gifxy(data): + """read the (width, height) from a GIF header""" + return struct.unpack(' tag. Set this to True if you want the image + to be viewable later with no internet connection in the notebook. + + Default is `True`, unless the keyword argument `url` is set, then + default value is `False`. + + Note that QtConsole is not able to display images if `embed` is set to `False` + width : int + Width in pixels to which to constrain the image in html + height : int + Height in pixels to which to constrain the image in html + retina : bool + Automatically set the width and height to half of the measured + width and height. + This only works for embedded images because it reads the width/height + from image data. + For non-embedded images, you can just set the desired display width + and height directly. + unconfined: bool + Set unconfined=True to disable max-width confinement of the image. + metadata: dict + Specify extra metadata to attach to the image. + + Examples + -------- + # embedded image data, works in qtconsole and notebook + # when passed positionally, the first arg can be any of raw image data, + # a URL, or a filename from which to load image data. + # The result is always embedding image data for inline images. + Image('http://www.google.fr/images/srpr/logo3w.png') + Image('/path/to/image.jpg') + Image(b'RAW_PNG_DATA...') + + # Specifying Image(url=...) does not embed the image data, + # it only generates `` tag with a link to the source. + # This will not work in the qtconsole or offline. + Image(url='http://www.google.fr/images/srpr/logo3w.png') + + """ + if isinstance(data, (Path, PurePath)): + data = str(data) + + if filename is not None: + ext = self._find_ext(filename) + elif url is not None: + ext = self._find_ext(url) + elif data is None: + raise ValueError("No image data found. Expecting filename, url, or data.") + elif isinstance(data, str) and ( + data.startswith('http') or _safe_exists(data) + ): + ext = self._find_ext(data) + else: + ext = None + + if format is None: + if ext is not None: + if ext == u'jpg' or ext == u'jpeg': + format = self._FMT_JPEG + elif ext == u'png': + format = self._FMT_PNG + elif ext == u'gif': + format = self._FMT_GIF + else: + format = ext.lower() + elif isinstance(data, bytes): + # infer image type from image data header, + # only if format has not been specified. + if data[:2] == _JPEG: + format = self._FMT_JPEG + + # failed to detect format, default png + if format is None: + format = self._FMT_PNG + + if format.lower() == 'jpg': + # jpg->jpeg + format = self._FMT_JPEG + + self.format = format.lower() + self.embed = embed if embed is not None else (url is None) + + if self.embed and self.format not in self._ACCEPTABLE_EMBEDDINGS: + raise ValueError("Cannot embed the '%s' image format" % (self.format)) + if self.embed: + self._mimetype = self._MIMETYPES.get(self.format) + + self.width = width + self.height = height + self.retina = retina + self.unconfined = unconfined + super(Image, self).__init__(data=data, url=url, filename=filename, + metadata=metadata) + + if self.width is None and self.metadata.get('width', {}): + self.width = metadata['width'] + + if self.height is None and self.metadata.get('height', {}): + self.height = metadata['height'] + + if retina: + self._retina_shape() + + + def _retina_shape(self): + """load pixel-doubled width and height from image data""" + if not self.embed: + return + if self.format == self._FMT_PNG: + w, h = _pngxy(self.data) + elif self.format == self._FMT_JPEG: + w, h = _jpegxy(self.data) + elif self.format == self._FMT_GIF: + w, h = _gifxy(self.data) + else: + # retina only supports png + return + self.width = w // 2 + self.height = h // 2 + + def reload(self): + """Reload the raw data from file or URL.""" + if self.embed: + super(Image,self).reload() + if self.retina: + self._retina_shape() + + def _repr_html_(self): + if not self.embed: + width = height = klass = '' + if self.width: + width = ' width="%d"' % self.width + if self.height: + height = ' height="%d"' % self.height + if self.unconfined: + klass = ' class="unconfined"' + return u''.format( + url=self.url, + width=width, + height=height, + klass=klass, + ) + + def _repr_mimebundle_(self, include=None, exclude=None): + """Return the image as a mimebundle + + Any new mimetype support should be implemented here. + """ + if self.embed: + mimetype = self._mimetype + data, metadata = self._data_and_metadata(always_both=True) + if metadata: + metadata = {mimetype: metadata} + return {mimetype: data}, metadata + else: + return {'text/html': self._repr_html_()} + + def _data_and_metadata(self, always_both=False): + """shortcut for returning metadata with shape information, if defined""" + try: + b64_data = b2a_base64(self.data).decode('ascii') + except TypeError: + raise FileNotFoundError( + "No such file or directory: '%s'" % (self.data)) + md = {} + if self.metadata: + md.update(self.metadata) + if self.width: + md['width'] = self.width + if self.height: + md['height'] = self.height + if self.unconfined: + md['unconfined'] = self.unconfined + if md or always_both: + return b64_data, md + else: + return b64_data + + def _repr_png_(self): + if self.embed and self.format == self._FMT_PNG: + return self._data_and_metadata() + + def _repr_jpeg_(self): + if self.embed and self.format == self._FMT_JPEG: + return self._data_and_metadata() + + def _find_ext(self, s): + base, ext = splitext(s) + + if not ext: + return base + + # `splitext` includes leading period, so we skip it + return ext[1:].lower() + + +class Video(DisplayObject): + + def __init__(self, data=None, url=None, filename=None, embed=False, + mimetype=None, width=None, height=None, html_attributes="controls"): + """Create a video object given raw data or an URL. + + When this object is returned by an input cell or passed to the + display function, it will result in the video being displayed + in the frontend. + + Parameters + ---------- + data : unicode, str or bytes + The raw video data or a URL or filename to load the data from. + Raw data will require passing ``embed=True``. + url : unicode + A URL for the video. If you specify ``url=``, + the image data will not be embedded. + filename : unicode + Path to a local file containing the video. + Will be interpreted as a local URL unless ``embed=True``. + embed : bool + Should the video be embedded using a data URI (True) or be + loaded using a