aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/python/ipython/py2/IPython/utils
diff options
context:
space:
mode:
authorMikhail Borisov <borisov.mikhail@gmail.com>2022-02-10 16:45:39 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:39 +0300
commita6a92afe03e02795227d2641b49819b687f088f8 (patch)
treef6984a1d27d5a7ec88a6fdd6e20cd5b7693b6ece /contrib/python/ipython/py2/IPython/utils
parentc6dc8b8bd530985bc4cce0137e9a5de32f1087cb (diff)
downloadydb-a6a92afe03e02795227d2641b49819b687f088f8.tar.gz
Restoring authorship annotation for Mikhail Borisov <borisov.mikhail@gmail.com>. Commit 1 of 2.
Diffstat (limited to 'contrib/python/ipython/py2/IPython/utils')
-rw-r--r--contrib/python/ipython/py2/IPython/utils/PyColorize.py680
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_process_cli.py156
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_process_common.py428
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_process_posix.py450
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_process_win32.py384
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_process_win32_controller.py1154
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_signatures.py1626
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_sysinfo.py2
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_tokenize_py2.py878
-rw-r--r--contrib/python/ipython/py2/IPython/utils/_tokenize_py3.py1190
-rw-r--r--contrib/python/ipython/py2/IPython/utils/capture.py312
-rw-r--r--contrib/python/ipython/py2/IPython/utils/coloransi.py374
-rw-r--r--contrib/python/ipython/py2/IPython/utils/contexts.py126
-rw-r--r--contrib/python/ipython/py2/IPython/utils/daemonize.py8
-rw-r--r--contrib/python/ipython/py2/IPython/utils/data.py74
-rw-r--r--contrib/python/ipython/py2/IPython/utils/decorators.py116
-rw-r--r--contrib/python/ipython/py2/IPython/utils/dir2.py90
-rw-r--r--contrib/python/ipython/py2/IPython/utils/encoding.py142
-rw-r--r--contrib/python/ipython/py2/IPython/utils/eventful.py14
-rw-r--r--contrib/python/ipython/py2/IPython/utils/frame.py196
-rw-r--r--contrib/python/ipython/py2/IPython/utils/generics.py68
-rw-r--r--contrib/python/ipython/py2/IPython/utils/importstring.py78
-rw-r--r--contrib/python/ipython/py2/IPython/utils/io.py440
-rw-r--r--contrib/python/ipython/py2/IPython/utils/ipstruct.py782
-rw-r--r--contrib/python/ipython/py2/IPython/utils/jsonutil.py10
-rw-r--r--contrib/python/ipython/py2/IPython/utils/localinterfaces.py10
-rw-r--r--contrib/python/ipython/py2/IPython/utils/log.py14
-rw-r--r--contrib/python/ipython/py2/IPython/utils/module_paths.py250
-rw-r--r--contrib/python/ipython/py2/IPython/utils/openpy.py496
-rw-r--r--contrib/python/ipython/py2/IPython/utils/path.py862
-rw-r--r--contrib/python/ipython/py2/IPython/utils/pickleutil.py10
-rw-r--r--contrib/python/ipython/py2/IPython/utils/process.py212
-rw-r--r--contrib/python/ipython/py2/IPython/utils/py3compat.py668
-rw-r--r--contrib/python/ipython/py2/IPython/utils/rlineimpl.py148
-rw-r--r--contrib/python/ipython/py2/IPython/utils/sentinel.py34
-rw-r--r--contrib/python/ipython/py2/IPython/utils/shimmodule.py184
-rw-r--r--contrib/python/ipython/py2/IPython/utils/signatures.py22
-rw-r--r--contrib/python/ipython/py2/IPython/utils/strdispatch.py136
-rw-r--r--contrib/python/ipython/py2/IPython/utils/sysinfo.py334
-rw-r--r--contrib/python/ipython/py2/IPython/utils/syspathcontext.py142
-rw-r--r--contrib/python/ipython/py2/IPython/utils/tempdir.py290
-rw-r--r--contrib/python/ipython/py2/IPython/utils/terminal.py222
-rw-r--r--contrib/python/ipython/py2/IPython/utils/text.py1552
-rw-r--r--contrib/python/ipython/py2/IPython/utils/timing.py236
-rw-r--r--contrib/python/ipython/py2/IPython/utils/tokenize2.py18
-rw-r--r--contrib/python/ipython/py2/IPython/utils/tokenutil.py254
-rw-r--r--contrib/python/ipython/py2/IPython/utils/traitlets.py14
-rw-r--r--contrib/python/ipython/py2/IPython/utils/tz.py92
-rw-r--r--contrib/python/ipython/py2/IPython/utils/ulinecache.py90
-rw-r--r--contrib/python/ipython/py2/IPython/utils/version.py72
-rw-r--r--contrib/python/ipython/py2/IPython/utils/warn.py92
-rw-r--r--contrib/python/ipython/py2/IPython/utils/wildcard.py224
52 files changed, 8228 insertions, 8228 deletions
diff --git a/contrib/python/ipython/py2/IPython/utils/PyColorize.py b/contrib/python/ipython/py2/IPython/utils/PyColorize.py
index 124eb2d4e3..13c03c3398 100644
--- a/contrib/python/ipython/py2/IPython/utils/PyColorize.py
+++ b/contrib/python/ipython/py2/IPython/utils/PyColorize.py
@@ -1,127 +1,127 @@
-# -*- coding: utf-8 -*-
-"""
-Class and program to colorize python source code for ANSI terminals.
-
-Based on an HTML code highlighter by Jurgen Hermann found at:
-http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
-
-Modifications by Fernando Perez (fperez@colorado.edu).
-
-Information on the original HTML highlighter follows:
-
-MoinMoin - Python Source Parser
-
-Title: Colorize Python source using the built-in tokenizer
-
-Submitter: Jurgen Hermann
-Last Updated:2001/04/06
-
-Version no:1.2
-
-Description:
-
-This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
-Python source code to HTML markup, rendering comments, keywords,
-operators, numeric and string literals in different colors.
-
-It shows how to use the built-in keyword, token and tokenize modules to
-scan Python source code and re-emit it with no changes to its original
-formatting (which is the hard part).
-"""
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-__all__ = ['ANSICodeColors','Parser']
-
-_scheme_default = 'Linux'
-
-
-# Imports
-import keyword
-import os
-import sys
-import token
-import tokenize
-
-try:
- generate_tokens = tokenize.generate_tokens
-except AttributeError:
- # Python 3. Note that we use the undocumented _tokenize because it expects
- # strings, not bytes. See also Python issue #9969.
- generate_tokens = tokenize._tokenize
-
-from IPython.utils.coloransi import TermColors, InputTermColors ,ColorScheme, ColorSchemeTable
-from IPython.utils.py3compat import PY3
-
+# -*- coding: utf-8 -*-
+"""
+Class and program to colorize python source code for ANSI terminals.
+
+Based on an HTML code highlighter by Jurgen Hermann found at:
+http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
+
+Modifications by Fernando Perez (fperez@colorado.edu).
+
+Information on the original HTML highlighter follows:
+
+MoinMoin - Python Source Parser
+
+Title: Colorize Python source using the built-in tokenizer
+
+Submitter: Jurgen Hermann
+Last Updated:2001/04/06
+
+Version no:1.2
+
+Description:
+
+This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
+Python source code to HTML markup, rendering comments, keywords,
+operators, numeric and string literals in different colors.
+
+It shows how to use the built-in keyword, token and tokenize modules to
+scan Python source code and re-emit it with no changes to its original
+formatting (which is the hard part).
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+__all__ = ['ANSICodeColors','Parser']
+
+_scheme_default = 'Linux'
+
+
+# Imports
+import keyword
+import os
+import sys
+import token
+import tokenize
+
+try:
+ generate_tokens = tokenize.generate_tokens
+except AttributeError:
+ # Python 3. Note that we use the undocumented _tokenize because it expects
+ # strings, not bytes. See also Python issue #9969.
+ generate_tokens = tokenize._tokenize
+
+from IPython.utils.coloransi import TermColors, InputTermColors ,ColorScheme, ColorSchemeTable
+from IPython.utils.py3compat import PY3
+
from .colorable import Colorable
-if PY3:
- from io import StringIO
-else:
- from StringIO import StringIO
-
-#############################################################################
-### Python Source Parser (does Hilighting)
-#############################################################################
-
-_KEYWORD = token.NT_OFFSET + 1
-_TEXT = token.NT_OFFSET + 2
-
-#****************************************************************************
-# Builtin color schemes
-
-Colors = TermColors # just a shorthand
-
-# Build a few color schemes
-NoColor = ColorScheme(
- 'NoColor',{
- 'header' : Colors.NoColor,
- token.NUMBER : Colors.NoColor,
- token.OP : Colors.NoColor,
- token.STRING : Colors.NoColor,
- tokenize.COMMENT : Colors.NoColor,
- token.NAME : Colors.NoColor,
- token.ERRORTOKEN : Colors.NoColor,
-
- _KEYWORD : Colors.NoColor,
- _TEXT : Colors.NoColor,
-
- 'in_prompt' : InputTermColors.NoColor, # Input prompt
- 'in_number' : InputTermColors.NoColor, # Input prompt number
- 'in_prompt2' : InputTermColors.NoColor, # Continuation prompt
- 'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal)
-
- 'out_prompt' : Colors.NoColor, # Output prompt
- 'out_number' : Colors.NoColor, # Output prompt number
-
- 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
- } )
-
-LinuxColors = ColorScheme(
- 'Linux',{
- 'header' : Colors.LightRed,
- token.NUMBER : Colors.LightCyan,
- token.OP : Colors.Yellow,
- token.STRING : Colors.LightBlue,
- tokenize.COMMENT : Colors.LightRed,
- token.NAME : Colors.Normal,
- token.ERRORTOKEN : Colors.Red,
-
- _KEYWORD : Colors.LightGreen,
- _TEXT : Colors.Yellow,
-
- 'in_prompt' : InputTermColors.Green,
- 'in_number' : InputTermColors.LightGreen,
- 'in_prompt2' : InputTermColors.Green,
- 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
-
- 'out_prompt' : Colors.Red,
- 'out_number' : Colors.LightRed,
-
- 'normal' : Colors.Normal # color off (usu. Colors.Normal)
- } )
-
+if PY3:
+ from io import StringIO
+else:
+ from StringIO import StringIO
+
+#############################################################################
+### Python Source Parser (does Hilighting)
+#############################################################################
+
+_KEYWORD = token.NT_OFFSET + 1
+_TEXT = token.NT_OFFSET + 2
+
+#****************************************************************************
+# Builtin color schemes
+
+Colors = TermColors # just a shorthand
+
+# Build a few color schemes
+NoColor = ColorScheme(
+ 'NoColor',{
+ 'header' : Colors.NoColor,
+ token.NUMBER : Colors.NoColor,
+ token.OP : Colors.NoColor,
+ token.STRING : Colors.NoColor,
+ tokenize.COMMENT : Colors.NoColor,
+ token.NAME : Colors.NoColor,
+ token.ERRORTOKEN : Colors.NoColor,
+
+ _KEYWORD : Colors.NoColor,
+ _TEXT : Colors.NoColor,
+
+ 'in_prompt' : InputTermColors.NoColor, # Input prompt
+ 'in_number' : InputTermColors.NoColor, # Input prompt number
+ 'in_prompt2' : InputTermColors.NoColor, # Continuation prompt
+ 'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal)
+
+ 'out_prompt' : Colors.NoColor, # Output prompt
+ 'out_number' : Colors.NoColor, # Output prompt number
+
+ 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
+ } )
+
+LinuxColors = ColorScheme(
+ 'Linux',{
+ 'header' : Colors.LightRed,
+ token.NUMBER : Colors.LightCyan,
+ token.OP : Colors.Yellow,
+ token.STRING : Colors.LightBlue,
+ tokenize.COMMENT : Colors.LightRed,
+ token.NAME : Colors.Normal,
+ token.ERRORTOKEN : Colors.Red,
+
+ _KEYWORD : Colors.LightGreen,
+ _TEXT : Colors.Yellow,
+
+ 'in_prompt' : InputTermColors.Green,
+ 'in_number' : InputTermColors.LightGreen,
+ 'in_prompt2' : InputTermColors.Green,
+ 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
+
+ 'out_prompt' : Colors.Red,
+ 'out_number' : Colors.LightRed,
+
+ 'normal' : Colors.Normal # color off (usu. Colors.Normal)
+ } )
+
NeutralColors = ColorScheme(
'Neutral',{
'header' : Colors.Red,
@@ -156,227 +156,227 @@ NeutralColors = ColorScheme(
if os.name == 'nt':
NeutralColors = LinuxColors.copy(name='Neutral')
-LightBGColors = ColorScheme(
- 'LightBG',{
- 'header' : Colors.Red,
- token.NUMBER : Colors.Cyan,
- token.OP : Colors.Blue,
- token.STRING : Colors.Blue,
- tokenize.COMMENT : Colors.Red,
- token.NAME : Colors.Normal,
- token.ERRORTOKEN : Colors.Red,
-
-
- _KEYWORD : Colors.Green,
- _TEXT : Colors.Blue,
-
- 'in_prompt' : InputTermColors.Blue,
- 'in_number' : InputTermColors.LightBlue,
- 'in_prompt2' : InputTermColors.Blue,
- 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
-
- 'out_prompt' : Colors.Red,
- 'out_number' : Colors.LightRed,
-
- 'normal' : Colors.Normal # color off (usu. Colors.Normal)
- } )
-
-# Build table of color schemes (needed by the parser)
+LightBGColors = ColorScheme(
+ 'LightBG',{
+ 'header' : Colors.Red,
+ token.NUMBER : Colors.Cyan,
+ token.OP : Colors.Blue,
+ token.STRING : Colors.Blue,
+ tokenize.COMMENT : Colors.Red,
+ token.NAME : Colors.Normal,
+ token.ERRORTOKEN : Colors.Red,
+
+
+ _KEYWORD : Colors.Green,
+ _TEXT : Colors.Blue,
+
+ 'in_prompt' : InputTermColors.Blue,
+ 'in_number' : InputTermColors.LightBlue,
+ 'in_prompt2' : InputTermColors.Blue,
+ 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
+
+ 'out_prompt' : Colors.Red,
+ 'out_number' : Colors.LightRed,
+
+ 'normal' : Colors.Normal # color off (usu. Colors.Normal)
+ } )
+
+# Build table of color schemes (needed by the parser)
ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors],
- _scheme_default)
-
+ _scheme_default)
+
class Parser(Colorable):
- """ Format colored Python source.
- """
-
+ """ Format colored Python source.
+ """
+
def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None):
- """ Create a parser with a specified color table and output channel.
-
- Call format() to process code.
- """
+ """ Create a parser with a specified color table and output channel.
+
+ Call format() to process code.
+ """
super(Parser, self).__init__(parent=parent)
- self.color_table = color_table and color_table or ANSICodeColors
- self.out = out
-
- def format(self, raw, out = None, scheme = ''):
- return self.format2(raw, out, scheme)[0]
-
- def format2(self, raw, out = None, scheme = ''):
- """ Parse and send the colored source.
-
- If out and scheme are not specified, the defaults (given to
- constructor) are used.
-
- out should be a file-type object. Optionally, out can be given as the
- string 'str' and the parser will automatically return the output in a
- string."""
-
- string_output = 0
- if out == 'str' or self.out == 'str' or \
- isinstance(self.out,StringIO):
- # XXX - I don't really like this state handling logic, but at this
- # point I don't want to make major changes, so adding the
- # isinstance() check is the simplest I can do to ensure correct
- # behavior.
- out_old = self.out
- self.out = StringIO()
- string_output = 1
- elif out is not None:
- self.out = out
-
- # Fast return of the unmodified input for NoColor scheme
- if scheme == 'NoColor':
- error = False
- self.out.write(raw)
- if string_output:
- return raw,error
- else:
- return None,error
-
- # local shorthands
- colors = self.color_table[scheme].colors
- self.colors = colors # put in object so __call__ sees it
-
- # Remove trailing whitespace and normalize tabs
- self.raw = raw.expandtabs().rstrip()
-
- # store line offsets in self.lines
- self.lines = [0, 0]
- pos = 0
- raw_find = self.raw.find
- lines_append = self.lines.append
- while 1:
- pos = raw_find('\n', pos) + 1
- if not pos: break
- lines_append(pos)
- lines_append(len(self.raw))
-
- # parse the source and write it
- self.pos = 0
- text = StringIO(self.raw)
-
- error = False
- try:
- for atoken in generate_tokens(text.readline):
- self(*atoken)
- except tokenize.TokenError as ex:
- msg = ex.args[0]
- line = ex.args[1][0]
- self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
- (colors[token.ERRORTOKEN],
- msg, self.raw[self.lines[line]:],
- colors.normal)
- )
- error = True
- self.out.write(colors.normal+'\n')
- if string_output:
- output = self.out.getvalue()
- self.out = out_old
- return (output, error)
- return (None, error)
-
- def __call__(self, toktype, toktext, start_pos, end_pos, line):
- """ Token handler, with syntax highlighting."""
- (srow,scol) = start_pos
- (erow,ecol) = end_pos
- colors = self.colors
- owrite = self.out.write
-
- # line separator, so this works across platforms
- linesep = os.linesep
-
- # calculate new positions
- oldpos = self.pos
- newpos = self.lines[srow] + scol
- self.pos = newpos + len(toktext)
-
- # send the original whitespace, if needed
- if newpos > oldpos:
- owrite(self.raw[oldpos:newpos])
-
- # skip indenting tokens
- if toktype in [token.INDENT, token.DEDENT]:
- self.pos = newpos
- return
-
- # map token type to a color group
- if token.LPAR <= toktype <= token.OP:
- toktype = token.OP
- elif toktype == token.NAME and keyword.iskeyword(toktext):
- toktype = _KEYWORD
- color = colors.get(toktype, colors[_TEXT])
-
- #print '<%s>' % toktext, # dbg
-
- # Triple quoted strings must be handled carefully so that backtracking
- # in pagers works correctly. We need color terminators on _each_ line.
- if linesep in toktext:
- toktext = toktext.replace(linesep, '%s%s%s' %
- (colors.normal,linesep,color))
-
- # send text
- owrite('%s%s%s' % (color,toktext,colors.normal))
-
-def main(argv=None):
- """Run as a command-line script: colorize a python file or stdin using ANSI
- color escapes and print to stdout.
-
- Inputs:
-
- - argv(None): a list of strings like sys.argv[1:] giving the command-line
- arguments. If None, use sys.argv[1:].
- """
-
- usage_msg = """%prog [options] [filename]
-
-Colorize a python file or stdin using ANSI color escapes and print to stdout.
-If no filename is given, or if filename is -, read standard input."""
-
- import optparse
- parser = optparse.OptionParser(usage=usage_msg)
- newopt = parser.add_option
- newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
- choices=['Linux','LightBG','NoColor'],default=_scheme_default,
- help="give the color scheme to use. Currently only 'Linux'\
- (default) and 'LightBG' and 'NoColor' are implemented (give without\
- quotes)")
-
- opts,args = parser.parse_args(argv)
-
- if len(args) > 1:
- parser.error("you must give at most one filename.")
-
- if len(args) == 0:
- fname = '-' # no filename given; setup to read from stdin
- else:
- fname = args[0]
-
- if fname == '-':
- stream = sys.stdin
- else:
- try:
- stream = open(fname)
- except IOError as msg:
- print(msg, file=sys.stderr)
- sys.exit(1)
-
- parser = Parser()
-
- # we need nested try blocks because pre-2.5 python doesn't support unified
- # try-except-finally
- try:
- try:
- # write colorized version to stdout
- parser.format(stream.read(),scheme=opts.scheme_name)
- except IOError as msg:
- # if user reads through a pager and quits, don't print traceback
- if msg.args != (32,'Broken pipe'):
- raise
- finally:
- if stream is not sys.stdin:
- stream.close() # in case a non-handled exception happened above
-
-if __name__ == "__main__":
- main()
+ self.color_table = color_table and color_table or ANSICodeColors
+ self.out = out
+
+ def format(self, raw, out = None, scheme = ''):
+ return self.format2(raw, out, scheme)[0]
+
+ def format2(self, raw, out = None, scheme = ''):
+ """ Parse and send the colored source.
+
+ If out and scheme are not specified, the defaults (given to
+ constructor) are used.
+
+ out should be a file-type object. Optionally, out can be given as the
+ string 'str' and the parser will automatically return the output in a
+ string."""
+
+ string_output = 0
+ if out == 'str' or self.out == 'str' or \
+ isinstance(self.out,StringIO):
+ # XXX - I don't really like this state handling logic, but at this
+ # point I don't want to make major changes, so adding the
+ # isinstance() check is the simplest I can do to ensure correct
+ # behavior.
+ out_old = self.out
+ self.out = StringIO()
+ string_output = 1
+ elif out is not None:
+ self.out = out
+
+ # Fast return of the unmodified input for NoColor scheme
+ if scheme == 'NoColor':
+ error = False
+ self.out.write(raw)
+ if string_output:
+ return raw,error
+ else:
+ return None,error
+
+ # local shorthands
+ colors = self.color_table[scheme].colors
+ self.colors = colors # put in object so __call__ sees it
+
+ # Remove trailing whitespace and normalize tabs
+ self.raw = raw.expandtabs().rstrip()
+
+ # store line offsets in self.lines
+ self.lines = [0, 0]
+ pos = 0
+ raw_find = self.raw.find
+ lines_append = self.lines.append
+ while 1:
+ pos = raw_find('\n', pos) + 1
+ if not pos: break
+ lines_append(pos)
+ lines_append(len(self.raw))
+
+ # parse the source and write it
+ self.pos = 0
+ text = StringIO(self.raw)
+
+ error = False
+ try:
+ for atoken in generate_tokens(text.readline):
+ self(*atoken)
+ except tokenize.TokenError as ex:
+ msg = ex.args[0]
+ line = ex.args[1][0]
+ self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
+ (colors[token.ERRORTOKEN],
+ msg, self.raw[self.lines[line]:],
+ colors.normal)
+ )
+ error = True
+ self.out.write(colors.normal+'\n')
+ if string_output:
+ output = self.out.getvalue()
+ self.out = out_old
+ return (output, error)
+ return (None, error)
+
+ def __call__(self, toktype, toktext, start_pos, end_pos, line):
+ """ Token handler, with syntax highlighting."""
+ (srow,scol) = start_pos
+ (erow,ecol) = end_pos
+ colors = self.colors
+ owrite = self.out.write
+
+ # line separator, so this works across platforms
+ linesep = os.linesep
+
+ # calculate new positions
+ oldpos = self.pos
+ newpos = self.lines[srow] + scol
+ self.pos = newpos + len(toktext)
+
+ # send the original whitespace, if needed
+ if newpos > oldpos:
+ owrite(self.raw[oldpos:newpos])
+
+ # skip indenting tokens
+ if toktype in [token.INDENT, token.DEDENT]:
+ self.pos = newpos
+ return
+
+ # map token type to a color group
+ if token.LPAR <= toktype <= token.OP:
+ toktype = token.OP
+ elif toktype == token.NAME and keyword.iskeyword(toktext):
+ toktype = _KEYWORD
+ color = colors.get(toktype, colors[_TEXT])
+
+ #print '<%s>' % toktext, # dbg
+
+ # Triple quoted strings must be handled carefully so that backtracking
+ # in pagers works correctly. We need color terminators on _each_ line.
+ if linesep in toktext:
+ toktext = toktext.replace(linesep, '%s%s%s' %
+ (colors.normal,linesep,color))
+
+ # send text
+ owrite('%s%s%s' % (color,toktext,colors.normal))
+
+def main(argv=None):
+ """Run as a command-line script: colorize a python file or stdin using ANSI
+ color escapes and print to stdout.
+
+ Inputs:
+
+ - argv(None): a list of strings like sys.argv[1:] giving the command-line
+ arguments. If None, use sys.argv[1:].
+ """
+
+ usage_msg = """%prog [options] [filename]
+
+Colorize a python file or stdin using ANSI color escapes and print to stdout.
+If no filename is given, or if filename is -, read standard input."""
+
+ import optparse
+ parser = optparse.OptionParser(usage=usage_msg)
+ newopt = parser.add_option
+ newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
+ choices=['Linux','LightBG','NoColor'],default=_scheme_default,
+ help="give the color scheme to use. Currently only 'Linux'\
+ (default) and 'LightBG' and 'NoColor' are implemented (give without\
+ quotes)")
+
+ opts,args = parser.parse_args(argv)
+
+ if len(args) > 1:
+ parser.error("you must give at most one filename.")
+
+ if len(args) == 0:
+ fname = '-' # no filename given; setup to read from stdin
+ else:
+ fname = args[0]
+
+ if fname == '-':
+ stream = sys.stdin
+ else:
+ try:
+ stream = open(fname)
+ except IOError as msg:
+ print(msg, file=sys.stderr)
+ sys.exit(1)
+
+ parser = Parser()
+
+ # we need nested try blocks because pre-2.5 python doesn't support unified
+ # try-except-finally
+ try:
+ try:
+ # write colorized version to stdout
+ parser.format(stream.read(),scheme=opts.scheme_name)
+ except IOError as msg:
+ # if user reads through a pager and quits, don't print traceback
+ if msg.args != (32,'Broken pipe'):
+ raise
+ finally:
+ if stream is not sys.stdin:
+ stream.close() # in case a non-handled exception happened above
+
+if __name__ == "__main__":
+ main()
diff --git a/contrib/python/ipython/py2/IPython/utils/_process_cli.py b/contrib/python/ipython/py2/IPython/utils/_process_cli.py
index a7b7b90b68..a65decf3b6 100644
--- a/contrib/python/ipython/py2/IPython/utils/_process_cli.py
+++ b/contrib/python/ipython/py2/IPython/utils/_process_cli.py
@@ -1,78 +1,78 @@
-"""cli-specific implementation of process utilities.
-
-cli - Common Language Infrastructure for IronPython. Code
- can run on any operating system. Check os.name for os-
- specific settings.
-
-This file is only meant to be imported by process.py, not by end-users.
-
-This file is largely untested. To become a full drop-in process
-interface for IronPython will probably require you to help fill
-in the details.
-"""
-
-# Import cli libraries:
-import clr
-import System
-
-# Import Python libraries:
-import os
-
-# Import IPython libraries:
-from IPython.utils import py3compat
-from ._process_common import arg_split
-
-def _find_cmd(cmd):
- """Find the full path to a command using which."""
- paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
- for path in paths:
- filename = os.path.join(path, cmd)
- if System.IO.File.Exists(filename):
- return py3compat.bytes_to_str(filename)
- raise OSError("command %r not found" % cmd)
-
-def system(cmd):
- """
- system(cmd) should work in a cli environment on Mac OSX, Linux,
- and Windows
- """
- psi = System.Diagnostics.ProcessStartInfo(cmd)
- psi.RedirectStandardOutput = True
- psi.RedirectStandardError = True
- psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
- psi.UseShellExecute = False
- # Start up process:
- reg = System.Diagnostics.Process.Start(psi)
-
-def getoutput(cmd):
- """
- getoutput(cmd) should work in a cli environment on Mac OSX, Linux,
- and Windows
- """
- psi = System.Diagnostics.ProcessStartInfo(cmd)
- psi.RedirectStandardOutput = True
- psi.RedirectStandardError = True
- psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
- psi.UseShellExecute = False
- # Start up process:
- reg = System.Diagnostics.Process.Start(psi)
- myOutput = reg.StandardOutput
- output = myOutput.ReadToEnd()
- myError = reg.StandardError
- error = myError.ReadToEnd()
- return output
-
-def check_pid(pid):
- """
- Check if a process with the given PID (pid) exists
- """
- try:
- System.Diagnostics.Process.GetProcessById(pid)
- # process with given pid is running
- return True
- except System.InvalidOperationException:
- # process wasn't started by this object (but is running)
- return True
- except System.ArgumentException:
- # process with given pid isn't running
- return False
+"""cli-specific implementation of process utilities.
+
+cli - Common Language Infrastructure for IronPython. Code
+ can run on any operating system. Check os.name for os-
+ specific settings.
+
+This file is only meant to be imported by process.py, not by end-users.
+
+This file is largely untested. To become a full drop-in process
+interface for IronPython will probably require you to help fill
+in the details.
+"""
+
+# Import cli libraries:
+import clr
+import System
+
+# Import Python libraries:
+import os
+
+# Import IPython libraries:
+from IPython.utils import py3compat
+from ._process_common import arg_split
+
+def _find_cmd(cmd):
+ """Find the full path to a command using which."""
+ paths = System.Environment.GetEnvironmentVariable("PATH").Split(os.pathsep)
+ for path in paths:
+ filename = os.path.join(path, cmd)
+ if System.IO.File.Exists(filename):
+ return py3compat.bytes_to_str(filename)
+ raise OSError("command %r not found" % cmd)
+
+def system(cmd):
+ """
+ system(cmd) should work in a cli environment on Mac OSX, Linux,
+ and Windows
+ """
+ psi = System.Diagnostics.ProcessStartInfo(cmd)
+ psi.RedirectStandardOutput = True
+ psi.RedirectStandardError = True
+ psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
+ psi.UseShellExecute = False
+ # Start up process:
+ reg = System.Diagnostics.Process.Start(psi)
+
+def getoutput(cmd):
+ """
+ getoutput(cmd) should work in a cli environment on Mac OSX, Linux,
+ and Windows
+ """
+ psi = System.Diagnostics.ProcessStartInfo(cmd)
+ psi.RedirectStandardOutput = True
+ psi.RedirectStandardError = True
+ psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
+ psi.UseShellExecute = False
+ # Start up process:
+ reg = System.Diagnostics.Process.Start(psi)
+ myOutput = reg.StandardOutput
+ output = myOutput.ReadToEnd()
+ myError = reg.StandardError
+ error = myError.ReadToEnd()
+ return output
+
+def check_pid(pid):
+ """
+ Check if a process with the given PID (pid) exists
+ """
+ try:
+ System.Diagnostics.Process.GetProcessById(pid)
+ # process with given pid is running
+ return True
+ except System.InvalidOperationException:
+ # process wasn't started by this object (but is running)
+ return True
+ except System.ArgumentException:
+ # process with given pid isn't running
+ return False
diff --git a/contrib/python/ipython/py2/IPython/utils/_process_common.py b/contrib/python/ipython/py2/IPython/utils/_process_common.py
index 9ede30d3f8..6851e41869 100644
--- a/contrib/python/ipython/py2/IPython/utils/_process_common.py
+++ b/contrib/python/ipython/py2/IPython/utils/_process_common.py
@@ -1,75 +1,75 @@
-"""Common utilities for the various process_* implementations.
-
-This file is only meant to be imported by the platform-specific implementations
-of subprocess utilities, and it contains tools that are common to all of them.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2010-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-import subprocess
-import shlex
-import sys
+"""Common utilities for the various process_* implementations.
+
+This file is only meant to be imported by the platform-specific implementations
+of subprocess utilities, and it contains tools that are common to all of them.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import subprocess
+import shlex
+import sys
import os
-
-from IPython.utils import py3compat
-
-#-----------------------------------------------------------------------------
-# Function definitions
-#-----------------------------------------------------------------------------
-
-def read_no_interrupt(p):
- """Read from a pipe ignoring EINTR errors.
-
- This is necessary because when reading from pipes with GUI event loops
- running in the background, often interrupts are raised that stop the
- command from completing."""
- import errno
-
- try:
- return p.read()
- except IOError as err:
- if err.errno != errno.EINTR:
- raise
-
-
-def process_handler(cmd, callback, stderr=subprocess.PIPE):
- """Open a command in a shell subprocess and execute a callback.
-
- This function provides common scaffolding for creating subprocess.Popen()
- calls. It creates a Popen object and then calls the callback with it.
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed by the system, using :class:`subprocess.Popen`.
- If a string is passed, it will be run in the system shell. If a list is
- passed, it will be used directly as arguments.
-
- callback : callable
- A one-argument function that will be called with the Popen object.
-
- stderr : file descriptor number, optional
- By default this is set to ``subprocess.PIPE``, but you can also pass the
- value ``subprocess.STDOUT`` to force the subprocess' stderr to go into
- the same file descriptor as its stdout. This is useful to read stdout
- and stderr combined in the order they are generated.
-
- Returns
- -------
- The return value of the provided callback is returned.
- """
- sys.stdout.flush()
- sys.stderr.flush()
- # On win32, close_fds can't be true when using pipes for stdin/out/err
- close_fds = sys.platform != 'win32'
+
+from IPython.utils import py3compat
+
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+def read_no_interrupt(p):
+ """Read from a pipe ignoring EINTR errors.
+
+ This is necessary because when reading from pipes with GUI event loops
+ running in the background, often interrupts are raised that stop the
+ command from completing."""
+ import errno
+
+ try:
+ return p.read()
+ except IOError as err:
+ if err.errno != errno.EINTR:
+ raise
+
+
+def process_handler(cmd, callback, stderr=subprocess.PIPE):
+ """Open a command in a shell subprocess and execute a callback.
+
+ This function provides common scaffolding for creating subprocess.Popen()
+ calls. It creates a Popen object and then calls the callback with it.
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed by the system, using :class:`subprocess.Popen`.
+ If a string is passed, it will be run in the system shell. If a list is
+ passed, it will be used directly as arguments.
+
+ callback : callable
+ A one-argument function that will be called with the Popen object.
+
+ stderr : file descriptor number, optional
+ By default this is set to ``subprocess.PIPE``, but you can also pass the
+ value ``subprocess.STDOUT`` to force the subprocess' stderr to go into
+ the same file descriptor as its stdout. This is useful to read stdout
+ and stderr combined in the order they are generated.
+
+ Returns
+ -------
+ The return value of the provided callback is returned.
+ """
+ sys.stdout.flush()
+ sys.stderr.flush()
+ # On win32, close_fds can't be true when using pipes for stdin/out/err
+ close_fds = sys.platform != 'win32'
# Determine if cmd should be run with system shell.
shell = isinstance(cmd, py3compat.string_types)
# On POSIX systems run shell commands with user-preferred shell.
@@ -78,146 +78,146 @@ def process_handler(cmd, callback, stderr=subprocess.PIPE):
executable = os.environ['SHELL']
p = subprocess.Popen(cmd, shell=shell,
executable=executable,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=stderr,
- close_fds=close_fds)
-
- try:
- out = callback(p)
- except KeyboardInterrupt:
- print('^C')
- sys.stdout.flush()
- sys.stderr.flush()
- out = None
- finally:
- # Make really sure that we don't leave processes behind, in case the
- # call above raises an exception
- # We start by assuming the subprocess finished (to avoid NameErrors
- # later depending on the path taken)
- if p.returncode is None:
- try:
- p.terminate()
- p.poll()
- except OSError:
- pass
- # One last try on our way out
- if p.returncode is None:
- try:
- p.kill()
- except OSError:
- pass
-
- return out
-
-
-def getoutput(cmd):
- """Run a command and return its stdout/stderr as a string.
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed in the system shell.
-
- Returns
- -------
- output : str
- A string containing the combination of stdout and stderr from the
- subprocess, in whatever order the subprocess originally wrote to its
- file descriptors (so the order of the information in this string is the
- correct order as would be seen if running the command in a terminal).
- """
- out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
- if out is None:
- return ''
- return py3compat.bytes_to_str(out)
-
-
-def getoutputerror(cmd):
- """Return (standard output, standard error) of executing cmd in a shell.
-
- Accepts the same arguments as os.system().
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed in the system shell.
-
- Returns
- -------
- stdout : str
- stderr : str
- """
- return get_output_error_code(cmd)[:2]
-
-def get_output_error_code(cmd):
- """Return (standard output, standard error, return code) of executing cmd
- in a shell.
-
- Accepts the same arguments as os.system().
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed in the system shell.
-
- Returns
- -------
- stdout : str
- stderr : str
- returncode: int
- """
-
- out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
- if out_err is None:
- return '', '', p.returncode
- out, err = out_err
- return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err), p.returncode
-
-def arg_split(s, posix=False, strict=True):
- """Split a command line's arguments in a shell-like manner.
-
- This is a modified version of the standard library's shlex.split()
- function, but with a default of posix=False for splitting, so that quotes
- in inputs are respected.
-
- if strict=False, then any errors shlex.split would raise will result in the
- unparsed remainder being the last element of the list, rather than raising.
- This is because we sometimes use arg_split to parse things other than
- command-line args.
- """
-
- # Unfortunately, python's shlex module is buggy with unicode input:
- # http://bugs.python.org/issue1170
- # At least encoding the input when it's unicode seems to help, but there
- # may be more problems lurking. Apparently this is fixed in python3.
- is_unicode = False
- if (not py3compat.PY3) and isinstance(s, unicode):
- is_unicode = True
- s = s.encode('utf-8')
- lex = shlex.shlex(s, posix=posix)
- lex.whitespace_split = True
- # Extract tokens, ensuring that things like leaving open quotes
- # does not cause this to raise. This is important, because we
- # sometimes pass Python source through this (e.g. %timeit f(" ")),
- # and it shouldn't raise an exception.
- # It may be a bad idea to parse things that are not command-line args
- # through this function, but we do, so let's be safe about it.
- lex.commenters='' #fix for GH-1269
- tokens = []
- while True:
- try:
- tokens.append(next(lex))
- except StopIteration:
- break
- except ValueError:
- if strict:
- raise
- # couldn't parse, get remaining blob as last token
- tokens.append(lex.token)
- break
-
- if is_unicode:
- # Convert the tokens back to unicode.
- tokens = [x.decode('utf-8') for x in tokens]
- return tokens
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ close_fds=close_fds)
+
+ try:
+ out = callback(p)
+ except KeyboardInterrupt:
+ print('^C')
+ sys.stdout.flush()
+ sys.stderr.flush()
+ out = None
+ finally:
+ # Make really sure that we don't leave processes behind, in case the
+ # call above raises an exception
+ # We start by assuming the subprocess finished (to avoid NameErrors
+ # later depending on the path taken)
+ if p.returncode is None:
+ try:
+ p.terminate()
+ p.poll()
+ except OSError:
+ pass
+ # One last try on our way out
+ if p.returncode is None:
+ try:
+ p.kill()
+ except OSError:
+ pass
+
+ return out
+
+
+def getoutput(cmd):
+ """Run a command and return its stdout/stderr as a string.
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ output : str
+ A string containing the combination of stdout and stderr from the
+ subprocess, in whatever order the subprocess originally wrote to its
+ file descriptors (so the order of the information in this string is the
+ correct order as would be seen if running the command in a terminal).
+ """
+ out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
+ if out is None:
+ return ''
+ return py3compat.bytes_to_str(out)
+
+
+def getoutputerror(cmd):
+ """Return (standard output, standard error) of executing cmd in a shell.
+
+ Accepts the same arguments as os.system().
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ stdout : str
+ stderr : str
+ """
+ return get_output_error_code(cmd)[:2]
+
+def get_output_error_code(cmd):
+ """Return (standard output, standard error, return code) of executing cmd
+ in a shell.
+
+ Accepts the same arguments as os.system().
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ stdout : str
+ stderr : str
+ returncode: int
+ """
+
+ out_err, p = process_handler(cmd, lambda p: (p.communicate(), p))
+ if out_err is None:
+ return '', '', p.returncode
+ out, err = out_err
+ return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err), p.returncode
+
+def arg_split(s, posix=False, strict=True):
+ """Split a command line's arguments in a shell-like manner.
+
+ This is a modified version of the standard library's shlex.split()
+ function, but with a default of posix=False for splitting, so that quotes
+ in inputs are respected.
+
+ if strict=False, then any errors shlex.split would raise will result in the
+ unparsed remainder being the last element of the list, rather than raising.
+ This is because we sometimes use arg_split to parse things other than
+ command-line args.
+ """
+
+ # Unfortunately, python's shlex module is buggy with unicode input:
+ # http://bugs.python.org/issue1170
+ # At least encoding the input when it's unicode seems to help, but there
+ # may be more problems lurking. Apparently this is fixed in python3.
+ is_unicode = False
+ if (not py3compat.PY3) and isinstance(s, unicode):
+ is_unicode = True
+ s = s.encode('utf-8')
+ lex = shlex.shlex(s, posix=posix)
+ lex.whitespace_split = True
+ # Extract tokens, ensuring that things like leaving open quotes
+ # does not cause this to raise. This is important, because we
+ # sometimes pass Python source through this (e.g. %timeit f(" ")),
+ # and it shouldn't raise an exception.
+ # It may be a bad idea to parse things that are not command-line args
+ # through this function, but we do, so let's be safe about it.
+ lex.commenters='' #fix for GH-1269
+ tokens = []
+ while True:
+ try:
+ tokens.append(next(lex))
+ except StopIteration:
+ break
+ except ValueError:
+ if strict:
+ raise
+ # couldn't parse, get remaining blob as last token
+ tokens.append(lex.token)
+ break
+
+ if is_unicode:
+ # Convert the tokens back to unicode.
+ tokens = [x.decode('utf-8') for x in tokens]
+ return tokens
diff --git a/contrib/python/ipython/py2/IPython/utils/_process_posix.py b/contrib/python/ipython/py2/IPython/utils/_process_posix.py
index ac3a9a0507..059e80c991 100644
--- a/contrib/python/ipython/py2/IPython/utils/_process_posix.py
+++ b/contrib/python/ipython/py2/IPython/utils/_process_posix.py
@@ -1,225 +1,225 @@
-"""Posix-specific implementation of process utilities.
-
-This file is only meant to be imported by process.py, not by end-users.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2010-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
-
-# Stdlib
-import errno
-import os
-import subprocess as sp
-import sys
-
-import pexpect
-
-# Our own
-from ._process_common import getoutput, arg_split
-from IPython.utils import py3compat
-from IPython.utils.encoding import DEFAULT_ENCODING
-
-#-----------------------------------------------------------------------------
-# Function definitions
-#-----------------------------------------------------------------------------
-
-def _find_cmd(cmd):
- """Find the full path to a command using which."""
-
- path = sp.Popen(['/usr/bin/env', 'which', cmd],
- stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
- return py3compat.bytes_to_str(path)
-
-
-class ProcessHandler(object):
- """Execute subprocesses under the control of pexpect.
- """
- # Timeout in seconds to wait on each reading of the subprocess' output.
- # This should not be set too low to avoid cpu overusage from our side,
- # since we read in a loop whose period is controlled by this timeout.
- read_timeout = 0.05
-
- # Timeout to give a process if we receive SIGINT, between sending the
- # SIGINT to the process and forcefully terminating it.
- terminate_timeout = 0.2
-
- # File object where stdout and stderr of the subprocess will be written
- logfile = None
-
- # Shell to call for subprocesses to execute
- _sh = None
-
- @property
- def sh(self):
- if self._sh is None:
- self._sh = pexpect.which('sh')
- if self._sh is None:
- raise OSError('"sh" shell not found')
-
- return self._sh
-
- def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
- """Arguments are used for pexpect calls."""
- self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
- None else read_timeout)
- self.terminate_timeout = (ProcessHandler.terminate_timeout if
- terminate_timeout is None else
- terminate_timeout)
- self.logfile = sys.stdout if logfile is None else logfile
-
- def getoutput(self, cmd):
- """Run a command and return its stdout/stderr as a string.
-
- Parameters
- ----------
- cmd : str
- A command to be executed in the system shell.
-
- Returns
- -------
- output : str
- A string containing the combination of stdout and stderr from the
- subprocess, in whatever order the subprocess originally wrote to its
- file descriptors (so the order of the information in this string is the
- correct order as would be seen if running the command in a terminal).
- """
- try:
- return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
- except KeyboardInterrupt:
- print('^C', file=sys.stderr, end='')
-
- def getoutput_pexpect(self, cmd):
- """Run a command and return its stdout/stderr as a string.
-
- Parameters
- ----------
- cmd : str
- A command to be executed in the system shell.
-
- Returns
- -------
- output : str
- A string containing the combination of stdout and stderr from the
- subprocess, in whatever order the subprocess originally wrote to its
- file descriptors (so the order of the information in this string is the
- correct order as would be seen if running the command in a terminal).
- """
- try:
- return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
- except KeyboardInterrupt:
- print('^C', file=sys.stderr, end='')
-
- def system(self, cmd):
- """Execute a command in a subshell.
-
- Parameters
- ----------
- cmd : str
- A command to be executed in the system shell.
-
- Returns
- -------
- int : child's exitstatus
- """
- # Get likely encoding for the output.
- enc = DEFAULT_ENCODING
-
- # Patterns to match on the output, for pexpect. We read input and
- # allow either a short timeout or EOF
- patterns = [pexpect.TIMEOUT, pexpect.EOF]
- # the index of the EOF pattern in the list.
- # even though we know it's 1, this call means we don't have to worry if
- # we change the above list, and forget to change this value:
- EOF_index = patterns.index(pexpect.EOF)
- # The size of the output stored so far in the process output buffer.
- # Since pexpect only appends to this buffer, each time we print we
- # record how far we've printed, so that next time we only print *new*
- # content from the buffer.
- out_size = 0
- try:
- # Since we're not really searching the buffer for text patterns, we
- # can set pexpect's search window to be tiny and it won't matter.
- # We only search for the 'patterns' timeout or EOF, which aren't in
- # the text itself.
- #child = pexpect.spawn(pcmd, searchwindowsize=1)
- if hasattr(pexpect, 'spawnb'):
- child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
- else:
- child = pexpect.spawn(self.sh, args=['-c', cmd]) # Vanilla Pexpect
- flush = sys.stdout.flush
- while True:
- # res is the index of the pattern that caused the match, so we
- # know whether we've finished (if we matched EOF) or not
- res_idx = child.expect_list(patterns, self.read_timeout)
- print(child.before[out_size:].decode(enc, 'replace'), end='')
- flush()
- if res_idx==EOF_index:
- break
- # Update the pointer to what we've already printed
- out_size = len(child.before)
- except KeyboardInterrupt:
- # We need to send ^C to the process. The ascii code for '^C' is 3
- # (the character is known as ETX for 'End of Text', see
- # curses.ascii.ETX).
- child.sendline(chr(3))
- # Read and print any more output the program might produce on its
- # way out.
- try:
- out_size = len(child.before)
- child.expect_list(patterns, self.terminate_timeout)
- print(child.before[out_size:].decode(enc, 'replace'), end='')
- sys.stdout.flush()
- except KeyboardInterrupt:
- # Impatient users tend to type it multiple times
- pass
- finally:
- # Ensure the subprocess really is terminated
- child.terminate(force=True)
- # add isalive check, to ensure exitstatus is set:
- child.isalive()
-
- # We follow the subprocess pattern, returning either the exit status
- # as a positive number, or the terminating signal as a negative
- # number.
- # on Linux, sh returns 128+n for signals terminating child processes on Linux
- # on BSD (OS X), the signal code is set instead
- if child.exitstatus is None:
- # on WIFSIGNALED, pexpect sets signalstatus, leaving exitstatus=None
- if child.signalstatus is None:
- # this condition may never occur,
- # but let's be certain we always return an integer.
- return 0
- return -child.signalstatus
- if child.exitstatus > 128:
- return -(child.exitstatus - 128)
- return child.exitstatus
-
-
-# Make system() with a functional interface for outside use. Note that we use
-# getoutput() from the _common utils, which is built on top of popen(). Using
-# pexpect to get subprocess output produces difficult to parse output, since
-# programs think they are talking to a tty and produce highly formatted output
-# (ls is a good example) that makes them hard.
-system = ProcessHandler().system
-
-def check_pid(pid):
- try:
- os.kill(pid, 0)
- except OSError as err:
- if err.errno == errno.ESRCH:
- return False
- elif err.errno == errno.EPERM:
- # Don't have permission to signal the process - probably means it exists
- return True
- raise
- else:
- return True
+"""Posix-specific implementation of process utilities.
+
+This file is only meant to be imported by process.py, not by end-users.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib
+import errno
+import os
+import subprocess as sp
+import sys
+
+import pexpect
+
+# Our own
+from ._process_common import getoutput, arg_split
+from IPython.utils import py3compat
+from IPython.utils.encoding import DEFAULT_ENCODING
+
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+def _find_cmd(cmd):
+ """Find the full path to a command using which."""
+
+ path = sp.Popen(['/usr/bin/env', 'which', cmd],
+ stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
+ return py3compat.bytes_to_str(path)
+
+
+class ProcessHandler(object):
+ """Execute subprocesses under the control of pexpect.
+ """
+ # Timeout in seconds to wait on each reading of the subprocess' output.
+ # This should not be set too low to avoid cpu overusage from our side,
+ # since we read in a loop whose period is controlled by this timeout.
+ read_timeout = 0.05
+
+ # Timeout to give a process if we receive SIGINT, between sending the
+ # SIGINT to the process and forcefully terminating it.
+ terminate_timeout = 0.2
+
+ # File object where stdout and stderr of the subprocess will be written
+ logfile = None
+
+ # Shell to call for subprocesses to execute
+ _sh = None
+
+ @property
+ def sh(self):
+ if self._sh is None:
+ self._sh = pexpect.which('sh')
+ if self._sh is None:
+ raise OSError('"sh" shell not found')
+
+ return self._sh
+
+ def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
+ """Arguments are used for pexpect calls."""
+ self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
+ None else read_timeout)
+ self.terminate_timeout = (ProcessHandler.terminate_timeout if
+ terminate_timeout is None else
+ terminate_timeout)
+ self.logfile = sys.stdout if logfile is None else logfile
+
+ def getoutput(self, cmd):
+ """Run a command and return its stdout/stderr as a string.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ output : str
+ A string containing the combination of stdout and stderr from the
+ subprocess, in whatever order the subprocess originally wrote to its
+ file descriptors (so the order of the information in this string is the
+ correct order as would be seen if running the command in a terminal).
+ """
+ try:
+ return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
+ except KeyboardInterrupt:
+ print('^C', file=sys.stderr, end='')
+
+ def getoutput_pexpect(self, cmd):
+ """Run a command and return its stdout/stderr as a string.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ output : str
+ A string containing the combination of stdout and stderr from the
+ subprocess, in whatever order the subprocess originally wrote to its
+ file descriptors (so the order of the information in this string is the
+ correct order as would be seen if running the command in a terminal).
+ """
+ try:
+ return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
+ except KeyboardInterrupt:
+ print('^C', file=sys.stderr, end='')
+
+ def system(self, cmd):
+ """Execute a command in a subshell.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ int : child's exitstatus
+ """
+ # Get likely encoding for the output.
+ enc = DEFAULT_ENCODING
+
+ # Patterns to match on the output, for pexpect. We read input and
+ # allow either a short timeout or EOF
+ patterns = [pexpect.TIMEOUT, pexpect.EOF]
+ # the index of the EOF pattern in the list.
+ # even though we know it's 1, this call means we don't have to worry if
+ # we change the above list, and forget to change this value:
+ EOF_index = patterns.index(pexpect.EOF)
+ # The size of the output stored so far in the process output buffer.
+ # Since pexpect only appends to this buffer, each time we print we
+ # record how far we've printed, so that next time we only print *new*
+ # content from the buffer.
+ out_size = 0
+ try:
+ # Since we're not really searching the buffer for text patterns, we
+ # can set pexpect's search window to be tiny and it won't matter.
+ # We only search for the 'patterns' timeout or EOF, which aren't in
+ # the text itself.
+ #child = pexpect.spawn(pcmd, searchwindowsize=1)
+ if hasattr(pexpect, 'spawnb'):
+ child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
+ else:
+ child = pexpect.spawn(self.sh, args=['-c', cmd]) # Vanilla Pexpect
+ flush = sys.stdout.flush
+ while True:
+ # res is the index of the pattern that caused the match, so we
+ # know whether we've finished (if we matched EOF) or not
+ res_idx = child.expect_list(patterns, self.read_timeout)
+ print(child.before[out_size:].decode(enc, 'replace'), end='')
+ flush()
+ if res_idx==EOF_index:
+ break
+ # Update the pointer to what we've already printed
+ out_size = len(child.before)
+ except KeyboardInterrupt:
+ # We need to send ^C to the process. The ascii code for '^C' is 3
+ # (the character is known as ETX for 'End of Text', see
+ # curses.ascii.ETX).
+ child.sendline(chr(3))
+ # Read and print any more output the program might produce on its
+ # way out.
+ try:
+ out_size = len(child.before)
+ child.expect_list(patterns, self.terminate_timeout)
+ print(child.before[out_size:].decode(enc, 'replace'), end='')
+ sys.stdout.flush()
+ except KeyboardInterrupt:
+ # Impatient users tend to type it multiple times
+ pass
+ finally:
+ # Ensure the subprocess really is terminated
+ child.terminate(force=True)
+ # add isalive check, to ensure exitstatus is set:
+ child.isalive()
+
+ # We follow the subprocess pattern, returning either the exit status
+ # as a positive number, or the terminating signal as a negative
+ # number.
+ # on Linux, sh returns 128+n for signals terminating child processes on Linux
+ # on BSD (OS X), the signal code is set instead
+ if child.exitstatus is None:
+ # on WIFSIGNALED, pexpect sets signalstatus, leaving exitstatus=None
+ if child.signalstatus is None:
+ # this condition may never occur,
+ # but let's be certain we always return an integer.
+ return 0
+ return -child.signalstatus
+ if child.exitstatus > 128:
+ return -(child.exitstatus - 128)
+ return child.exitstatus
+
+
+# Make system() with a functional interface for outside use. Note that we use
+# getoutput() from the _common utils, which is built on top of popen(). Using
+# pexpect to get subprocess output produces difficult to parse output, since
+# programs think they are talking to a tty and produce highly formatted output
+# (ls is a good example) that makes them hard.
+system = ProcessHandler().system
+
+def check_pid(pid):
+ try:
+ os.kill(pid, 0)
+ except OSError as err:
+ if err.errno == errno.ESRCH:
+ return False
+ elif err.errno == errno.EPERM:
+ # Don't have permission to signal the process - probably means it exists
+ return True
+ raise
+ else:
+ return True
diff --git a/contrib/python/ipython/py2/IPython/utils/_process_win32.py b/contrib/python/ipython/py2/IPython/utils/_process_win32.py
index 3ac59b2c29..6d7d0f4197 100644
--- a/contrib/python/ipython/py2/IPython/utils/_process_win32.py
+++ b/contrib/python/ipython/py2/IPython/utils/_process_win32.py
@@ -1,192 +1,192 @@
-"""Windows-specific implementation of process utilities.
-
-This file is only meant to be imported by process.py, not by end-users.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2010-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
-
-# stdlib
-import os
-import sys
-import ctypes
-
-from ctypes import c_int, POINTER
-from ctypes.wintypes import LPCWSTR, HLOCAL
-from subprocess import STDOUT
-
-# our own imports
-from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
-from . import py3compat
-from .encoding import DEFAULT_ENCODING
-
-#-----------------------------------------------------------------------------
-# Function definitions
-#-----------------------------------------------------------------------------
-
-class AvoidUNCPath(object):
- """A context manager to protect command execution from UNC paths.
-
- In the Win32 API, commands can't be invoked with the cwd being a UNC path.
- This context manager temporarily changes directory to the 'C:' drive on
- entering, and restores the original working directory on exit.
-
- The context manager returns the starting working directory *if* it made a
- change and None otherwise, so that users can apply the necessary adjustment
- to their system calls in the event of a change.
-
- Examples
- --------
- ::
- cmd = 'dir'
- with AvoidUNCPath() as path:
- if path is not None:
- cmd = '"pushd %s &&"%s' % (path, cmd)
- os.system(cmd)
- """
- def __enter__(self):
- self.path = py3compat.getcwd()
- self.is_unc_path = self.path.startswith(r"\\")
- if self.is_unc_path:
- # change to c drive (as cmd.exe cannot handle UNC addresses)
- os.chdir("C:")
- return self.path
- else:
- # We return None to signal that there was no change in the working
- # directory
- return None
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.is_unc_path:
- os.chdir(self.path)
-
-
-def _find_cmd(cmd):
- """Find the full path to a .bat or .exe using the win32api module."""
- try:
- from win32api import SearchPath
- except ImportError:
- raise ImportError('you need to have pywin32 installed for this to work')
- else:
- PATH = os.environ['PATH']
- extensions = ['.exe', '.com', '.bat', '.py']
- path = None
- for ext in extensions:
- try:
- path = SearchPath(PATH, cmd, ext)[0]
- except:
- pass
- if path is None:
- raise OSError("command %r not found" % cmd)
- else:
- return path
-
-
-def _system_body(p):
- """Callback for _system."""
- enc = DEFAULT_ENCODING
- for line in read_no_interrupt(p.stdout).splitlines():
- line = line.decode(enc, 'replace')
- print(line, file=sys.stdout)
- for line in read_no_interrupt(p.stderr).splitlines():
- line = line.decode(enc, 'replace')
- print(line, file=sys.stderr)
-
- # Wait to finish for returncode
- return p.wait()
-
-
-def system(cmd):
- """Win32 version of os.system() that works with network shares.
-
- Note that this implementation returns None, as meant for use in IPython.
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed in the system shell.
-
- Returns
- -------
- None : we explicitly do NOT return the subprocess status code, as this
- utility is meant to be used extensively in IPython, where any return value
- would trigger :func:`sys.displayhook` calls.
- """
- # The controller provides interactivity with both
- # stdin and stdout
- #import _process_win32_controller
- #_process_win32_controller.system(cmd)
-
- with AvoidUNCPath() as path:
- if path is not None:
- cmd = '"pushd %s &&"%s' % (path, cmd)
- return process_handler(cmd, _system_body)
-
-def getoutput(cmd):
- """Return standard output of executing cmd in a shell.
-
- Accepts the same arguments as os.system().
-
- Parameters
- ----------
- cmd : str or list
- A command to be executed in the system shell.
-
- Returns
- -------
- stdout : str
- """
-
- with AvoidUNCPath() as path:
- if path is not None:
- cmd = '"pushd %s &&"%s' % (path, cmd)
- out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
-
- if out is None:
- out = b''
- return py3compat.bytes_to_str(out)
-
-try:
- CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
- CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
- CommandLineToArgvW.restype = POINTER(LPCWSTR)
- LocalFree = ctypes.windll.kernel32.LocalFree
- LocalFree.res_type = HLOCAL
- LocalFree.arg_types = [HLOCAL]
-
- def arg_split(commandline, posix=False, strict=True):
- """Split a command line's arguments in a shell-like manner.
-
- This is a special version for windows that use a ctypes call to CommandLineToArgvW
- to do the argv splitting. The posix paramter is ignored.
-
- If strict=False, process_common.arg_split(...strict=False) is used instead.
- """
- #CommandLineToArgvW returns path to executable if called with empty string.
- if commandline.strip() == "":
- return []
- if not strict:
- # not really a cl-arg, fallback on _process_common
- return py_arg_split(commandline, posix=posix, strict=strict)
- argvn = c_int()
- result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
- result_array_type = LPCWSTR * argvn.value
- result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
- retval = LocalFree(result_pointer)
- return result
-except AttributeError:
- arg_split = py_arg_split
-
-def check_pid(pid):
- # OpenProcess returns 0 if no such process (of ours) exists
- # positive int otherwise
- return bool(ctypes.windll.kernel32.OpenProcess(1,0,pid))
+"""Windows-specific implementation of process utilities.
+
+This file is only meant to be imported by process.py, not by end-users.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# stdlib
+import os
+import sys
+import ctypes
+
+from ctypes import c_int, POINTER
+from ctypes.wintypes import LPCWSTR, HLOCAL
+from subprocess import STDOUT
+
+# our own imports
+from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
+from . import py3compat
+from .encoding import DEFAULT_ENCODING
+
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+class AvoidUNCPath(object):
+ """A context manager to protect command execution from UNC paths.
+
+ In the Win32 API, commands can't be invoked with the cwd being a UNC path.
+ This context manager temporarily changes directory to the 'C:' drive on
+ entering, and restores the original working directory on exit.
+
+ The context manager returns the starting working directory *if* it made a
+ change and None otherwise, so that users can apply the necessary adjustment
+ to their system calls in the event of a change.
+
+ Examples
+ --------
+ ::
+ cmd = 'dir'
+ with AvoidUNCPath() as path:
+ if path is not None:
+ cmd = '"pushd %s &&"%s' % (path, cmd)
+ os.system(cmd)
+ """
+ def __enter__(self):
+ self.path = py3compat.getcwd()
+ self.is_unc_path = self.path.startswith(r"\\")
+ if self.is_unc_path:
+ # change to c drive (as cmd.exe cannot handle UNC addresses)
+ os.chdir("C:")
+ return self.path
+ else:
+ # We return None to signal that there was no change in the working
+ # directory
+ return None
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.is_unc_path:
+ os.chdir(self.path)
+
+
+def _find_cmd(cmd):
+ """Find the full path to a .bat or .exe using the win32api module."""
+ try:
+ from win32api import SearchPath
+ except ImportError:
+ raise ImportError('you need to have pywin32 installed for this to work')
+ else:
+ PATH = os.environ['PATH']
+ extensions = ['.exe', '.com', '.bat', '.py']
+ path = None
+ for ext in extensions:
+ try:
+ path = SearchPath(PATH, cmd, ext)[0]
+ except:
+ pass
+ if path is None:
+ raise OSError("command %r not found" % cmd)
+ else:
+ return path
+
+
+def _system_body(p):
+ """Callback for _system."""
+ enc = DEFAULT_ENCODING
+ for line in read_no_interrupt(p.stdout).splitlines():
+ line = line.decode(enc, 'replace')
+ print(line, file=sys.stdout)
+ for line in read_no_interrupt(p.stderr).splitlines():
+ line = line.decode(enc, 'replace')
+ print(line, file=sys.stderr)
+
+ # Wait to finish for returncode
+ return p.wait()
+
+
+def system(cmd):
+ """Win32 version of os.system() that works with network shares.
+
+ Note that this implementation returns None, as meant for use in IPython.
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ None : we explicitly do NOT return the subprocess status code, as this
+ utility is meant to be used extensively in IPython, where any return value
+ would trigger :func:`sys.displayhook` calls.
+ """
+ # The controller provides interactivity with both
+ # stdin and stdout
+ #import _process_win32_controller
+ #_process_win32_controller.system(cmd)
+
+ with AvoidUNCPath() as path:
+ if path is not None:
+ cmd = '"pushd %s &&"%s' % (path, cmd)
+ return process_handler(cmd, _system_body)
+
+def getoutput(cmd):
+ """Return standard output of executing cmd in a shell.
+
+ Accepts the same arguments as os.system().
+
+ Parameters
+ ----------
+ cmd : str or list
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ stdout : str
+ """
+
+ with AvoidUNCPath() as path:
+ if path is not None:
+ cmd = '"pushd %s &&"%s' % (path, cmd)
+ out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
+
+ if out is None:
+ out = b''
+ return py3compat.bytes_to_str(out)
+
+try:
+ CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
+ CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
+ CommandLineToArgvW.restype = POINTER(LPCWSTR)
+ LocalFree = ctypes.windll.kernel32.LocalFree
+ LocalFree.res_type = HLOCAL
+ LocalFree.arg_types = [HLOCAL]
+
+ def arg_split(commandline, posix=False, strict=True):
+ """Split a command line's arguments in a shell-like manner.
+
+ This is a special version for windows that use a ctypes call to CommandLineToArgvW
+ to do the argv splitting. The posix paramter is ignored.
+
+ If strict=False, process_common.arg_split(...strict=False) is used instead.
+ """
+ #CommandLineToArgvW returns path to executable if called with empty string.
+ if commandline.strip() == "":
+ return []
+ if not strict:
+ # not really a cl-arg, fallback on _process_common
+ return py_arg_split(commandline, posix=posix, strict=strict)
+ argvn = c_int()
+ result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
+ result_array_type = LPCWSTR * argvn.value
+ result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
+ retval = LocalFree(result_pointer)
+ return result
+except AttributeError:
+ arg_split = py_arg_split
+
+def check_pid(pid):
+ # OpenProcess returns 0 if no such process (of ours) exists
+ # positive int otherwise
+ return bool(ctypes.windll.kernel32.OpenProcess(1,0,pid))
diff --git a/contrib/python/ipython/py2/IPython/utils/_process_win32_controller.py b/contrib/python/ipython/py2/IPython/utils/_process_win32_controller.py
index 555eec23b3..607e411916 100644
--- a/contrib/python/ipython/py2/IPython/utils/_process_win32_controller.py
+++ b/contrib/python/ipython/py2/IPython/utils/_process_win32_controller.py
@@ -1,577 +1,577 @@
-"""Windows-specific implementation of process utilities with direct WinAPI.
-
-This file is meant to be used by process.py
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2010-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-from __future__ import print_function
-
-# stdlib
-import os, sys, threading
-import ctypes, msvcrt
-
-# local imports
-from . import py3compat
-
-# Win32 API types needed for the API calls
-from ctypes import POINTER
-from ctypes.wintypes import HANDLE, HLOCAL, LPVOID, WORD, DWORD, BOOL, \
- ULONG, LPCWSTR
-LPDWORD = POINTER(DWORD)
-LPHANDLE = POINTER(HANDLE)
-ULONG_PTR = POINTER(ULONG)
-class SECURITY_ATTRIBUTES(ctypes.Structure):
- _fields_ = [("nLength", DWORD),
- ("lpSecurityDescriptor", LPVOID),
- ("bInheritHandle", BOOL)]
-LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
-class STARTUPINFO(ctypes.Structure):
- _fields_ = [("cb", DWORD),
- ("lpReserved", LPCWSTR),
- ("lpDesktop", LPCWSTR),
- ("lpTitle", LPCWSTR),
- ("dwX", DWORD),
- ("dwY", DWORD),
- ("dwXSize", DWORD),
- ("dwYSize", DWORD),
- ("dwXCountChars", DWORD),
- ("dwYCountChars", DWORD),
- ("dwFillAttribute", DWORD),
- ("dwFlags", DWORD),
- ("wShowWindow", WORD),
- ("cbReserved2", WORD),
- ("lpReserved2", LPVOID),
- ("hStdInput", HANDLE),
- ("hStdOutput", HANDLE),
- ("hStdError", HANDLE)]
-LPSTARTUPINFO = POINTER(STARTUPINFO)
-class PROCESS_INFORMATION(ctypes.Structure):
- _fields_ = [("hProcess", HANDLE),
- ("hThread", HANDLE),
- ("dwProcessId", DWORD),
- ("dwThreadId", DWORD)]
-LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
-
-# Win32 API constants needed
-ERROR_HANDLE_EOF = 38
-ERROR_BROKEN_PIPE = 109
-ERROR_NO_DATA = 232
-HANDLE_FLAG_INHERIT = 0x0001
-STARTF_USESTDHANDLES = 0x0100
-CREATE_SUSPENDED = 0x0004
-CREATE_NEW_CONSOLE = 0x0010
-CREATE_NO_WINDOW = 0x08000000
-STILL_ACTIVE = 259
-WAIT_TIMEOUT = 0x0102
-WAIT_FAILED = 0xFFFFFFFF
-INFINITE = 0xFFFFFFFF
-DUPLICATE_SAME_ACCESS = 0x00000002
-ENABLE_ECHO_INPUT = 0x0004
-ENABLE_LINE_INPUT = 0x0002
-ENABLE_PROCESSED_INPUT = 0x0001
-
-# Win32 API functions needed
-GetLastError = ctypes.windll.kernel32.GetLastError
-GetLastError.argtypes = []
-GetLastError.restype = DWORD
-
-CreateFile = ctypes.windll.kernel32.CreateFileW
-CreateFile.argtypes = [LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE]
-CreateFile.restype = HANDLE
-
-CreatePipe = ctypes.windll.kernel32.CreatePipe
-CreatePipe.argtypes = [POINTER(HANDLE), POINTER(HANDLE),
- LPSECURITY_ATTRIBUTES, DWORD]
-CreatePipe.restype = BOOL
-
-CreateProcess = ctypes.windll.kernel32.CreateProcessW
-CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
- LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
- LPPROCESS_INFORMATION]
-CreateProcess.restype = BOOL
-
-GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
-GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
-GetExitCodeProcess.restype = BOOL
-
-GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
-GetCurrentProcess.argtypes = []
-GetCurrentProcess.restype = HANDLE
-
-ResumeThread = ctypes.windll.kernel32.ResumeThread
-ResumeThread.argtypes = [HANDLE]
-ResumeThread.restype = DWORD
-
-ReadFile = ctypes.windll.kernel32.ReadFile
-ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
-ReadFile.restype = BOOL
-
-WriteFile = ctypes.windll.kernel32.WriteFile
-WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
-WriteFile.restype = BOOL
-
-GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
-GetConsoleMode.argtypes = [HANDLE, LPDWORD]
-GetConsoleMode.restype = BOOL
-
-SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
-SetConsoleMode.argtypes = [HANDLE, DWORD]
-SetConsoleMode.restype = BOOL
-
-FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
-FlushConsoleInputBuffer.argtypes = [HANDLE]
-FlushConsoleInputBuffer.restype = BOOL
-
-WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
-WaitForSingleObject.argtypes = [HANDLE, DWORD]
-WaitForSingleObject.restype = DWORD
-
-DuplicateHandle = ctypes.windll.kernel32.DuplicateHandle
-DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, LPHANDLE,
- DWORD, BOOL, DWORD]
-DuplicateHandle.restype = BOOL
-
-SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
-SetHandleInformation.argtypes = [HANDLE, DWORD, DWORD]
-SetHandleInformation.restype = BOOL
-
-CloseHandle = ctypes.windll.kernel32.CloseHandle
-CloseHandle.argtypes = [HANDLE]
-CloseHandle.restype = BOOL
-
-CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
-CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(ctypes.c_int)]
-CommandLineToArgvW.restype = POINTER(LPCWSTR)
-
-LocalFree = ctypes.windll.kernel32.LocalFree
-LocalFree.argtypes = [HLOCAL]
-LocalFree.restype = HLOCAL
-
-class AvoidUNCPath(object):
- """A context manager to protect command execution from UNC paths.
-
- In the Win32 API, commands can't be invoked with the cwd being a UNC path.
- This context manager temporarily changes directory to the 'C:' drive on
- entering, and restores the original working directory on exit.
-
- The context manager returns the starting working directory *if* it made a
- change and None otherwise, so that users can apply the necessary adjustment
- to their system calls in the event of a change.
-
- Examples
- --------
- ::
- cmd = 'dir'
- with AvoidUNCPath() as path:
- if path is not None:
- cmd = '"pushd %s &&"%s' % (path, cmd)
- os.system(cmd)
- """
- def __enter__(self):
- self.path = py3compat.getcwd()
- self.is_unc_path = self.path.startswith(r"\\")
- if self.is_unc_path:
- # change to c drive (as cmd.exe cannot handle UNC addresses)
- os.chdir("C:")
- return self.path
- else:
- # We return None to signal that there was no change in the working
- # directory
- return None
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.is_unc_path:
- os.chdir(self.path)
-
-
-class Win32ShellCommandController(object):
- """Runs a shell command in a 'with' context.
-
- This implementation is Win32-specific.
-
- Example:
- # Runs the command interactively with default console stdin/stdout
- with ShellCommandController('python -i') as scc:
- scc.run()
-
- # Runs the command using the provided functions for stdin/stdout
- def my_stdout_func(s):
- # print or save the string 's'
- write_to_stdout(s)
- def my_stdin_func():
- # If input is available, return it as a string.
- if input_available():
- return get_input()
- # If no input available, return None after a short delay to
- # keep from blocking.
- else:
- time.sleep(0.01)
- return None
-
- with ShellCommandController('python -i') as scc:
- scc.run(my_stdout_func, my_stdin_func)
- """
-
- def __init__(self, cmd, mergeout = True):
- """Initializes the shell command controller.
-
- The cmd is the program to execute, and mergeout is
- whether to blend stdout and stderr into one output
- in stdout. Merging them together in this fashion more
- reliably keeps stdout and stderr in the correct order
- especially for interactive shell usage.
- """
- self.cmd = cmd
- self.mergeout = mergeout
-
- def __enter__(self):
- cmd = self.cmd
- mergeout = self.mergeout
-
- self.hstdout, self.hstdin, self.hstderr = None, None, None
- self.piProcInfo = None
- try:
- p_hstdout, c_hstdout, p_hstderr, \
- c_hstderr, p_hstdin, c_hstdin = [None]*6
-
- # SECURITY_ATTRIBUTES with inherit handle set to True
- saAttr = SECURITY_ATTRIBUTES()
- saAttr.nLength = ctypes.sizeof(saAttr)
- saAttr.bInheritHandle = True
- saAttr.lpSecurityDescriptor = None
-
- def create_pipe(uninherit):
- """Creates a Windows pipe, which consists of two handles.
-
- The 'uninherit' parameter controls which handle is not
- inherited by the child process.
- """
- handles = HANDLE(), HANDLE()
- if not CreatePipe(ctypes.byref(handles[0]),
- ctypes.byref(handles[1]), ctypes.byref(saAttr), 0):
- raise ctypes.WinError()
- if not SetHandleInformation(handles[uninherit],
- HANDLE_FLAG_INHERIT, 0):
- raise ctypes.WinError()
- return handles[0].value, handles[1].value
-
- p_hstdout, c_hstdout = create_pipe(uninherit=0)
- # 'mergeout' signals that stdout and stderr should be merged.
- # We do that by using one pipe for both of them.
- if mergeout:
- c_hstderr = HANDLE()
- if not DuplicateHandle(GetCurrentProcess(), c_hstdout,
- GetCurrentProcess(), ctypes.byref(c_hstderr),
- 0, True, DUPLICATE_SAME_ACCESS):
- raise ctypes.WinError()
- else:
- p_hstderr, c_hstderr = create_pipe(uninherit=0)
- c_hstdin, p_hstdin = create_pipe(uninherit=1)
-
- # Create the process object
- piProcInfo = PROCESS_INFORMATION()
- siStartInfo = STARTUPINFO()
- siStartInfo.cb = ctypes.sizeof(siStartInfo)
- siStartInfo.hStdInput = c_hstdin
- siStartInfo.hStdOutput = c_hstdout
- siStartInfo.hStdError = c_hstderr
- siStartInfo.dwFlags = STARTF_USESTDHANDLES
- dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
-
- if not CreateProcess(None,
- u"cmd.exe /c " + cmd,
- None, None, True, dwCreationFlags,
- None, None, ctypes.byref(siStartInfo),
- ctypes.byref(piProcInfo)):
- raise ctypes.WinError()
-
- # Close this process's versions of the child handles
- CloseHandle(c_hstdin)
- c_hstdin = None
- CloseHandle(c_hstdout)
- c_hstdout = None
- if c_hstderr is not None:
- CloseHandle(c_hstderr)
- c_hstderr = None
-
- # Transfer ownership of the parent handles to the object
- self.hstdin = p_hstdin
- p_hstdin = None
- self.hstdout = p_hstdout
- p_hstdout = None
- if not mergeout:
- self.hstderr = p_hstderr
- p_hstderr = None
- self.piProcInfo = piProcInfo
-
- finally:
- if p_hstdin:
- CloseHandle(p_hstdin)
- if c_hstdin:
- CloseHandle(c_hstdin)
- if p_hstdout:
- CloseHandle(p_hstdout)
- if c_hstdout:
- CloseHandle(c_hstdout)
- if p_hstderr:
- CloseHandle(p_hstderr)
- if c_hstderr:
- CloseHandle(c_hstderr)
-
- return self
-
- def _stdin_thread(self, handle, hprocess, func, stdout_func):
- exitCode = DWORD()
- bytesWritten = DWORD(0)
- while True:
- #print("stdin thread loop start")
- # Get the input string (may be bytes or unicode)
- data = func()
-
- # None signals to poll whether the process has exited
- if data is None:
- #print("checking for process completion")
- if not GetExitCodeProcess(hprocess, ctypes.byref(exitCode)):
- raise ctypes.WinError()
- if exitCode.value != STILL_ACTIVE:
- return
- # TESTING: Does zero-sized writefile help?
- if not WriteFile(handle, "", 0,
- ctypes.byref(bytesWritten), None):
- raise ctypes.WinError()
- continue
- #print("\nGot str %s\n" % repr(data), file=sys.stderr)
-
- # Encode the string to the console encoding
- if isinstance(data, unicode): #FIXME: Python3
- data = data.encode('utf_8')
-
- # What we have now must be a string of bytes
- if not isinstance(data, str): #FIXME: Python3
- raise RuntimeError("internal stdin function string error")
-
- # An empty string signals EOF
- if len(data) == 0:
- return
-
- # In a windows console, sometimes the input is echoed,
- # but sometimes not. How do we determine when to do this?
- stdout_func(data)
- # WriteFile may not accept all the data at once.
- # Loop until everything is processed
- while len(data) != 0:
- #print("Calling writefile")
- if not WriteFile(handle, data, len(data),
- ctypes.byref(bytesWritten), None):
- # This occurs at exit
- if GetLastError() == ERROR_NO_DATA:
- return
- raise ctypes.WinError()
- #print("Called writefile")
- data = data[bytesWritten.value:]
-
- def _stdout_thread(self, handle, func):
- # Allocate the output buffer
- data = ctypes.create_string_buffer(4096)
- while True:
- bytesRead = DWORD(0)
- if not ReadFile(handle, data, 4096,
- ctypes.byref(bytesRead), None):
- le = GetLastError()
- if le == ERROR_BROKEN_PIPE:
- return
- else:
- raise ctypes.WinError()
- # FIXME: Python3
- s = data.value[0:bytesRead.value]
- #print("\nv: %s" % repr(s), file=sys.stderr)
- func(s.decode('utf_8', 'replace'))
-
- def run(self, stdout_func = None, stdin_func = None, stderr_func = None):
- """Runs the process, using the provided functions for I/O.
-
- The function stdin_func should return strings whenever a
- character or characters become available.
- The functions stdout_func and stderr_func are called whenever
- something is printed to stdout or stderr, respectively.
- These functions are called from different threads (but not
- concurrently, because of the GIL).
- """
- if stdout_func is None and stdin_func is None and stderr_func is None:
- return self._run_stdio()
-
- if stderr_func is not None and self.mergeout:
- raise RuntimeError("Shell command was initiated with "
- "merged stdin/stdout, but a separate stderr_func "
- "was provided to the run() method")
-
- # Create a thread for each input/output handle
- stdin_thread = None
- threads = []
- if stdin_func:
- stdin_thread = threading.Thread(target=self._stdin_thread,
- args=(self.hstdin, self.piProcInfo.hProcess,
- stdin_func, stdout_func))
- threads.append(threading.Thread(target=self._stdout_thread,
- args=(self.hstdout, stdout_func)))
- if not self.mergeout:
- if stderr_func is None:
- stderr_func = stdout_func
- threads.append(threading.Thread(target=self._stdout_thread,
- args=(self.hstderr, stderr_func)))
- # Start the I/O threads and the process
- if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
- raise ctypes.WinError()
- if stdin_thread is not None:
- stdin_thread.start()
- for thread in threads:
- thread.start()
- # Wait for the process to complete
- if WaitForSingleObject(self.piProcInfo.hProcess, INFINITE) == \
- WAIT_FAILED:
- raise ctypes.WinError()
- # Wait for the I/O threads to complete
- for thread in threads:
- thread.join()
-
- # Wait for the stdin thread to complete
- if stdin_thread is not None:
- stdin_thread.join()
-
- def _stdin_raw_nonblock(self):
- """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
- # WARNING: This is experimental, and produces inconsistent results.
- # It's possible for the handle not to be appropriate for use
- # with WaitForSingleObject, among other things.
- handle = msvcrt.get_osfhandle(sys.stdin.fileno())
- result = WaitForSingleObject(handle, 100)
- if result == WAIT_FAILED:
- raise ctypes.WinError()
- elif result == WAIT_TIMEOUT:
- print(".", end='')
- return None
- else:
- data = ctypes.create_string_buffer(256)
- bytesRead = DWORD(0)
- print('?', end='')
-
- if not ReadFile(handle, data, 256,
- ctypes.byref(bytesRead), None):
- raise ctypes.WinError()
- # This ensures the non-blocking works with an actual console
- # Not checking the error, so the processing will still work with
- # other handle types
- FlushConsoleInputBuffer(handle)
-
- data = data.value
- data = data.replace('\r\n', '\n')
- data = data.replace('\r', '\n')
- print(repr(data) + " ", end='')
- return data
-
- def _stdin_raw_block(self):
- """Use a blocking stdin read"""
- # The big problem with the blocking read is that it doesn't
- # exit when it's supposed to in all contexts. An extra
- # key-press may be required to trigger the exit.
- try:
- data = sys.stdin.read(1)
- data = data.replace('\r', '\n')
- return data
- except WindowsError as we:
- if we.winerror == ERROR_NO_DATA:
- # This error occurs when the pipe is closed
- return None
- else:
- # Otherwise let the error propagate
- raise we
-
- def _stdout_raw(self, s):
- """Writes the string to stdout"""
- print(s, end='', file=sys.stdout)
- sys.stdout.flush()
-
- def _stderr_raw(self, s):
- """Writes the string to stdout"""
- print(s, end='', file=sys.stderr)
- sys.stderr.flush()
-
- def _run_stdio(self):
- """Runs the process using the system standard I/O.
-
- IMPORTANT: stdin needs to be asynchronous, so the Python
- sys.stdin object is not used. Instead,
- msvcrt.kbhit/getwch are used asynchronously.
- """
- # Disable Line and Echo mode
- #lpMode = DWORD()
- #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
- #if GetConsoleMode(handle, ctypes.byref(lpMode)):
- # set_console_mode = True
- # if not SetConsoleMode(handle, lpMode.value &
- # ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
- # raise ctypes.WinError()
-
- if self.mergeout:
- return self.run(stdout_func = self._stdout_raw,
- stdin_func = self._stdin_raw_block)
- else:
- return self.run(stdout_func = self._stdout_raw,
- stdin_func = self._stdin_raw_block,
- stderr_func = self._stderr_raw)
-
- # Restore the previous console mode
- #if set_console_mode:
- # if not SetConsoleMode(handle, lpMode.value):
- # raise ctypes.WinError()
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.hstdin:
- CloseHandle(self.hstdin)
- self.hstdin = None
- if self.hstdout:
- CloseHandle(self.hstdout)
- self.hstdout = None
- if self.hstderr:
- CloseHandle(self.hstderr)
- self.hstderr = None
- if self.piProcInfo is not None:
- CloseHandle(self.piProcInfo.hProcess)
- CloseHandle(self.piProcInfo.hThread)
- self.piProcInfo = None
-
-
-def system(cmd):
- """Win32 version of os.system() that works with network shares.
-
- Note that this implementation returns None, as meant for use in IPython.
-
- Parameters
- ----------
- cmd : str
- A command to be executed in the system shell.
-
- Returns
- -------
- None : we explicitly do NOT return the subprocess status code, as this
- utility is meant to be used extensively in IPython, where any return value
- would trigger :func:`sys.displayhook` calls.
- """
- with AvoidUNCPath() as path:
- if path is not None:
- cmd = '"pushd %s &&"%s' % (path, cmd)
- with Win32ShellCommandController(cmd) as scc:
- scc.run()
-
-
-if __name__ == "__main__":
- print("Test starting!")
- #system("cmd")
- system("python -i")
- print("Test finished!")
+"""Windows-specific implementation of process utilities with direct WinAPI.
+
+This file is meant to be used by process.py
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+# stdlib
+import os, sys, threading
+import ctypes, msvcrt
+
+# local imports
+from . import py3compat
+
+# Win32 API types needed for the API calls
+from ctypes import POINTER
+from ctypes.wintypes import HANDLE, HLOCAL, LPVOID, WORD, DWORD, BOOL, \
+ ULONG, LPCWSTR
+LPDWORD = POINTER(DWORD)
+LPHANDLE = POINTER(HANDLE)
+ULONG_PTR = POINTER(ULONG)
+class SECURITY_ATTRIBUTES(ctypes.Structure):
+ _fields_ = [("nLength", DWORD),
+ ("lpSecurityDescriptor", LPVOID),
+ ("bInheritHandle", BOOL)]
+LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
+class STARTUPINFO(ctypes.Structure):
+ _fields_ = [("cb", DWORD),
+ ("lpReserved", LPCWSTR),
+ ("lpDesktop", LPCWSTR),
+ ("lpTitle", LPCWSTR),
+ ("dwX", DWORD),
+ ("dwY", DWORD),
+ ("dwXSize", DWORD),
+ ("dwYSize", DWORD),
+ ("dwXCountChars", DWORD),
+ ("dwYCountChars", DWORD),
+ ("dwFillAttribute", DWORD),
+ ("dwFlags", DWORD),
+ ("wShowWindow", WORD),
+ ("cbReserved2", WORD),
+ ("lpReserved2", LPVOID),
+ ("hStdInput", HANDLE),
+ ("hStdOutput", HANDLE),
+ ("hStdError", HANDLE)]
+LPSTARTUPINFO = POINTER(STARTUPINFO)
+class PROCESS_INFORMATION(ctypes.Structure):
+ _fields_ = [("hProcess", HANDLE),
+ ("hThread", HANDLE),
+ ("dwProcessId", DWORD),
+ ("dwThreadId", DWORD)]
+LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
+
+# Win32 API constants needed
+ERROR_HANDLE_EOF = 38
+ERROR_BROKEN_PIPE = 109
+ERROR_NO_DATA = 232
+HANDLE_FLAG_INHERIT = 0x0001
+STARTF_USESTDHANDLES = 0x0100
+CREATE_SUSPENDED = 0x0004
+CREATE_NEW_CONSOLE = 0x0010
+CREATE_NO_WINDOW = 0x08000000
+STILL_ACTIVE = 259
+WAIT_TIMEOUT = 0x0102
+WAIT_FAILED = 0xFFFFFFFF
+INFINITE = 0xFFFFFFFF
+DUPLICATE_SAME_ACCESS = 0x00000002
+ENABLE_ECHO_INPUT = 0x0004
+ENABLE_LINE_INPUT = 0x0002
+ENABLE_PROCESSED_INPUT = 0x0001
+
+# Win32 API functions needed
+GetLastError = ctypes.windll.kernel32.GetLastError
+GetLastError.argtypes = []
+GetLastError.restype = DWORD
+
+CreateFile = ctypes.windll.kernel32.CreateFileW
+CreateFile.argtypes = [LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE]
+CreateFile.restype = HANDLE
+
+CreatePipe = ctypes.windll.kernel32.CreatePipe
+CreatePipe.argtypes = [POINTER(HANDLE), POINTER(HANDLE),
+ LPSECURITY_ATTRIBUTES, DWORD]
+CreatePipe.restype = BOOL
+
+CreateProcess = ctypes.windll.kernel32.CreateProcessW
+CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
+ LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
+ LPPROCESS_INFORMATION]
+CreateProcess.restype = BOOL
+
+GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
+GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
+GetExitCodeProcess.restype = BOOL
+
+GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
+GetCurrentProcess.argtypes = []
+GetCurrentProcess.restype = HANDLE
+
+ResumeThread = ctypes.windll.kernel32.ResumeThread
+ResumeThread.argtypes = [HANDLE]
+ResumeThread.restype = DWORD
+
+ReadFile = ctypes.windll.kernel32.ReadFile
+ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
+ReadFile.restype = BOOL
+
+WriteFile = ctypes.windll.kernel32.WriteFile
+WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
+WriteFile.restype = BOOL
+
+GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
+GetConsoleMode.argtypes = [HANDLE, LPDWORD]
+GetConsoleMode.restype = BOOL
+
+SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
+SetConsoleMode.argtypes = [HANDLE, DWORD]
+SetConsoleMode.restype = BOOL
+
+FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
+FlushConsoleInputBuffer.argtypes = [HANDLE]
+FlushConsoleInputBuffer.restype = BOOL
+
+WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
+WaitForSingleObject.argtypes = [HANDLE, DWORD]
+WaitForSingleObject.restype = DWORD
+
+DuplicateHandle = ctypes.windll.kernel32.DuplicateHandle
+DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, LPHANDLE,
+ DWORD, BOOL, DWORD]
+DuplicateHandle.restype = BOOL
+
+SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
+SetHandleInformation.argtypes = [HANDLE, DWORD, DWORD]
+SetHandleInformation.restype = BOOL
+
+CloseHandle = ctypes.windll.kernel32.CloseHandle
+CloseHandle.argtypes = [HANDLE]
+CloseHandle.restype = BOOL
+
+CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
+CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(ctypes.c_int)]
+CommandLineToArgvW.restype = POINTER(LPCWSTR)
+
+LocalFree = ctypes.windll.kernel32.LocalFree
+LocalFree.argtypes = [HLOCAL]
+LocalFree.restype = HLOCAL
+
+class AvoidUNCPath(object):
+ """A context manager to protect command execution from UNC paths.
+
+ In the Win32 API, commands can't be invoked with the cwd being a UNC path.
+ This context manager temporarily changes directory to the 'C:' drive on
+ entering, and restores the original working directory on exit.
+
+ The context manager returns the starting working directory *if* it made a
+ change and None otherwise, so that users can apply the necessary adjustment
+ to their system calls in the event of a change.
+
+ Examples
+ --------
+ ::
+ cmd = 'dir'
+ with AvoidUNCPath() as path:
+ if path is not None:
+ cmd = '"pushd %s &&"%s' % (path, cmd)
+ os.system(cmd)
+ """
+ def __enter__(self):
+ self.path = py3compat.getcwd()
+ self.is_unc_path = self.path.startswith(r"\\")
+ if self.is_unc_path:
+ # change to c drive (as cmd.exe cannot handle UNC addresses)
+ os.chdir("C:")
+ return self.path
+ else:
+ # We return None to signal that there was no change in the working
+ # directory
+ return None
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.is_unc_path:
+ os.chdir(self.path)
+
+
+class Win32ShellCommandController(object):
+ """Runs a shell command in a 'with' context.
+
+ This implementation is Win32-specific.
+
+ Example:
+ # Runs the command interactively with default console stdin/stdout
+ with ShellCommandController('python -i') as scc:
+ scc.run()
+
+ # Runs the command using the provided functions for stdin/stdout
+ def my_stdout_func(s):
+ # print or save the string 's'
+ write_to_stdout(s)
+ def my_stdin_func():
+ # If input is available, return it as a string.
+ if input_available():
+ return get_input()
+ # If no input available, return None after a short delay to
+ # keep from blocking.
+ else:
+ time.sleep(0.01)
+ return None
+
+ with ShellCommandController('python -i') as scc:
+ scc.run(my_stdout_func, my_stdin_func)
+ """
+
+ def __init__(self, cmd, mergeout = True):
+ """Initializes the shell command controller.
+
+ The cmd is the program to execute, and mergeout is
+ whether to blend stdout and stderr into one output
+ in stdout. Merging them together in this fashion more
+ reliably keeps stdout and stderr in the correct order
+ especially for interactive shell usage.
+ """
+ self.cmd = cmd
+ self.mergeout = mergeout
+
+ def __enter__(self):
+ cmd = self.cmd
+ mergeout = self.mergeout
+
+ self.hstdout, self.hstdin, self.hstderr = None, None, None
+ self.piProcInfo = None
+ try:
+ p_hstdout, c_hstdout, p_hstderr, \
+ c_hstderr, p_hstdin, c_hstdin = [None]*6
+
+ # SECURITY_ATTRIBUTES with inherit handle set to True
+ saAttr = SECURITY_ATTRIBUTES()
+ saAttr.nLength = ctypes.sizeof(saAttr)
+ saAttr.bInheritHandle = True
+ saAttr.lpSecurityDescriptor = None
+
+ def create_pipe(uninherit):
+ """Creates a Windows pipe, which consists of two handles.
+
+ The 'uninherit' parameter controls which handle is not
+ inherited by the child process.
+ """
+ handles = HANDLE(), HANDLE()
+ if not CreatePipe(ctypes.byref(handles[0]),
+ ctypes.byref(handles[1]), ctypes.byref(saAttr), 0):
+ raise ctypes.WinError()
+ if not SetHandleInformation(handles[uninherit],
+ HANDLE_FLAG_INHERIT, 0):
+ raise ctypes.WinError()
+ return handles[0].value, handles[1].value
+
+ p_hstdout, c_hstdout = create_pipe(uninherit=0)
+ # 'mergeout' signals that stdout and stderr should be merged.
+ # We do that by using one pipe for both of them.
+ if mergeout:
+ c_hstderr = HANDLE()
+ if not DuplicateHandle(GetCurrentProcess(), c_hstdout,
+ GetCurrentProcess(), ctypes.byref(c_hstderr),
+ 0, True, DUPLICATE_SAME_ACCESS):
+ raise ctypes.WinError()
+ else:
+ p_hstderr, c_hstderr = create_pipe(uninherit=0)
+ c_hstdin, p_hstdin = create_pipe(uninherit=1)
+
+ # Create the process object
+ piProcInfo = PROCESS_INFORMATION()
+ siStartInfo = STARTUPINFO()
+ siStartInfo.cb = ctypes.sizeof(siStartInfo)
+ siStartInfo.hStdInput = c_hstdin
+ siStartInfo.hStdOutput = c_hstdout
+ siStartInfo.hStdError = c_hstderr
+ siStartInfo.dwFlags = STARTF_USESTDHANDLES
+ dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
+
+ if not CreateProcess(None,
+ u"cmd.exe /c " + cmd,
+ None, None, True, dwCreationFlags,
+ None, None, ctypes.byref(siStartInfo),
+ ctypes.byref(piProcInfo)):
+ raise ctypes.WinError()
+
+ # Close this process's versions of the child handles
+ CloseHandle(c_hstdin)
+ c_hstdin = None
+ CloseHandle(c_hstdout)
+ c_hstdout = None
+ if c_hstderr is not None:
+ CloseHandle(c_hstderr)
+ c_hstderr = None
+
+ # Transfer ownership of the parent handles to the object
+ self.hstdin = p_hstdin
+ p_hstdin = None
+ self.hstdout = p_hstdout
+ p_hstdout = None
+ if not mergeout:
+ self.hstderr = p_hstderr
+ p_hstderr = None
+ self.piProcInfo = piProcInfo
+
+ finally:
+ if p_hstdin:
+ CloseHandle(p_hstdin)
+ if c_hstdin:
+ CloseHandle(c_hstdin)
+ if p_hstdout:
+ CloseHandle(p_hstdout)
+ if c_hstdout:
+ CloseHandle(c_hstdout)
+ if p_hstderr:
+ CloseHandle(p_hstderr)
+ if c_hstderr:
+ CloseHandle(c_hstderr)
+
+ return self
+
+ def _stdin_thread(self, handle, hprocess, func, stdout_func):
+ exitCode = DWORD()
+ bytesWritten = DWORD(0)
+ while True:
+ #print("stdin thread loop start")
+ # Get the input string (may be bytes or unicode)
+ data = func()
+
+ # None signals to poll whether the process has exited
+ if data is None:
+ #print("checking for process completion")
+ if not GetExitCodeProcess(hprocess, ctypes.byref(exitCode)):
+ raise ctypes.WinError()
+ if exitCode.value != STILL_ACTIVE:
+ return
+ # TESTING: Does zero-sized writefile help?
+ if not WriteFile(handle, "", 0,
+ ctypes.byref(bytesWritten), None):
+ raise ctypes.WinError()
+ continue
+ #print("\nGot str %s\n" % repr(data), file=sys.stderr)
+
+ # Encode the string to the console encoding
+ if isinstance(data, unicode): #FIXME: Python3
+ data = data.encode('utf_8')
+
+ # What we have now must be a string of bytes
+ if not isinstance(data, str): #FIXME: Python3
+ raise RuntimeError("internal stdin function string error")
+
+ # An empty string signals EOF
+ if len(data) == 0:
+ return
+
+ # In a windows console, sometimes the input is echoed,
+ # but sometimes not. How do we determine when to do this?
+ stdout_func(data)
+ # WriteFile may not accept all the data at once.
+ # Loop until everything is processed
+ while len(data) != 0:
+ #print("Calling writefile")
+ if not WriteFile(handle, data, len(data),
+ ctypes.byref(bytesWritten), None):
+ # This occurs at exit
+ if GetLastError() == ERROR_NO_DATA:
+ return
+ raise ctypes.WinError()
+ #print("Called writefile")
+ data = data[bytesWritten.value:]
+
+ def _stdout_thread(self, handle, func):
+ # Allocate the output buffer
+ data = ctypes.create_string_buffer(4096)
+ while True:
+ bytesRead = DWORD(0)
+ if not ReadFile(handle, data, 4096,
+ ctypes.byref(bytesRead), None):
+ le = GetLastError()
+ if le == ERROR_BROKEN_PIPE:
+ return
+ else:
+ raise ctypes.WinError()
+ # FIXME: Python3
+ s = data.value[0:bytesRead.value]
+ #print("\nv: %s" % repr(s), file=sys.stderr)
+ func(s.decode('utf_8', 'replace'))
+
+ def run(self, stdout_func = None, stdin_func = None, stderr_func = None):
+ """Runs the process, using the provided functions for I/O.
+
+ The function stdin_func should return strings whenever a
+ character or characters become available.
+ The functions stdout_func and stderr_func are called whenever
+ something is printed to stdout or stderr, respectively.
+ These functions are called from different threads (but not
+ concurrently, because of the GIL).
+ """
+ if stdout_func is None and stdin_func is None and stderr_func is None:
+ return self._run_stdio()
+
+ if stderr_func is not None and self.mergeout:
+ raise RuntimeError("Shell command was initiated with "
+ "merged stdin/stdout, but a separate stderr_func "
+ "was provided to the run() method")
+
+ # Create a thread for each input/output handle
+ stdin_thread = None
+ threads = []
+ if stdin_func:
+ stdin_thread = threading.Thread(target=self._stdin_thread,
+ args=(self.hstdin, self.piProcInfo.hProcess,
+ stdin_func, stdout_func))
+ threads.append(threading.Thread(target=self._stdout_thread,
+ args=(self.hstdout, stdout_func)))
+ if not self.mergeout:
+ if stderr_func is None:
+ stderr_func = stdout_func
+ threads.append(threading.Thread(target=self._stdout_thread,
+ args=(self.hstderr, stderr_func)))
+ # Start the I/O threads and the process
+ if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
+ raise ctypes.WinError()
+ if stdin_thread is not None:
+ stdin_thread.start()
+ for thread in threads:
+ thread.start()
+ # Wait for the process to complete
+ if WaitForSingleObject(self.piProcInfo.hProcess, INFINITE) == \
+ WAIT_FAILED:
+ raise ctypes.WinError()
+ # Wait for the I/O threads to complete
+ for thread in threads:
+ thread.join()
+
+ # Wait for the stdin thread to complete
+ if stdin_thread is not None:
+ stdin_thread.join()
+
+ def _stdin_raw_nonblock(self):
+ """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
+ # WARNING: This is experimental, and produces inconsistent results.
+ # It's possible for the handle not to be appropriate for use
+ # with WaitForSingleObject, among other things.
+ handle = msvcrt.get_osfhandle(sys.stdin.fileno())
+ result = WaitForSingleObject(handle, 100)
+ if result == WAIT_FAILED:
+ raise ctypes.WinError()
+ elif result == WAIT_TIMEOUT:
+ print(".", end='')
+ return None
+ else:
+ data = ctypes.create_string_buffer(256)
+ bytesRead = DWORD(0)
+ print('?', end='')
+
+ if not ReadFile(handle, data, 256,
+ ctypes.byref(bytesRead), None):
+ raise ctypes.WinError()
+ # This ensures the non-blocking works with an actual console
+ # Not checking the error, so the processing will still work with
+ # other handle types
+ FlushConsoleInputBuffer(handle)
+
+ data = data.value
+ data = data.replace('\r\n', '\n')
+ data = data.replace('\r', '\n')
+ print(repr(data) + " ", end='')
+ return data
+
+ def _stdin_raw_block(self):
+ """Use a blocking stdin read"""
+ # The big problem with the blocking read is that it doesn't
+ # exit when it's supposed to in all contexts. An extra
+ # key-press may be required to trigger the exit.
+ try:
+ data = sys.stdin.read(1)
+ data = data.replace('\r', '\n')
+ return data
+ except WindowsError as we:
+ if we.winerror == ERROR_NO_DATA:
+ # This error occurs when the pipe is closed
+ return None
+ else:
+ # Otherwise let the error propagate
+ raise we
+
+ def _stdout_raw(self, s):
+ """Writes the string to stdout"""
+ print(s, end='', file=sys.stdout)
+ sys.stdout.flush()
+
+ def _stderr_raw(self, s):
+ """Writes the string to stdout"""
+ print(s, end='', file=sys.stderr)
+ sys.stderr.flush()
+
+ def _run_stdio(self):
+ """Runs the process using the system standard I/O.
+
+ IMPORTANT: stdin needs to be asynchronous, so the Python
+ sys.stdin object is not used. Instead,
+ msvcrt.kbhit/getwch are used asynchronously.
+ """
+ # Disable Line and Echo mode
+ #lpMode = DWORD()
+ #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
+ #if GetConsoleMode(handle, ctypes.byref(lpMode)):
+ # set_console_mode = True
+ # if not SetConsoleMode(handle, lpMode.value &
+ # ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
+ # raise ctypes.WinError()
+
+ if self.mergeout:
+ return self.run(stdout_func = self._stdout_raw,
+ stdin_func = self._stdin_raw_block)
+ else:
+ return self.run(stdout_func = self._stdout_raw,
+ stdin_func = self._stdin_raw_block,
+ stderr_func = self._stderr_raw)
+
+ # Restore the previous console mode
+ #if set_console_mode:
+ # if not SetConsoleMode(handle, lpMode.value):
+ # raise ctypes.WinError()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.hstdin:
+ CloseHandle(self.hstdin)
+ self.hstdin = None
+ if self.hstdout:
+ CloseHandle(self.hstdout)
+ self.hstdout = None
+ if self.hstderr:
+ CloseHandle(self.hstderr)
+ self.hstderr = None
+ if self.piProcInfo is not None:
+ CloseHandle(self.piProcInfo.hProcess)
+ CloseHandle(self.piProcInfo.hThread)
+ self.piProcInfo = None
+
+
+def system(cmd):
+ """Win32 version of os.system() that works with network shares.
+
+ Note that this implementation returns None, as meant for use in IPython.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ None : we explicitly do NOT return the subprocess status code, as this
+ utility is meant to be used extensively in IPython, where any return value
+ would trigger :func:`sys.displayhook` calls.
+ """
+ with AvoidUNCPath() as path:
+ if path is not None:
+ cmd = '"pushd %s &&"%s' % (path, cmd)
+ with Win32ShellCommandController(cmd) as scc:
+ scc.run()
+
+
+if __name__ == "__main__":
+ print("Test starting!")
+ #system("cmd")
+ system("python -i")
+ print("Test finished!")
diff --git a/contrib/python/ipython/py2/IPython/utils/_signatures.py b/contrib/python/ipython/py2/IPython/utils/_signatures.py
index 20f52b98ed..9f403618ce 100644
--- a/contrib/python/ipython/py2/IPython/utils/_signatures.py
+++ b/contrib/python/ipython/py2/IPython/utils/_signatures.py
@@ -1,818 +1,818 @@
-"""Function signature objects for callables.
-
-Back port of Python 3.3's function signature tools from the inspect module,
-modified to be compatible with Python 2.7 and 3.2+.
-"""
-
-#-----------------------------------------------------------------------------
-# Python 3.3 stdlib inspect.py is public domain
+"""Function signature objects for callables.
+
+Back port of Python 3.3's function signature tools from the inspect module,
+modified to be compatible with Python 2.7 and 3.2+.
+"""
+
+#-----------------------------------------------------------------------------
+# Python 3.3 stdlib inspect.py is public domain
+#
+# Backports Copyright (C) 2013 Aaron Iles
+# Used under Apache License Version 2.0
#
-# Backports Copyright (C) 2013 Aaron Iles
-# Used under Apache License Version 2.0
-#
-# Further Changes are Copyright (C) 2013 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-from __future__ import absolute_import, division, print_function
-import itertools
-import functools
-import re
-import types
+# Further Changes are Copyright (C) 2013 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import absolute_import, division, print_function
+import itertools
+import functools
+import re
+import types
import inspect
-
-
-# patch for single-file
-# we don't support 2.6, so we can just import OrderedDict
-from collections import OrderedDict
-
-__version__ = '0.3'
-# end patch
-
-__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature']
-
-
-_WrapperDescriptor = type(type.__call__)
-_MethodWrapper = type(all.__call__)
-
-_NonUserDefinedCallables = (_WrapperDescriptor,
- _MethodWrapper,
- types.BuiltinFunctionType)
-
-
-def formatannotation(annotation, base_module=None):
- if isinstance(annotation, type):
- if annotation.__module__ in ('builtins', '__builtin__', base_module):
- return annotation.__name__
- return annotation.__module__+'.'+annotation.__name__
- return repr(annotation)
-
-
-def _get_user_defined_method(cls, method_name, *nested):
- try:
- if cls is type:
- return
- meth = getattr(cls, method_name)
- for name in nested:
- meth = getattr(meth, name, meth)
- except AttributeError:
- return
- else:
- if not isinstance(meth, _NonUserDefinedCallables):
- # Once '__signature__' will be added to 'C'-level
- # callables, this check won't be necessary
- return meth
-
-
-def signature(obj):
- '''Get a signature object for the passed callable.'''
-
- if not callable(obj):
- raise TypeError('{0!r} is not a callable object'.format(obj))
-
+
+
+# patch for single-file
+# we don't support 2.6, so we can just import OrderedDict
+from collections import OrderedDict
+
+__version__ = '0.3'
+# end patch
+
+__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature']
+
+
+_WrapperDescriptor = type(type.__call__)
+_MethodWrapper = type(all.__call__)
+
+_NonUserDefinedCallables = (_WrapperDescriptor,
+ _MethodWrapper,
+ types.BuiltinFunctionType)
+
+
+def formatannotation(annotation, base_module=None):
+ if isinstance(annotation, type):
+ if annotation.__module__ in ('builtins', '__builtin__', base_module):
+ return annotation.__name__
+ return annotation.__module__+'.'+annotation.__name__
+ return repr(annotation)
+
+
+def _get_user_defined_method(cls, method_name, *nested):
+ try:
+ if cls is type:
+ return
+ meth = getattr(cls, method_name)
+ for name in nested:
+ meth = getattr(meth, name, meth)
+ except AttributeError:
+ return
+ else:
+ if not isinstance(meth, _NonUserDefinedCallables):
+ # Once '__signature__' will be added to 'C'-level
+ # callables, this check won't be necessary
+ return meth
+
+
+def signature(obj):
+ '''Get a signature object for the passed callable.'''
+
+ if not callable(obj):
+ raise TypeError('{0!r} is not a callable object'.format(obj))
+
if inspect.ismethod(obj):
- if obj.__self__ is None:
- # Unbound method - treat it as a function (no distinction in Py 3)
- obj = obj.__func__
- else:
- # Bound method: trim off the first parameter (typically self or cls)
- sig = signature(obj.__func__)
- return sig.replace(parameters=tuple(sig.parameters.values())[1:])
-
- try:
- sig = obj.__signature__
- except AttributeError:
- pass
- else:
- if sig is not None:
- return sig
-
- try:
- # Was this function wrapped by a decorator?
- wrapped = obj.__wrapped__
- except AttributeError:
- pass
- else:
- return signature(wrapped)
-
+ if obj.__self__ is None:
+ # Unbound method - treat it as a function (no distinction in Py 3)
+ obj = obj.__func__
+ else:
+ # Bound method: trim off the first parameter (typically self or cls)
+ sig = signature(obj.__func__)
+ return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+
+ try:
+ sig = obj.__signature__
+ except AttributeError:
+ pass
+ else:
+ if sig is not None:
+ return sig
+
+ try:
+ # Was this function wrapped by a decorator?
+ wrapped = obj.__wrapped__
+ except AttributeError:
+ pass
+ else:
+ return signature(wrapped)
+
if inspect.isfunction(obj):
- return Signature.from_function(obj)
-
- if isinstance(obj, functools.partial):
- sig = signature(obj.func)
-
- new_params = OrderedDict(sig.parameters.items())
-
- partial_args = obj.args or ()
- partial_keywords = obj.keywords or {}
- try:
- ba = sig.bind_partial(*partial_args, **partial_keywords)
- except TypeError as ex:
- msg = 'partial object {0!r} has incorrect arguments'.format(obj)
- raise ValueError(msg)
-
- for arg_name, arg_value in ba.arguments.items():
- param = new_params[arg_name]
- if arg_name in partial_keywords:
- # We set a new default value, because the following code
- # is correct:
- #
- # >>> def foo(a): print(a)
- # >>> print(partial(partial(foo, a=10), a=20)())
- # 20
- # >>> print(partial(partial(foo, a=10), a=20)(a=30))
- # 30
- #
- # So, with 'partial' objects, passing a keyword argument is
- # like setting a new default value for the corresponding
- # parameter
- #
- # We also mark this parameter with '_partial_kwarg'
- # flag. Later, in '_bind', the 'default' value of this
- # parameter will be added to 'kwargs', to simulate
- # the 'functools.partial' real call.
- new_params[arg_name] = param.replace(default=arg_value,
- _partial_kwarg=True)
-
- elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
- not param._partial_kwarg):
- new_params.pop(arg_name)
-
- return sig.replace(parameters=new_params.values())
-
- sig = None
- if isinstance(obj, type):
- # obj is a class or a metaclass
-
- # First, let's see if it has an overloaded __call__ defined
- # in its metaclass
- call = _get_user_defined_method(type(obj), '__call__')
- if call is not None:
- sig = signature(call)
- else:
- # Now we check if the 'obj' class has a '__new__' method
- new = _get_user_defined_method(obj, '__new__')
- if new is not None:
- sig = signature(new)
- else:
- # Finally, we should have at least __init__ implemented
- init = _get_user_defined_method(obj, '__init__')
- if init is not None:
- sig = signature(init)
- elif not isinstance(obj, _NonUserDefinedCallables):
- # An object with __call__
- # We also check that the 'obj' is not an instance of
- # _WrapperDescriptor or _MethodWrapper to avoid
- # infinite recursion (and even potential segfault)
- call = _get_user_defined_method(type(obj), '__call__', 'im_func')
- if call is not None:
- sig = signature(call)
-
- if sig is not None:
- return sig
-
- if isinstance(obj, types.BuiltinFunctionType):
- # Raise a nicer error message for builtins
- msg = 'no signature found for builtin function {0!r}'.format(obj)
- raise ValueError(msg)
-
- raise ValueError('callable {0!r} is not supported by signature'.format(obj))
-
-
-class _void(object):
- '''A private marker - used in Parameter & Signature'''
-
-
-class _empty(object):
- pass
-
-
-class _ParameterKind(int):
- def __new__(self, *args, **kwargs):
- obj = int.__new__(self, *args)
- obj._name = kwargs['name']
- return obj
-
- def __str__(self):
- return self._name
-
- def __repr__(self):
- return '<_ParameterKind: {0!r}>'.format(self._name)
-
-
-_POSITIONAL_ONLY = _ParameterKind(0, name='POSITIONAL_ONLY')
-_POSITIONAL_OR_KEYWORD = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
-_VAR_POSITIONAL = _ParameterKind(2, name='VAR_POSITIONAL')
-_KEYWORD_ONLY = _ParameterKind(3, name='KEYWORD_ONLY')
-_VAR_KEYWORD = _ParameterKind(4, name='VAR_KEYWORD')
-
-
-class Parameter(object):
- '''Represents a parameter in a function signature.
-
- Has the following public attributes:
-
- * name : str
- The name of the parameter as a string.
- * default : object
- The default value for the parameter if specified. If the
- parameter has no default value, this attribute is not set.
- * annotation
- The annotation for the parameter if specified. If the
- parameter has no annotation, this attribute is not set.
- * kind : str
- Describes how argument values are bound to the parameter.
- Possible values: `Parameter.POSITIONAL_ONLY`,
- `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
- `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
- '''
-
- __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg')
-
- POSITIONAL_ONLY = _POSITIONAL_ONLY
- POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD
- VAR_POSITIONAL = _VAR_POSITIONAL
- KEYWORD_ONLY = _KEYWORD_ONLY
- VAR_KEYWORD = _VAR_KEYWORD
-
- empty = _empty
-
- def __init__(self, name, kind, default=_empty, annotation=_empty,
- _partial_kwarg=False):
-
- if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD,
- _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD):
- raise ValueError("invalid value for 'Parameter.kind' attribute")
- self._kind = kind
-
- if default is not _empty:
- if kind in (_VAR_POSITIONAL, _VAR_KEYWORD):
- msg = '{0} parameters cannot have default values'.format(kind)
- raise ValueError(msg)
- self._default = default
- self._annotation = annotation
-
- if name is None:
- if kind != _POSITIONAL_ONLY:
- raise ValueError("None is not a valid name for a "
- "non-positional-only parameter")
- self._name = name
- else:
- name = str(name)
- if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I):
- msg = '{0!r} is not a valid parameter name'.format(name)
- raise ValueError(msg)
- self._name = name
-
- self._partial_kwarg = _partial_kwarg
-
- @property
- def name(self):
- return self._name
-
- @property
- def default(self):
- return self._default
-
- @property
- def annotation(self):
- return self._annotation
-
- @property
- def kind(self):
- return self._kind
-
- def replace(self, name=_void, kind=_void, annotation=_void,
- default=_void, _partial_kwarg=_void):
- '''Creates a customized copy of the Parameter.'''
-
- if name is _void:
- name = self._name
-
- if kind is _void:
- kind = self._kind
-
- if annotation is _void:
- annotation = self._annotation
-
- if default is _void:
- default = self._default
-
- if _partial_kwarg is _void:
- _partial_kwarg = self._partial_kwarg
-
- return type(self)(name, kind, default=default, annotation=annotation,
- _partial_kwarg=_partial_kwarg)
-
- def __str__(self):
- kind = self.kind
-
- formatted = self._name
- if kind == _POSITIONAL_ONLY:
- if formatted is None:
- formatted = ''
- formatted = '<{0}>'.format(formatted)
-
- # Add annotation and default value
- if self._annotation is not _empty:
- formatted = '{0}:{1}'.format(formatted,
- formatannotation(self._annotation))
-
- if self._default is not _empty:
- formatted = '{0}={1}'.format(formatted, repr(self._default))
-
- if kind == _VAR_POSITIONAL:
- formatted = '*' + formatted
- elif kind == _VAR_KEYWORD:
- formatted = '**' + formatted
-
- return formatted
-
- def __repr__(self):
- return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__,
- id(self), self.name)
-
- def __hash__(self):
- msg = "unhashable type: '{0}'".format(self.__class__.__name__)
- raise TypeError(msg)
-
- def __eq__(self, other):
- return (issubclass(other.__class__, Parameter) and
- self._name == other._name and
- self._kind == other._kind and
- self._default == other._default and
- self._annotation == other._annotation)
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
-
-class BoundArguments(object):
- '''Result of :meth:`Signature.bind` call. Holds the mapping of arguments
- to the function's parameters.
-
- Has the following public attributes:
-
- arguments : :class:`collections.OrderedDict`
- An ordered mutable mapping of parameters' names to arguments' values.
- Does not contain arguments' default values.
- signature : :class:`Signature`
- The Signature object that created this instance.
- args : tuple
- Tuple of positional arguments values.
- kwargs : dict
- Dict of keyword arguments values.
- '''
-
- def __init__(self, signature, arguments):
- self.arguments = arguments
- self._signature = signature
-
- @property
- def signature(self):
- return self._signature
-
- @property
- def args(self):
- args = []
- for param_name, param in self._signature.parameters.items():
- if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
- param._partial_kwarg):
- # Keyword arguments mapped by 'functools.partial'
- # (Parameter._partial_kwarg is True) are mapped
- # in 'BoundArguments.kwargs', along with VAR_KEYWORD &
- # KEYWORD_ONLY
- break
-
- try:
- arg = self.arguments[param_name]
- except KeyError:
- # We're done here. Other arguments
- # will be mapped in 'BoundArguments.kwargs'
- break
- else:
- if param.kind == _VAR_POSITIONAL:
- # *args
- args.extend(arg)
- else:
- # plain argument
- args.append(arg)
-
- return tuple(args)
-
- @property
- def kwargs(self):
- kwargs = {}
- kwargs_started = False
- for param_name, param in self._signature.parameters.items():
- if not kwargs_started:
- if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
- param._partial_kwarg):
- kwargs_started = True
- else:
- if param_name not in self.arguments:
- kwargs_started = True
- continue
-
- if not kwargs_started:
- continue
-
- try:
- arg = self.arguments[param_name]
- except KeyError:
- pass
- else:
- if param.kind == _VAR_KEYWORD:
- # **kwargs
- kwargs.update(arg)
- else:
- # plain keyword argument
- kwargs[param_name] = arg
-
- return kwargs
-
- def __hash__(self):
- msg = "unhashable type: '{0}'".format(self.__class__.__name__)
- raise TypeError(msg)
-
- def __eq__(self, other):
- return (issubclass(other.__class__, BoundArguments) and
- self.signature == other.signature and
- self.arguments == other.arguments)
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
-
-class Signature(object):
- '''A Signature object represents the overall signature of a function.
- It stores a Parameter object for each parameter accepted by the
- function, as well as information specific to the function itself.
-
- A Signature object has the following public attributes:
-
- parameters : :class:`collections.OrderedDict`
- An ordered mapping of parameters' names to the corresponding
- Parameter objects (keyword-only arguments are in the same order
- as listed in `code.co_varnames`).
- return_annotation
- The annotation for the return type of the function if specified.
- If the function has no annotation for its return type, this
- attribute is not set.
- '''
-
- __slots__ = ('_return_annotation', '_parameters')
-
- _parameter_cls = Parameter
- _bound_arguments_cls = BoundArguments
-
- empty = _empty
-
- def __init__(self, parameters=None, return_annotation=_empty,
- __validate_parameters__=True):
- '''Constructs Signature from the given list of Parameter
- objects and 'return_annotation'. All arguments are optional.
- '''
-
- if parameters is None:
- params = OrderedDict()
- else:
- if __validate_parameters__:
- params = OrderedDict()
- top_kind = _POSITIONAL_ONLY
-
- for idx, param in enumerate(parameters):
- kind = param.kind
- if kind < top_kind:
- msg = 'wrong parameter order: {0} before {1}'
- msg = msg.format(top_kind, param.kind)
- raise ValueError(msg)
- else:
- top_kind = kind
-
- name = param.name
- if name is None:
- name = str(idx)
- param = param.replace(name=name)
-
- if name in params:
- msg = 'duplicate parameter name: {0!r}'.format(name)
- raise ValueError(msg)
- params[name] = param
- else:
- params = OrderedDict(((param.name, param)
- for param in parameters))
-
- self._parameters = params
- self._return_annotation = return_annotation
-
- @classmethod
- def from_function(cls, func):
- '''Constructs Signature for the given python function'''
-
+ return Signature.from_function(obj)
+
+ if isinstance(obj, functools.partial):
+ sig = signature(obj.func)
+
+ new_params = OrderedDict(sig.parameters.items())
+
+ partial_args = obj.args or ()
+ partial_keywords = obj.keywords or {}
+ try:
+ ba = sig.bind_partial(*partial_args, **partial_keywords)
+ except TypeError as ex:
+ msg = 'partial object {0!r} has incorrect arguments'.format(obj)
+ raise ValueError(msg)
+
+ for arg_name, arg_value in ba.arguments.items():
+ param = new_params[arg_name]
+ if arg_name in partial_keywords:
+ # We set a new default value, because the following code
+ # is correct:
+ #
+ # >>> def foo(a): print(a)
+ # >>> print(partial(partial(foo, a=10), a=20)())
+ # 20
+ # >>> print(partial(partial(foo, a=10), a=20)(a=30))
+ # 30
+ #
+ # So, with 'partial' objects, passing a keyword argument is
+ # like setting a new default value for the corresponding
+ # parameter
+ #
+ # We also mark this parameter with '_partial_kwarg'
+ # flag. Later, in '_bind', the 'default' value of this
+ # parameter will be added to 'kwargs', to simulate
+ # the 'functools.partial' real call.
+ new_params[arg_name] = param.replace(default=arg_value,
+ _partial_kwarg=True)
+
+ elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
+ not param._partial_kwarg):
+ new_params.pop(arg_name)
+
+ return sig.replace(parameters=new_params.values())
+
+ sig = None
+ if isinstance(obj, type):
+ # obj is a class or a metaclass
+
+ # First, let's see if it has an overloaded __call__ defined
+ # in its metaclass
+ call = _get_user_defined_method(type(obj), '__call__')
+ if call is not None:
+ sig = signature(call)
+ else:
+ # Now we check if the 'obj' class has a '__new__' method
+ new = _get_user_defined_method(obj, '__new__')
+ if new is not None:
+ sig = signature(new)
+ else:
+ # Finally, we should have at least __init__ implemented
+ init = _get_user_defined_method(obj, '__init__')
+ if init is not None:
+ sig = signature(init)
+ elif not isinstance(obj, _NonUserDefinedCallables):
+ # An object with __call__
+ # We also check that the 'obj' is not an instance of
+ # _WrapperDescriptor or _MethodWrapper to avoid
+ # infinite recursion (and even potential segfault)
+ call = _get_user_defined_method(type(obj), '__call__', 'im_func')
+ if call is not None:
+ sig = signature(call)
+
+ if sig is not None:
+ return sig
+
+ if isinstance(obj, types.BuiltinFunctionType):
+ # Raise a nicer error message for builtins
+ msg = 'no signature found for builtin function {0!r}'.format(obj)
+ raise ValueError(msg)
+
+ raise ValueError('callable {0!r} is not supported by signature'.format(obj))
+
+
+class _void(object):
+ '''A private marker - used in Parameter & Signature'''
+
+
+class _empty(object):
+ pass
+
+
+class _ParameterKind(int):
+ def __new__(self, *args, **kwargs):
+ obj = int.__new__(self, *args)
+ obj._name = kwargs['name']
+ return obj
+
+ def __str__(self):
+ return self._name
+
+ def __repr__(self):
+ return '<_ParameterKind: {0!r}>'.format(self._name)
+
+
+_POSITIONAL_ONLY = _ParameterKind(0, name='POSITIONAL_ONLY')
+_POSITIONAL_OR_KEYWORD = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
+_VAR_POSITIONAL = _ParameterKind(2, name='VAR_POSITIONAL')
+_KEYWORD_ONLY = _ParameterKind(3, name='KEYWORD_ONLY')
+_VAR_KEYWORD = _ParameterKind(4, name='VAR_KEYWORD')
+
+
+class Parameter(object):
+ '''Represents a parameter in a function signature.
+
+ Has the following public attributes:
+
+ * name : str
+ The name of the parameter as a string.
+ * default : object
+ The default value for the parameter if specified. If the
+ parameter has no default value, this attribute is not set.
+ * annotation
+ The annotation for the parameter if specified. If the
+ parameter has no annotation, this attribute is not set.
+ * kind : str
+ Describes how argument values are bound to the parameter.
+ Possible values: `Parameter.POSITIONAL_ONLY`,
+ `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
+ `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
+ '''
+
+ __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg')
+
+ POSITIONAL_ONLY = _POSITIONAL_ONLY
+ POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD
+ VAR_POSITIONAL = _VAR_POSITIONAL
+ KEYWORD_ONLY = _KEYWORD_ONLY
+ VAR_KEYWORD = _VAR_KEYWORD
+
+ empty = _empty
+
+ def __init__(self, name, kind, default=_empty, annotation=_empty,
+ _partial_kwarg=False):
+
+ if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD,
+ _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD):
+ raise ValueError("invalid value for 'Parameter.kind' attribute")
+ self._kind = kind
+
+ if default is not _empty:
+ if kind in (_VAR_POSITIONAL, _VAR_KEYWORD):
+ msg = '{0} parameters cannot have default values'.format(kind)
+ raise ValueError(msg)
+ self._default = default
+ self._annotation = annotation
+
+ if name is None:
+ if kind != _POSITIONAL_ONLY:
+ raise ValueError("None is not a valid name for a "
+ "non-positional-only parameter")
+ self._name = name
+ else:
+ name = str(name)
+ if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I):
+ msg = '{0!r} is not a valid parameter name'.format(name)
+ raise ValueError(msg)
+ self._name = name
+
+ self._partial_kwarg = _partial_kwarg
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def default(self):
+ return self._default
+
+ @property
+ def annotation(self):
+ return self._annotation
+
+ @property
+ def kind(self):
+ return self._kind
+
+ def replace(self, name=_void, kind=_void, annotation=_void,
+ default=_void, _partial_kwarg=_void):
+ '''Creates a customized copy of the Parameter.'''
+
+ if name is _void:
+ name = self._name
+
+ if kind is _void:
+ kind = self._kind
+
+ if annotation is _void:
+ annotation = self._annotation
+
+ if default is _void:
+ default = self._default
+
+ if _partial_kwarg is _void:
+ _partial_kwarg = self._partial_kwarg
+
+ return type(self)(name, kind, default=default, annotation=annotation,
+ _partial_kwarg=_partial_kwarg)
+
+ def __str__(self):
+ kind = self.kind
+
+ formatted = self._name
+ if kind == _POSITIONAL_ONLY:
+ if formatted is None:
+ formatted = ''
+ formatted = '<{0}>'.format(formatted)
+
+ # Add annotation and default value
+ if self._annotation is not _empty:
+ formatted = '{0}:{1}'.format(formatted,
+ formatannotation(self._annotation))
+
+ if self._default is not _empty:
+ formatted = '{0}={1}'.format(formatted, repr(self._default))
+
+ if kind == _VAR_POSITIONAL:
+ formatted = '*' + formatted
+ elif kind == _VAR_KEYWORD:
+ formatted = '**' + formatted
+
+ return formatted
+
+ def __repr__(self):
+ return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__,
+ id(self), self.name)
+
+ def __hash__(self):
+ msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+ raise TypeError(msg)
+
+ def __eq__(self, other):
+ return (issubclass(other.__class__, Parameter) and
+ self._name == other._name and
+ self._kind == other._kind and
+ self._default == other._default and
+ self._annotation == other._annotation)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+class BoundArguments(object):
+ '''Result of :meth:`Signature.bind` call. Holds the mapping of arguments
+ to the function's parameters.
+
+ Has the following public attributes:
+
+ arguments : :class:`collections.OrderedDict`
+ An ordered mutable mapping of parameters' names to arguments' values.
+ Does not contain arguments' default values.
+ signature : :class:`Signature`
+ The Signature object that created this instance.
+ args : tuple
+ Tuple of positional arguments values.
+ kwargs : dict
+ Dict of keyword arguments values.
+ '''
+
+ def __init__(self, signature, arguments):
+ self.arguments = arguments
+ self._signature = signature
+
+ @property
+ def signature(self):
+ return self._signature
+
+ @property
+ def args(self):
+ args = []
+ for param_name, param in self._signature.parameters.items():
+ if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+ param._partial_kwarg):
+ # Keyword arguments mapped by 'functools.partial'
+ # (Parameter._partial_kwarg is True) are mapped
+ # in 'BoundArguments.kwargs', along with VAR_KEYWORD &
+ # KEYWORD_ONLY
+ break
+
+ try:
+ arg = self.arguments[param_name]
+ except KeyError:
+ # We're done here. Other arguments
+ # will be mapped in 'BoundArguments.kwargs'
+ break
+ else:
+ if param.kind == _VAR_POSITIONAL:
+ # *args
+ args.extend(arg)
+ else:
+ # plain argument
+ args.append(arg)
+
+ return tuple(args)
+
+ @property
+ def kwargs(self):
+ kwargs = {}
+ kwargs_started = False
+ for param_name, param in self._signature.parameters.items():
+ if not kwargs_started:
+ if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+ param._partial_kwarg):
+ kwargs_started = True
+ else:
+ if param_name not in self.arguments:
+ kwargs_started = True
+ continue
+
+ if not kwargs_started:
+ continue
+
+ try:
+ arg = self.arguments[param_name]
+ except KeyError:
+ pass
+ else:
+ if param.kind == _VAR_KEYWORD:
+ # **kwargs
+ kwargs.update(arg)
+ else:
+ # plain keyword argument
+ kwargs[param_name] = arg
+
+ return kwargs
+
+ def __hash__(self):
+ msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+ raise TypeError(msg)
+
+ def __eq__(self, other):
+ return (issubclass(other.__class__, BoundArguments) and
+ self.signature == other.signature and
+ self.arguments == other.arguments)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+class Signature(object):
+ '''A Signature object represents the overall signature of a function.
+ It stores a Parameter object for each parameter accepted by the
+ function, as well as information specific to the function itself.
+
+ A Signature object has the following public attributes:
+
+ parameters : :class:`collections.OrderedDict`
+ An ordered mapping of parameters' names to the corresponding
+ Parameter objects (keyword-only arguments are in the same order
+ as listed in `code.co_varnames`).
+ return_annotation
+ The annotation for the return type of the function if specified.
+ If the function has no annotation for its return type, this
+ attribute is not set.
+ '''
+
+ __slots__ = ('_return_annotation', '_parameters')
+
+ _parameter_cls = Parameter
+ _bound_arguments_cls = BoundArguments
+
+ empty = _empty
+
+ def __init__(self, parameters=None, return_annotation=_empty,
+ __validate_parameters__=True):
+ '''Constructs Signature from the given list of Parameter
+ objects and 'return_annotation'. All arguments are optional.
+ '''
+
+ if parameters is None:
+ params = OrderedDict()
+ else:
+ if __validate_parameters__:
+ params = OrderedDict()
+ top_kind = _POSITIONAL_ONLY
+
+ for idx, param in enumerate(parameters):
+ kind = param.kind
+ if kind < top_kind:
+ msg = 'wrong parameter order: {0} before {1}'
+ msg = msg.format(top_kind, param.kind)
+ raise ValueError(msg)
+ else:
+ top_kind = kind
+
+ name = param.name
+ if name is None:
+ name = str(idx)
+ param = param.replace(name=name)
+
+ if name in params:
+ msg = 'duplicate parameter name: {0!r}'.format(name)
+ raise ValueError(msg)
+ params[name] = param
+ else:
+ params = OrderedDict(((param.name, param)
+ for param in parameters))
+
+ self._parameters = params
+ self._return_annotation = return_annotation
+
+ @classmethod
+ def from_function(cls, func):
+ '''Constructs Signature for the given python function'''
+
if not inspect.isfunction(func):
- raise TypeError('{0!r} is not a Python function'.format(func))
-
- Parameter = cls._parameter_cls
-
- # Parameter information.
- func_code = func.__code__
- pos_count = func_code.co_argcount
- arg_names = func_code.co_varnames
- positional = tuple(arg_names[:pos_count])
- keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0)
- keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
- annotations = getattr(func, '__annotations__', {})
- defaults = func.__defaults__
- kwdefaults = getattr(func, '__kwdefaults__', None)
-
- if defaults:
- pos_default_count = len(defaults)
- else:
- pos_default_count = 0
-
- parameters = []
-
- # Non-keyword-only parameters w/o defaults.
- non_default_count = pos_count - pos_default_count
- for name in positional[:non_default_count]:
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_POSITIONAL_OR_KEYWORD))
-
- # ... w/ defaults.
- for offset, name in enumerate(positional[non_default_count:]):
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_POSITIONAL_OR_KEYWORD,
- default=defaults[offset]))
-
- # *args
- if func_code.co_flags & 0x04:
- name = arg_names[pos_count + keyword_only_count]
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_VAR_POSITIONAL))
-
- # Keyword-only parameters.
- for name in keyword_only:
- default = _empty
- if kwdefaults is not None:
- default = kwdefaults.get(name, _empty)
-
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_KEYWORD_ONLY,
- default=default))
- # **kwargs
- if func_code.co_flags & 0x08:
- index = pos_count + keyword_only_count
- if func_code.co_flags & 0x04:
- index += 1
-
- name = arg_names[index]
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_VAR_KEYWORD))
-
- return cls(parameters,
- return_annotation=annotations.get('return', _empty),
- __validate_parameters__=False)
-
- @property
- def parameters(self):
- try:
- return types.MappingProxyType(self._parameters)
- except AttributeError:
- return OrderedDict(self._parameters.items())
-
- @property
- def return_annotation(self):
- return self._return_annotation
-
- def replace(self, parameters=_void, return_annotation=_void):
- '''Creates a customized copy of the Signature.
- Pass 'parameters' and/or 'return_annotation' arguments
- to override them in the new copy.
- '''
-
- if parameters is _void:
- parameters = self.parameters.values()
-
- if return_annotation is _void:
- return_annotation = self._return_annotation
-
- return type(self)(parameters,
- return_annotation=return_annotation)
-
- def __hash__(self):
- msg = "unhashable type: '{0}'".format(self.__class__.__name__)
- raise TypeError(msg)
-
- def __eq__(self, other):
- if (not issubclass(type(other), Signature) or
- self.return_annotation != other.return_annotation or
- len(self.parameters) != len(other.parameters)):
- return False
-
- other_positions = dict((param, idx)
- for idx, param in enumerate(other.parameters.keys()))
-
- for idx, (param_name, param) in enumerate(self.parameters.items()):
- if param.kind == _KEYWORD_ONLY:
- try:
- other_param = other.parameters[param_name]
- except KeyError:
- return False
- else:
- if param != other_param:
- return False
- else:
- try:
- other_idx = other_positions[param_name]
- except KeyError:
- return False
- else:
- if (idx != other_idx or
- param != other.parameters[param_name]):
- return False
-
- return True
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
- def _bind(self, args, kwargs, partial=False):
- '''Private method. Don't use directly.'''
-
- arguments = OrderedDict()
-
- parameters = iter(self.parameters.values())
- parameters_ex = ()
- arg_vals = iter(args)
-
- if partial:
- # Support for binding arguments to 'functools.partial' objects.
- # See 'functools.partial' case in 'signature()' implementation
- # for details.
- for param_name, param in self.parameters.items():
- if (param._partial_kwarg and param_name not in kwargs):
- # Simulating 'functools.partial' behavior
- kwargs[param_name] = param.default
-
- while True:
- # Let's iterate through the positional arguments and corresponding
- # parameters
- try:
- arg_val = next(arg_vals)
- except StopIteration:
- # No more positional arguments
- try:
- param = next(parameters)
- except StopIteration:
- # No more parameters. That's it. Just need to check that
- # we have no `kwargs` after this while loop
- break
- else:
- if param.kind == _VAR_POSITIONAL:
- # That's OK, just empty *args. Let's start parsing
- # kwargs
- break
- elif param.name in kwargs:
- if param.kind == _POSITIONAL_ONLY:
- msg = '{arg!r} parameter is positional only, ' \
- 'but was passed as a keyword'
- msg = msg.format(arg=param.name)
- raise TypeError(msg)
- parameters_ex = (param,)
- break
- elif (param.kind == _VAR_KEYWORD or
- param.default is not _empty):
- # That's fine too - we have a default value for this
- # parameter. So, lets start parsing `kwargs`, starting
- # with the current parameter
- parameters_ex = (param,)
- break
- else:
- if partial:
- parameters_ex = (param,)
- break
- else:
- msg = '{arg!r} parameter lacking default value'
- msg = msg.format(arg=param.name)
- raise TypeError(msg)
- else:
- # We have a positional argument to process
- try:
- param = next(parameters)
- except StopIteration:
- raise TypeError('too many positional arguments')
- else:
- if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
- # Looks like we have no parameter for this positional
- # argument
- raise TypeError('too many positional arguments')
-
- if param.kind == _VAR_POSITIONAL:
- # We have an '*args'-like argument, let's fill it with
- # all positional arguments we have left and move on to
- # the next phase
- values = [arg_val]
- values.extend(arg_vals)
- arguments[param.name] = tuple(values)
- break
-
- if param.name in kwargs:
- raise TypeError('multiple values for argument '
- '{arg!r}'.format(arg=param.name))
-
- arguments[param.name] = arg_val
-
- # Now, we iterate through the remaining parameters to process
- # keyword arguments
- kwargs_param = None
- for param in itertools.chain(parameters_ex, parameters):
- if param.kind == _POSITIONAL_ONLY:
- # This should never happen in case of a properly built
- # Signature object (but let's have this check here
- # to ensure correct behaviour just in case)
- raise TypeError('{arg!r} parameter is positional only, '
- 'but was passed as a keyword'. \
- format(arg=param.name))
-
- if param.kind == _VAR_KEYWORD:
- # Memorize that we have a '**kwargs'-like parameter
- kwargs_param = param
- continue
-
- param_name = param.name
- try:
- arg_val = kwargs.pop(param_name)
- except KeyError:
- # We have no value for this parameter. It's fine though,
- # if it has a default value, or it is an '*args'-like
- # parameter, left alone by the processing of positional
- # arguments.
- if (not partial and param.kind != _VAR_POSITIONAL and
- param.default is _empty):
- raise TypeError('{arg!r} parameter lacking default value'. \
- format(arg=param_name))
-
- else:
- arguments[param_name] = arg_val
-
- if kwargs:
- if kwargs_param is not None:
- # Process our '**kwargs'-like parameter
- arguments[kwargs_param.name] = kwargs
- else:
- raise TypeError('too many keyword arguments')
-
- return self._bound_arguments_cls(self, arguments)
-
- def bind(self, *args, **kwargs):
- '''Get a :class:`BoundArguments` object, that maps the passed `args`
- and `kwargs` to the function's signature. Raises :exc:`TypeError`
- if the passed arguments can not be bound.
- '''
- return self._bind(args, kwargs)
-
- def bind_partial(self, *args, **kwargs):
- '''Get a :class:`BoundArguments` object, that partially maps the
- passed `args` and `kwargs` to the function's signature.
- Raises :exc:`TypeError` if the passed arguments can not be bound.
- '''
- return self._bind(args, kwargs, partial=True)
-
- def __str__(self):
- result = []
- render_kw_only_separator = True
- for idx, param in enumerate(self.parameters.values()):
- formatted = str(param)
-
- kind = param.kind
- if kind == _VAR_POSITIONAL:
- # OK, we have an '*args'-like parameter, so we won't need
- # a '*' to separate keyword-only arguments
- render_kw_only_separator = False
- elif kind == _KEYWORD_ONLY and render_kw_only_separator:
- # We have a keyword-only parameter to render and we haven't
- # rendered an '*args'-like parameter before, so add a '*'
- # separator to the parameters list ("foo(arg1, *, arg2)" case)
- result.append('*')
- # This condition should be only triggered once, so
- # reset the flag
- render_kw_only_separator = False
-
- result.append(formatted)
-
- rendered = '({0})'.format(', '.join(result))
-
- if self.return_annotation is not _empty:
- anno = formatannotation(self.return_annotation)
- rendered += ' -> {0}'.format(anno)
-
- return rendered
-
+ raise TypeError('{0!r} is not a Python function'.format(func))
+
+ Parameter = cls._parameter_cls
+
+ # Parameter information.
+ func_code = func.__code__
+ pos_count = func_code.co_argcount
+ arg_names = func_code.co_varnames
+ positional = tuple(arg_names[:pos_count])
+ keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0)
+ keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
+ annotations = getattr(func, '__annotations__', {})
+ defaults = func.__defaults__
+ kwdefaults = getattr(func, '__kwdefaults__', None)
+
+ if defaults:
+ pos_default_count = len(defaults)
+ else:
+ pos_default_count = 0
+
+ parameters = []
+
+ # Non-keyword-only parameters w/o defaults.
+ non_default_count = pos_count - pos_default_count
+ for name in positional[:non_default_count]:
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_POSITIONAL_OR_KEYWORD))
+
+ # ... w/ defaults.
+ for offset, name in enumerate(positional[non_default_count:]):
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_POSITIONAL_OR_KEYWORD,
+ default=defaults[offset]))
+
+ # *args
+ if func_code.co_flags & 0x04:
+ name = arg_names[pos_count + keyword_only_count]
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_VAR_POSITIONAL))
+
+ # Keyword-only parameters.
+ for name in keyword_only:
+ default = _empty
+ if kwdefaults is not None:
+ default = kwdefaults.get(name, _empty)
+
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_KEYWORD_ONLY,
+ default=default))
+ # **kwargs
+ if func_code.co_flags & 0x08:
+ index = pos_count + keyword_only_count
+ if func_code.co_flags & 0x04:
+ index += 1
+
+ name = arg_names[index]
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_VAR_KEYWORD))
+
+ return cls(parameters,
+ return_annotation=annotations.get('return', _empty),
+ __validate_parameters__=False)
+
+ @property
+ def parameters(self):
+ try:
+ return types.MappingProxyType(self._parameters)
+ except AttributeError:
+ return OrderedDict(self._parameters.items())
+
+ @property
+ def return_annotation(self):
+ return self._return_annotation
+
+ def replace(self, parameters=_void, return_annotation=_void):
+ '''Creates a customized copy of the Signature.
+ Pass 'parameters' and/or 'return_annotation' arguments
+ to override them in the new copy.
+ '''
+
+ if parameters is _void:
+ parameters = self.parameters.values()
+
+ if return_annotation is _void:
+ return_annotation = self._return_annotation
+
+ return type(self)(parameters,
+ return_annotation=return_annotation)
+
+ def __hash__(self):
+ msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+ raise TypeError(msg)
+
+ def __eq__(self, other):
+ if (not issubclass(type(other), Signature) or
+ self.return_annotation != other.return_annotation or
+ len(self.parameters) != len(other.parameters)):
+ return False
+
+ other_positions = dict((param, idx)
+ for idx, param in enumerate(other.parameters.keys()))
+
+ for idx, (param_name, param) in enumerate(self.parameters.items()):
+ if param.kind == _KEYWORD_ONLY:
+ try:
+ other_param = other.parameters[param_name]
+ except KeyError:
+ return False
+ else:
+ if param != other_param:
+ return False
+ else:
+ try:
+ other_idx = other_positions[param_name]
+ except KeyError:
+ return False
+ else:
+ if (idx != other_idx or
+ param != other.parameters[param_name]):
+ return False
+
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def _bind(self, args, kwargs, partial=False):
+ '''Private method. Don't use directly.'''
+
+ arguments = OrderedDict()
+
+ parameters = iter(self.parameters.values())
+ parameters_ex = ()
+ arg_vals = iter(args)
+
+ if partial:
+ # Support for binding arguments to 'functools.partial' objects.
+ # See 'functools.partial' case in 'signature()' implementation
+ # for details.
+ for param_name, param in self.parameters.items():
+ if (param._partial_kwarg and param_name not in kwargs):
+ # Simulating 'functools.partial' behavior
+ kwargs[param_name] = param.default
+
+ while True:
+ # Let's iterate through the positional arguments and corresponding
+ # parameters
+ try:
+ arg_val = next(arg_vals)
+ except StopIteration:
+ # No more positional arguments
+ try:
+ param = next(parameters)
+ except StopIteration:
+ # No more parameters. That's it. Just need to check that
+ # we have no `kwargs` after this while loop
+ break
+ else:
+ if param.kind == _VAR_POSITIONAL:
+ # That's OK, just empty *args. Let's start parsing
+ # kwargs
+ break
+ elif param.name in kwargs:
+ if param.kind == _POSITIONAL_ONLY:
+ msg = '{arg!r} parameter is positional only, ' \
+ 'but was passed as a keyword'
+ msg = msg.format(arg=param.name)
+ raise TypeError(msg)
+ parameters_ex = (param,)
+ break
+ elif (param.kind == _VAR_KEYWORD or
+ param.default is not _empty):
+ # That's fine too - we have a default value for this
+ # parameter. So, lets start parsing `kwargs`, starting
+ # with the current parameter
+ parameters_ex = (param,)
+ break
+ else:
+ if partial:
+ parameters_ex = (param,)
+ break
+ else:
+ msg = '{arg!r} parameter lacking default value'
+ msg = msg.format(arg=param.name)
+ raise TypeError(msg)
+ else:
+ # We have a positional argument to process
+ try:
+ param = next(parameters)
+ except StopIteration:
+ raise TypeError('too many positional arguments')
+ else:
+ if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
+ # Looks like we have no parameter for this positional
+ # argument
+ raise TypeError('too many positional arguments')
+
+ if param.kind == _VAR_POSITIONAL:
+ # We have an '*args'-like argument, let's fill it with
+ # all positional arguments we have left and move on to
+ # the next phase
+ values = [arg_val]
+ values.extend(arg_vals)
+ arguments[param.name] = tuple(values)
+ break
+
+ if param.name in kwargs:
+ raise TypeError('multiple values for argument '
+ '{arg!r}'.format(arg=param.name))
+
+ arguments[param.name] = arg_val
+
+ # Now, we iterate through the remaining parameters to process
+ # keyword arguments
+ kwargs_param = None
+ for param in itertools.chain(parameters_ex, parameters):
+ if param.kind == _POSITIONAL_ONLY:
+ # This should never happen in case of a properly built
+ # Signature object (but let's have this check here
+ # to ensure correct behaviour just in case)
+ raise TypeError('{arg!r} parameter is positional only, '
+ 'but was passed as a keyword'. \
+ format(arg=param.name))
+
+ if param.kind == _VAR_KEYWORD:
+ # Memorize that we have a '**kwargs'-like parameter
+ kwargs_param = param
+ continue
+
+ param_name = param.name
+ try:
+ arg_val = kwargs.pop(param_name)
+ except KeyError:
+ # We have no value for this parameter. It's fine though,
+ # if it has a default value, or it is an '*args'-like
+ # parameter, left alone by the processing of positional
+ # arguments.
+ if (not partial and param.kind != _VAR_POSITIONAL and
+ param.default is _empty):
+ raise TypeError('{arg!r} parameter lacking default value'. \
+ format(arg=param_name))
+
+ else:
+ arguments[param_name] = arg_val
+
+ if kwargs:
+ if kwargs_param is not None:
+ # Process our '**kwargs'-like parameter
+ arguments[kwargs_param.name] = kwargs
+ else:
+ raise TypeError('too many keyword arguments')
+
+ return self._bound_arguments_cls(self, arguments)
+
+ def bind(self, *args, **kwargs):
+ '''Get a :class:`BoundArguments` object, that maps the passed `args`
+ and `kwargs` to the function's signature. Raises :exc:`TypeError`
+ if the passed arguments can not be bound.
+ '''
+ return self._bind(args, kwargs)
+
+ def bind_partial(self, *args, **kwargs):
+ '''Get a :class:`BoundArguments` object, that partially maps the
+ passed `args` and `kwargs` to the function's signature.
+ Raises :exc:`TypeError` if the passed arguments can not be bound.
+ '''
+ return self._bind(args, kwargs, partial=True)
+
+ def __str__(self):
+ result = []
+ render_kw_only_separator = True
+ for idx, param in enumerate(self.parameters.values()):
+ formatted = str(param)
+
+ kind = param.kind
+ if kind == _VAR_POSITIONAL:
+ # OK, we have an '*args'-like parameter, so we won't need
+ # a '*' to separate keyword-only arguments
+ render_kw_only_separator = False
+ elif kind == _KEYWORD_ONLY and render_kw_only_separator:
+ # We have a keyword-only parameter to render and we haven't
+ # rendered an '*args'-like parameter before, so add a '*'
+ # separator to the parameters list ("foo(arg1, *, arg2)" case)
+ result.append('*')
+ # This condition should be only triggered once, so
+ # reset the flag
+ render_kw_only_separator = False
+
+ result.append(formatted)
+
+ rendered = '({0})'.format(', '.join(result))
+
+ if self.return_annotation is not _empty:
+ anno = formatannotation(self.return_annotation)
+ rendered += ' -> {0}'.format(anno)
+
+ return rendered
+
diff --git a/contrib/python/ipython/py2/IPython/utils/_sysinfo.py b/contrib/python/ipython/py2/IPython/utils/_sysinfo.py
index 21dd2fcceb..f3422bbb3c 100644
--- a/contrib/python/ipython/py2/IPython/utils/_sysinfo.py
+++ b/contrib/python/ipython/py2/IPython/utils/_sysinfo.py
@@ -1,2 +1,2 @@
-# GENERATED BY setup.py
+# GENERATED BY setup.py
commit = u"2348ebbe4"
diff --git a/contrib/python/ipython/py2/IPython/utils/_tokenize_py2.py b/contrib/python/ipython/py2/IPython/utils/_tokenize_py2.py
index 195df96ee5..ffd7cc5e71 100644
--- a/contrib/python/ipython/py2/IPython/utils/_tokenize_py2.py
+++ b/contrib/python/ipython/py2/IPython/utils/_tokenize_py2.py
@@ -1,439 +1,439 @@
-"""Patched version of standard library tokenize, to deal with various bugs.
-
-Patches
-
-- Relevant parts of Gareth Rees' patch for Python issue #12691 (untokenizing),
- manually applied.
-- Newlines in comments and blank lines should be either NL or NEWLINE, depending
- on whether they are in a multi-line statement. Filed as Python issue #17061.
-
--------------------------------------------------------------------------------
-Tokenization help for Python programs.
-
-generate_tokens(readline) is a generator that breaks a stream of
-text into Python tokens. It accepts a readline-like method which is called
-repeatedly to get the next line of input (or "" for EOF). It generates
-5-tuples with these members:
-
- the token type (see token.py)
- the token (a string)
- the starting (row, column) indices of the token (a 2-tuple of ints)
- the ending (row, column) indices of the token (a 2-tuple of ints)
- the original line (string)
-
-It is designed to match the working of the Python tokenizer exactly, except
-that it produces COMMENT tokens for comments and gives type OP for all
-operators
-
-Older entry points
- tokenize_loop(readline, tokeneater)
- tokenize(readline, tokeneater=printtoken)
-are the same, except instead of generating tokens, tokeneater is a callback
-function to which the 5 fields described above are passed as 5 arguments,
-each time a new token is found."""
-from __future__ import print_function
-
-__author__ = 'Ka-Ping Yee <ping@lfw.org>'
-__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
- 'Skip Montanaro, Raymond Hettinger')
-
-import string, re
-from token import *
-
-import token
-__all__ = [x for x in dir(token) if not x.startswith("_")]
-__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
-del x
-del token
-
-__all__ += ["TokenError"]
-
-COMMENT = N_TOKENS
-tok_name[COMMENT] = 'COMMENT'
-NL = N_TOKENS + 1
-tok_name[NL] = 'NL'
-N_TOKENS += 2
-
-def group(*choices): return '(' + '|'.join(choices) + ')'
-def any(*choices): return group(*choices) + '*'
-def maybe(*choices): return group(*choices) + '?'
-
-Whitespace = r'[ \f\t]*'
-Comment = r'#[^\r\n]*'
-Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'[a-zA-Z_]\w*'
-
-Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
-Binnumber = r'0[bB][01]+[lL]?'
-Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?\d+'
-Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
-Expfloat = r'\d+' + Exponent
-Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
-Number = group(Imagnumber, Floatnumber, Intnumber)
-
-# Tail end of ' string.
-Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
-# Tail end of " string.
-Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
-# Tail end of ''' string.
-Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
-# Tail end of """ string.
-Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
-# Single-line ' or " string.
-String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
- r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
-
-# Because of leftmost-then-longest match semantics, be sure to put the
-# longest operators first (e.g., if = came before ==, == would get
-# recognized as two instances of =).
-Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
- r"//=?",
- r"[+\-*/%&|^=<>]=?",
- r"~")
-
-Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'[:;.,`@]')
-Funny = group(Operator, Bracket, Special)
-
-PlainToken = group(Number, Funny, String, Name)
-Token = Ignore + PlainToken
-
-# First (or only) line of ' or " string.
-ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
- group("'", r'\\\r?\n'),
- r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
- group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
-PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-
-tokenprog, pseudoprog, single3prog, double3prog = map(
- re.compile, (Token, PseudoToken, Single3, Double3))
-endprogs = {"'": re.compile(Single), '"': re.compile(Double),
- "'''": single3prog, '"""': double3prog,
- "r'''": single3prog, 'r"""': double3prog,
- "u'''": single3prog, 'u"""': double3prog,
- "ur'''": single3prog, 'ur"""': double3prog,
- "R'''": single3prog, 'R"""': double3prog,
- "U'''": single3prog, 'U"""': double3prog,
- "uR'''": single3prog, 'uR"""': double3prog,
- "Ur'''": single3prog, 'Ur"""': double3prog,
- "UR'''": single3prog, 'UR"""': double3prog,
- "b'''": single3prog, 'b"""': double3prog,
- "br'''": single3prog, 'br"""': double3prog,
- "B'''": single3prog, 'B"""': double3prog,
- "bR'''": single3prog, 'bR"""': double3prog,
- "Br'''": single3prog, 'Br"""': double3prog,
- "BR'''": single3prog, 'BR"""': double3prog,
- 'r': None, 'R': None, 'u': None, 'U': None,
- 'b': None, 'B': None}
-
-triple_quoted = {}
-for t in ("'''", '"""',
- "r'''", 'r"""', "R'''", 'R"""',
- "u'''", 'u"""', "U'''", 'U"""',
- "ur'''", 'ur"""', "Ur'''", 'Ur"""',
- "uR'''", 'uR"""', "UR'''", 'UR"""',
- "b'''", 'b"""', "B'''", 'B"""',
- "br'''", 'br"""', "Br'''", 'Br"""',
- "bR'''", 'bR"""', "BR'''", 'BR"""'):
- triple_quoted[t] = t
-single_quoted = {}
-for t in ("'", '"',
- "r'", 'r"', "R'", 'R"',
- "u'", 'u"', "U'", 'U"',
- "ur'", 'ur"', "Ur'", 'Ur"',
- "uR'", 'uR"', "UR'", 'UR"',
- "b'", 'b"', "B'", 'B"',
- "br'", 'br"', "Br'", 'Br"',
- "bR'", 'bR"', "BR'", 'BR"' ):
- single_quoted[t] = t
-
-tabsize = 8
-
-class TokenError(Exception): pass
-
-class StopTokenizing(Exception): pass
-
-def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
- srow, scol = srow_scol
- erow, ecol = erow_ecol
- print("%d,%d-%d,%d:\t%s\t%s" % \
- (srow, scol, erow, ecol, tok_name[type], repr(token)))
-
-def tokenize(readline, tokeneater=printtoken):
- """
- The tokenize() function accepts two parameters: one representing the
- input stream, and one providing an output mechanism for tokenize().
-
- The first parameter, readline, must be a callable object which provides
- the same interface as the readline() method of built-in file objects.
- Each call to the function should return one line of input as a string.
-
- The second parameter, tokeneater, must also be a callable object. It is
- called once for each token, with five arguments, corresponding to the
- tuples generated by generate_tokens().
- """
- try:
- tokenize_loop(readline, tokeneater)
- except StopTokenizing:
- pass
-
-# backwards compatible interface
-def tokenize_loop(readline, tokeneater):
- for token_info in generate_tokens(readline):
- tokeneater(*token_info)
-
-class Untokenizer:
-
- def __init__(self):
- self.tokens = []
- self.prev_row = 1
- self.prev_col = 0
-
- def add_whitespace(self, start):
- row, col = start
- assert row >= self.prev_row
- col_offset = col - self.prev_col
- if col_offset > 0:
- self.tokens.append(" " * col_offset)
- elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
- # Line was backslash-continued
- self.tokens.append(" ")
-
- def untokenize(self, tokens):
- iterable = iter(tokens)
- for t in iterable:
- if len(t) == 2:
- self.compat(t, iterable)
- break
- tok_type, token, start, end = t[:4]
- self.add_whitespace(start)
- self.tokens.append(token)
- self.prev_row, self.prev_col = end
- if tok_type in (NEWLINE, NL):
- self.prev_row += 1
- self.prev_col = 0
- return "".join(self.tokens)
-
- def compat(self, token, iterable):
- # This import is here to avoid problems when the itertools
- # module is not built yet and tokenize is imported.
- from itertools import chain
- startline = False
- prevstring = False
- indents = []
- toks_append = self.tokens.append
- for tok in chain([token], iterable):
- toknum, tokval = tok[:2]
-
- if toknum in (NAME, NUMBER):
- tokval += ' '
-
- # Insert a space between two consecutive strings
- if toknum == STRING:
- if prevstring:
- tokval = ' ' + tokval
- prevstring = True
- else:
- prevstring = False
-
- if toknum == INDENT:
- indents.append(tokval)
- continue
- elif toknum == DEDENT:
- indents.pop()
- continue
- elif toknum in (NEWLINE, NL):
- startline = True
- elif startline and indents:
- toks_append(indents[-1])
- startline = False
- toks_append(tokval)
-
-def untokenize(iterable):
- """Transform tokens back into Python source code.
-
- Each element returned by the iterable must be a token sequence
- with at least two elements, a token number and token value. If
- only two tokens are passed, the resulting output is poor.
-
- Round-trip invariant for full input:
- Untokenized source will match input source exactly
-
- Round-trip invariant for limited intput:
- # Output text will tokenize the back to the input
- t1 = [tok[:2] for tok in generate_tokens(f.readline)]
- newcode = untokenize(t1)
- readline = iter(newcode.splitlines(1)).next
- t2 = [tok[:2] for tok in generate_tokens(readline)]
- assert t1 == t2
- """
- ut = Untokenizer()
- return ut.untokenize(iterable)
-
-def generate_tokens(readline):
- """
- The generate_tokens() generator requires one argment, readline, which
- must be a callable object which provides the same interface as the
- readline() method of built-in file objects. Each call to the function
- should return one line of input as a string. Alternately, readline
- can be a callable function terminating with StopIteration:
- readline = open(myfile).next # Example of alternate readline
-
- The generator produces 5-tuples with these members: the token type; the
- token string; a 2-tuple (srow, scol) of ints specifying the row and
- column where the token begins in the source; a 2-tuple (erow, ecol) of
- ints specifying the row and column where the token ends in the source;
- and the line on which the token was found. The line passed is the
- logical line; continuation lines are included.
- """
- lnum = parenlev = continued = 0
- namechars, numchars = string.ascii_letters + '_', '0123456789'
- contstr, needcont = '', 0
- contline = None
- indents = [0]
-
- while 1: # loop over lines in stream
- try:
- line = readline()
- except StopIteration:
- line = ''
- lnum += 1
- pos, max = 0, len(line)
-
- if contstr: # continued string
- if not line:
- raise TokenError("EOF in multi-line string", strstart)
- endmatch = endprog.match(line)
- if endmatch:
- pos = end = endmatch.end(0)
- yield (STRING, contstr + line[:end],
- strstart, (lnum, end), contline + line)
- contstr, needcont = '', 0
- contline = None
- elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
- yield (ERRORTOKEN, contstr + line,
- strstart, (lnum, len(line)), contline)
- contstr = ''
- contline = None
- continue
- else:
- contstr = contstr + line
- contline = contline + line
- continue
-
- elif parenlev == 0 and not continued: # new statement
- if not line: break
- column = 0
- while pos < max: # measure leading whitespace
- if line[pos] == ' ':
- column += 1
- elif line[pos] == '\t':
- column = (column//tabsize + 1)*tabsize
- elif line[pos] == '\f':
- column = 0
- else:
- break
- pos += 1
- if pos == max:
- break
-
- if line[pos] in '#\r\n': # skip comments or blank lines
- if line[pos] == '#':
- comment_token = line[pos:].rstrip('\r\n')
- nl_pos = pos + len(comment_token)
- yield (COMMENT, comment_token,
- (lnum, pos), (lnum, pos + len(comment_token)), line)
- yield (NEWLINE, line[nl_pos:],
- (lnum, nl_pos), (lnum, len(line)), line)
- else:
- yield (NEWLINE, line[pos:],
- (lnum, pos), (lnum, len(line)), line)
- continue
-
- if column > indents[-1]: # count indents or dedents
- indents.append(column)
- yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
- while column < indents[-1]:
- if column not in indents:
- raise IndentationError(
- "unindent does not match any outer indentation level",
- ("<tokenize>", lnum, pos, line))
- indents = indents[:-1]
- yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
-
- else: # continued statement
- if not line:
- raise TokenError("EOF in multi-line statement", (lnum, 0))
- continued = 0
-
- while pos < max:
- pseudomatch = pseudoprog.match(line, pos)
- if pseudomatch: # scan for tokens
- start, end = pseudomatch.span(1)
- spos, epos, pos = (lnum, start), (lnum, end), end
- token, initial = line[start:end], line[start]
-
- if initial in numchars or \
- (initial == '.' and token != '.'): # ordinary number
- yield (NUMBER, token, spos, epos, line)
- elif initial in '\r\n':
- yield (NL if parenlev > 0 else NEWLINE,
- token, spos, epos, line)
- elif initial == '#':
- assert not token.endswith("\n")
- yield (COMMENT, token, spos, epos, line)
- elif token in triple_quoted:
- endprog = endprogs[token]
- endmatch = endprog.match(line, pos)
- if endmatch: # all on one line
- pos = endmatch.end(0)
- token = line[start:pos]
- yield (STRING, token, spos, (lnum, pos), line)
- else:
- strstart = (lnum, start) # multiple lines
- contstr = line[start:]
- contline = line
- break
- elif initial in single_quoted or \
- token[:2] in single_quoted or \
- token[:3] in single_quoted:
- if token[-1] == '\n': # continued string
- strstart = (lnum, start)
- endprog = (endprogs[initial] or endprogs[token[1]] or
- endprogs[token[2]])
- contstr, needcont = line[start:], 1
- contline = line
- break
- else: # ordinary string
- yield (STRING, token, spos, epos, line)
- elif initial in namechars: # ordinary name
- yield (NAME, token, spos, epos, line)
- elif initial == '\\': # continued stmt
- continued = 1
- else:
- if initial in '([{':
- parenlev += 1
- elif initial in ')]}':
- parenlev -= 1
- yield (OP, token, spos, epos, line)
- else:
- yield (ERRORTOKEN, line[pos],
- (lnum, pos), (lnum, pos+1), line)
- pos += 1
-
- for indent in indents[1:]: # pop remaining indent levels
- yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
- yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
-
-if __name__ == '__main__': # testing
- import sys
- if len(sys.argv) > 1:
- tokenize(open(sys.argv[1]).readline)
- else:
- tokenize(sys.stdin.readline)
+"""Patched version of standard library tokenize, to deal with various bugs.
+
+Patches
+
+- Relevant parts of Gareth Rees' patch for Python issue #12691 (untokenizing),
+ manually applied.
+- Newlines in comments and blank lines should be either NL or NEWLINE, depending
+ on whether they are in a multi-line statement. Filed as Python issue #17061.
+
+-------------------------------------------------------------------------------
+Tokenization help for Python programs.
+
+generate_tokens(readline) is a generator that breaks a stream of
+text into Python tokens. It accepts a readline-like method which is called
+repeatedly to get the next line of input (or "" for EOF). It generates
+5-tuples with these members:
+
+ the token type (see token.py)
+ the token (a string)
+ the starting (row, column) indices of the token (a 2-tuple of ints)
+ the ending (row, column) indices of the token (a 2-tuple of ints)
+ the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators
+
+Older entry points
+ tokenize_loop(readline, tokeneater)
+ tokenize(readline, tokeneater=printtoken)
+are the same, except instead of generating tokens, tokeneater is a callback
+function to which the 5 fields described above are passed as 5 arguments,
+each time a new token is found."""
+from __future__ import print_function
+
+__author__ = 'Ka-Ping Yee <ping@lfw.org>'
+__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
+ 'Skip Montanaro, Raymond Hettinger')
+
+import string, re
+from token import *
+
+import token
+__all__ = [x for x in dir(token) if not x.startswith("_")]
+__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
+del x
+del token
+
+__all__ += ["TokenError"]
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+N_TOKENS += 2
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'[a-zA-Z_]\w*'
+
+Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
+Decnumber = r'[1-9]\d*[lL]?'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?\d+'
+Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
+Expfloat = r'\d+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
+# Single-line ' or " string.
+String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+ r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
+ r"//=?",
+ r"[+\-*/%&|^=<>]=?",
+ r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'[:;.,`@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+ group("'", r'\\\r?\n'),
+ r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+ group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+ re.compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": re.compile(Single), '"': re.compile(Double),
+ "'''": single3prog, '"""': double3prog,
+ "r'''": single3prog, 'r"""': double3prog,
+ "u'''": single3prog, 'u"""': double3prog,
+ "ur'''": single3prog, 'ur"""': double3prog,
+ "R'''": single3prog, 'R"""': double3prog,
+ "U'''": single3prog, 'U"""': double3prog,
+ "uR'''": single3prog, 'uR"""': double3prog,
+ "Ur'''": single3prog, 'Ur"""': double3prog,
+ "UR'''": single3prog, 'UR"""': double3prog,
+ "b'''": single3prog, 'b"""': double3prog,
+ "br'''": single3prog, 'br"""': double3prog,
+ "B'''": single3prog, 'B"""': double3prog,
+ "bR'''": single3prog, 'bR"""': double3prog,
+ "Br'''": single3prog, 'Br"""': double3prog,
+ "BR'''": single3prog, 'BR"""': double3prog,
+ 'r': None, 'R': None, 'u': None, 'U': None,
+ 'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+ "r'''", 'r"""', "R'''", 'R"""',
+ "u'''", 'u"""', "U'''", 'U"""',
+ "ur'''", 'ur"""', "Ur'''", 'Ur"""',
+ "uR'''", 'uR"""', "UR'''", 'UR"""',
+ "b'''", 'b"""', "B'''", 'B"""',
+ "br'''", 'br"""', "Br'''", 'Br"""',
+ "bR'''", 'bR"""', "BR'''", 'BR"""'):
+ triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+ "r'", 'r"', "R'", 'R"',
+ "u'", 'u"', "U'", 'U"',
+ "ur'", 'ur"', "Ur'", 'Ur"',
+ "uR'", 'uR"', "UR'", 'UR"',
+ "b'", 'b"', "B'", 'B"',
+ "br'", 'br"', "Br'", 'Br"',
+ "bR'", 'bR"', "BR'", 'BR"' ):
+ single_quoted[t] = t
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+class StopTokenizing(Exception): pass
+
+def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
+ srow, scol = srow_scol
+ erow, ecol = erow_ecol
+ print("%d,%d-%d,%d:\t%s\t%s" % \
+ (srow, scol, erow, ecol, tok_name[type], repr(token)))
+
+def tokenize(readline, tokeneater=printtoken):
+ """
+ The tokenize() function accepts two parameters: one representing the
+ input stream, and one providing an output mechanism for tokenize().
+
+ The first parameter, readline, must be a callable object which provides
+ the same interface as the readline() method of built-in file objects.
+ Each call to the function should return one line of input as a string.
+
+ The second parameter, tokeneater, must also be a callable object. It is
+ called once for each token, with five arguments, corresponding to the
+ tuples generated by generate_tokens().
+ """
+ try:
+ tokenize_loop(readline, tokeneater)
+ except StopTokenizing:
+ pass
+
+# backwards compatible interface
+def tokenize_loop(readline, tokeneater):
+ for token_info in generate_tokens(readline):
+ tokeneater(*token_info)
+
+class Untokenizer:
+
+ def __init__(self):
+ self.tokens = []
+ self.prev_row = 1
+ self.prev_col = 0
+
+ def add_whitespace(self, start):
+ row, col = start
+ assert row >= self.prev_row
+ col_offset = col - self.prev_col
+ if col_offset > 0:
+ self.tokens.append(" " * col_offset)
+ elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
+ # Line was backslash-continued
+ self.tokens.append(" ")
+
+ def untokenize(self, tokens):
+ iterable = iter(tokens)
+ for t in iterable:
+ if len(t) == 2:
+ self.compat(t, iterable)
+ break
+ tok_type, token, start, end = t[:4]
+ self.add_whitespace(start)
+ self.tokens.append(token)
+ self.prev_row, self.prev_col = end
+ if tok_type in (NEWLINE, NL):
+ self.prev_row += 1
+ self.prev_col = 0
+ return "".join(self.tokens)
+
+ def compat(self, token, iterable):
+ # This import is here to avoid problems when the itertools
+ # module is not built yet and tokenize is imported.
+ from itertools import chain
+ startline = False
+ prevstring = False
+ indents = []
+ toks_append = self.tokens.append
+ for tok in chain([token], iterable):
+ toknum, tokval = tok[:2]
+
+ if toknum in (NAME, NUMBER):
+ tokval += ' '
+
+ # Insert a space between two consecutive strings
+ if toknum == STRING:
+ if prevstring:
+ tokval = ' ' + tokval
+ prevstring = True
+ else:
+ prevstring = False
+
+ if toknum == INDENT:
+ indents.append(tokval)
+ continue
+ elif toknum == DEDENT:
+ indents.pop()
+ continue
+ elif toknum in (NEWLINE, NL):
+ startline = True
+ elif startline and indents:
+ toks_append(indents[-1])
+ startline = False
+ toks_append(tokval)
+
+def untokenize(iterable):
+ """Transform tokens back into Python source code.
+
+ Each element returned by the iterable must be a token sequence
+ with at least two elements, a token number and token value. If
+ only two tokens are passed, the resulting output is poor.
+
+ Round-trip invariant for full input:
+ Untokenized source will match input source exactly
+
+ Round-trip invariant for limited intput:
+ # Output text will tokenize the back to the input
+ t1 = [tok[:2] for tok in generate_tokens(f.readline)]
+ newcode = untokenize(t1)
+ readline = iter(newcode.splitlines(1)).next
+ t2 = [tok[:2] for tok in generate_tokens(readline)]
+ assert t1 == t2
+ """
+ ut = Untokenizer()
+ return ut.untokenize(iterable)
+
+def generate_tokens(readline):
+ """
+ The generate_tokens() generator requires one argment, readline, which
+ must be a callable object which provides the same interface as the
+ readline() method of built-in file objects. Each call to the function
+ should return one line of input as a string. Alternately, readline
+ can be a callable function terminating with StopIteration:
+ readline = open(myfile).next # Example of alternate readline
+
+ The generator produces 5-tuples with these members: the token type; the
+ token string; a 2-tuple (srow, scol) of ints specifying the row and
+ column where the token begins in the source; a 2-tuple (erow, ecol) of
+ ints specifying the row and column where the token ends in the source;
+ and the line on which the token was found. The line passed is the
+ logical line; continuation lines are included.
+ """
+ lnum = parenlev = continued = 0
+ namechars, numchars = string.ascii_letters + '_', '0123456789'
+ contstr, needcont = '', 0
+ contline = None
+ indents = [0]
+
+ while 1: # loop over lines in stream
+ try:
+ line = readline()
+ except StopIteration:
+ line = ''
+ lnum += 1
+ pos, max = 0, len(line)
+
+ if contstr: # continued string
+ if not line:
+ raise TokenError("EOF in multi-line string", strstart)
+ endmatch = endprog.match(line)
+ if endmatch:
+ pos = end = endmatch.end(0)
+ yield (STRING, contstr + line[:end],
+ strstart, (lnum, end), contline + line)
+ contstr, needcont = '', 0
+ contline = None
+ elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+ yield (ERRORTOKEN, contstr + line,
+ strstart, (lnum, len(line)), contline)
+ contstr = ''
+ contline = None
+ continue
+ else:
+ contstr = contstr + line
+ contline = contline + line
+ continue
+
+ elif parenlev == 0 and not continued: # new statement
+ if not line: break
+ column = 0
+ while pos < max: # measure leading whitespace
+ if line[pos] == ' ':
+ column += 1
+ elif line[pos] == '\t':
+ column = (column//tabsize + 1)*tabsize
+ elif line[pos] == '\f':
+ column = 0
+ else:
+ break
+ pos += 1
+ if pos == max:
+ break
+
+ if line[pos] in '#\r\n': # skip comments or blank lines
+ if line[pos] == '#':
+ comment_token = line[pos:].rstrip('\r\n')
+ nl_pos = pos + len(comment_token)
+ yield (COMMENT, comment_token,
+ (lnum, pos), (lnum, pos + len(comment_token)), line)
+ yield (NEWLINE, line[nl_pos:],
+ (lnum, nl_pos), (lnum, len(line)), line)
+ else:
+ yield (NEWLINE, line[pos:],
+ (lnum, pos), (lnum, len(line)), line)
+ continue
+
+ if column > indents[-1]: # count indents or dedents
+ indents.append(column)
+ yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+ while column < indents[-1]:
+ if column not in indents:
+ raise IndentationError(
+ "unindent does not match any outer indentation level",
+ ("<tokenize>", lnum, pos, line))
+ indents = indents[:-1]
+ yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+ else: # continued statement
+ if not line:
+ raise TokenError("EOF in multi-line statement", (lnum, 0))
+ continued = 0
+
+ while pos < max:
+ pseudomatch = pseudoprog.match(line, pos)
+ if pseudomatch: # scan for tokens
+ start, end = pseudomatch.span(1)
+ spos, epos, pos = (lnum, start), (lnum, end), end
+ token, initial = line[start:end], line[start]
+
+ if initial in numchars or \
+ (initial == '.' and token != '.'): # ordinary number
+ yield (NUMBER, token, spos, epos, line)
+ elif initial in '\r\n':
+ yield (NL if parenlev > 0 else NEWLINE,
+ token, spos, epos, line)
+ elif initial == '#':
+ assert not token.endswith("\n")
+ yield (COMMENT, token, spos, epos, line)
+ elif token in triple_quoted:
+ endprog = endprogs[token]
+ endmatch = endprog.match(line, pos)
+ if endmatch: # all on one line
+ pos = endmatch.end(0)
+ token = line[start:pos]
+ yield (STRING, token, spos, (lnum, pos), line)
+ else:
+ strstart = (lnum, start) # multiple lines
+ contstr = line[start:]
+ contline = line
+ break
+ elif initial in single_quoted or \
+ token[:2] in single_quoted or \
+ token[:3] in single_quoted:
+ if token[-1] == '\n': # continued string
+ strstart = (lnum, start)
+ endprog = (endprogs[initial] or endprogs[token[1]] or
+ endprogs[token[2]])
+ contstr, needcont = line[start:], 1
+ contline = line
+ break
+ else: # ordinary string
+ yield (STRING, token, spos, epos, line)
+ elif initial in namechars: # ordinary name
+ yield (NAME, token, spos, epos, line)
+ elif initial == '\\': # continued stmt
+ continued = 1
+ else:
+ if initial in '([{':
+ parenlev += 1
+ elif initial in ')]}':
+ parenlev -= 1
+ yield (OP, token, spos, epos, line)
+ else:
+ yield (ERRORTOKEN, line[pos],
+ (lnum, pos), (lnum, pos+1), line)
+ pos += 1
+
+ for indent in indents[1:]: # pop remaining indent levels
+ yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
+ yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+
+if __name__ == '__main__': # testing
+ import sys
+ if len(sys.argv) > 1:
+ tokenize(open(sys.argv[1]).readline)
+ else:
+ tokenize(sys.stdin.readline)
diff --git a/contrib/python/ipython/py2/IPython/utils/_tokenize_py3.py b/contrib/python/ipython/py2/IPython/utils/_tokenize_py3.py
index ee1fd9e639..ca85023c32 100644
--- a/contrib/python/ipython/py2/IPython/utils/_tokenize_py3.py
+++ b/contrib/python/ipython/py2/IPython/utils/_tokenize_py3.py
@@ -1,595 +1,595 @@
-"""Patched version of standard library tokenize, to deal with various bugs.
-
-Based on Python 3.2 code.
-
-Patches:
-
-- Gareth Rees' patch for Python issue #12691 (untokenizing)
- - Except we don't encode the output of untokenize
- - Python 2 compatible syntax, so that it can be byte-compiled at installation
-- Newlines in comments and blank lines should be either NL or NEWLINE, depending
- on whether they are in a multi-line statement. Filed as Python issue #17061.
-- Export generate_tokens & TokenError
-- u and rb literals are allowed under Python 3.3 and above.
-
-------------------------------------------------------------------------------
-Tokenization help for Python programs.
-
-tokenize(readline) is a generator that breaks a stream of bytes into
-Python tokens. It decodes the bytes according to PEP-0263 for
-determining source file encoding.
-
-It accepts a readline-like method which is called repeatedly to get the
-next line of input (or b"" for EOF). It generates 5-tuples with these
-members:
-
- the token type (see token.py)
- the token (a string)
- the starting (row, column) indices of the token (a 2-tuple of ints)
- the ending (row, column) indices of the token (a 2-tuple of ints)
- the original line (string)
-
-It is designed to match the working of the Python tokenizer exactly, except
-that it produces COMMENT tokens for comments and gives type OP for all
-operators. Additionally, all token lists start with an ENCODING token
-which tells you which encoding was used to decode the bytes stream.
-"""
-from __future__ import absolute_import
-
-__author__ = 'Ka-Ping Yee <ping@lfw.org>'
-__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
- 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
- 'Michael Foord')
-import builtins
-import re
-import sys
-from token import *
-from codecs import lookup, BOM_UTF8
-import collections
-from io import TextIOWrapper
-cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
-
-import token
-__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
- "NL", "untokenize", "ENCODING", "TokenInfo"]
-del token
-
-__all__ += ["generate_tokens", "TokenError"]
-
-COMMENT = N_TOKENS
-tok_name[COMMENT] = 'COMMENT'
-NL = N_TOKENS + 1
-tok_name[NL] = 'NL'
-ENCODING = N_TOKENS + 2
-tok_name[ENCODING] = 'ENCODING'
-N_TOKENS += 3
-
-class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
- def __repr__(self):
- annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
- return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
- self._replace(type=annotated_type))
-
-def group(*choices): return '(' + '|'.join(choices) + ')'
-def any(*choices): return group(*choices) + '*'
-def maybe(*choices): return group(*choices) + '?'
-
-# Note: we use unicode matching for names ("\w") but ascii matching for
-# number literals.
-Whitespace = r'[ \f\t]*'
-Comment = r'#[^\r\n]*'
-Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'\w+'
-
-Hexnumber = r'0[xX][0-9a-fA-F]+'
-Binnumber = r'0[bB][01]+'
-Octnumber = r'0[oO][0-7]+'
-Decnumber = r'(?:0+|[1-9][0-9]*)'
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?[0-9]+'
-Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
-Expfloat = r'[0-9]+' + Exponent
-Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
-Number = group(Imagnumber, Floatnumber, Intnumber)
-
-if sys.version_info.minor >= 3:
- StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
-else:
- StringPrefix = r'(?:[bB]?[rR]?)?'
-
-# Tail end of ' string.
-Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
-# Tail end of " string.
-Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
-# Tail end of ''' string.
-Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
-# Tail end of """ string.
-Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group(StringPrefix + "'''", StringPrefix + '"""')
-# Single-line ' or " string.
-String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
- StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
-
-# Because of leftmost-then-longest match semantics, be sure to put the
-# longest operators first (e.g., if = came before ==, == would get
-# recognized as two instances of =).
-Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
- r"//=?", r"->",
- r"[+\-*/%&|^=<>]=?",
- r"~")
-
-Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
-Funny = group(Operator, Bracket, Special)
-
-PlainToken = group(Number, Funny, String, Name)
-Token = Ignore + PlainToken
-
-# First (or only) line of ' or " string.
-ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
- group("'", r'\\\r?\n'),
- StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
- group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
-PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-
-def _compile(expr):
- return re.compile(expr, re.UNICODE)
-
-tokenprog, pseudoprog, single3prog, double3prog = map(
- _compile, (Token, PseudoToken, Single3, Double3))
-endprogs = {"'": _compile(Single), '"': _compile(Double),
- "'''": single3prog, '"""': double3prog,
- "r'''": single3prog, 'r"""': double3prog,
- "b'''": single3prog, 'b"""': double3prog,
- "R'''": single3prog, 'R"""': double3prog,
- "B'''": single3prog, 'B"""': double3prog,
- "br'''": single3prog, 'br"""': double3prog,
- "bR'''": single3prog, 'bR"""': double3prog,
- "Br'''": single3prog, 'Br"""': double3prog,
- "BR'''": single3prog, 'BR"""': double3prog,
- 'r': None, 'R': None, 'b': None, 'B': None}
-
-triple_quoted = {}
-for t in ("'''", '"""',
- "r'''", 'r"""', "R'''", 'R"""',
- "b'''", 'b"""', "B'''", 'B"""',
- "br'''", 'br"""', "Br'''", 'Br"""',
- "bR'''", 'bR"""', "BR'''", 'BR"""'):
- triple_quoted[t] = t
-single_quoted = {}
-for t in ("'", '"',
- "r'", 'r"', "R'", 'R"',
- "b'", 'b"', "B'", 'B"',
- "br'", 'br"', "Br'", 'Br"',
- "bR'", 'bR"', "BR'", 'BR"' ):
- single_quoted[t] = t
-
-if sys.version_info.minor >= 3:
- # Python 3.3
- for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
- _t2 = _prefix+'"""'
- endprogs[_t2] = double3prog
- triple_quoted[_t2] = _t2
- _t1 = _prefix + "'''"
- endprogs[_t1] = single3prog
- triple_quoted[_t1] = _t1
- single_quoted[_prefix+'"'] = _prefix+'"'
- single_quoted[_prefix+"'"] = _prefix+"'"
- del _prefix, _t2, _t1
- endprogs['u'] = None
- endprogs['U'] = None
-
-del _compile
-
-tabsize = 8
-
-class TokenError(Exception): pass
-
-class StopTokenizing(Exception): pass
-
-
-class Untokenizer:
-
- def __init__(self):
- self.tokens = []
- self.prev_row = 1
- self.prev_col = 0
- self.encoding = 'utf-8'
-
- def add_whitespace(self, tok_type, start):
- row, col = start
- assert row >= self.prev_row
- col_offset = col - self.prev_col
- if col_offset > 0:
- self.tokens.append(" " * col_offset)
- elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
- # Line was backslash-continued.
- self.tokens.append(" ")
-
- def untokenize(self, tokens):
- iterable = iter(tokens)
- for t in iterable:
- if len(t) == 2:
- self.compat(t, iterable)
- break
- tok_type, token, start, end = t[:4]
- if tok_type == ENCODING:
- self.encoding = token
- continue
- self.add_whitespace(tok_type, start)
- self.tokens.append(token)
- self.prev_row, self.prev_col = end
- if tok_type in (NEWLINE, NL):
- self.prev_row += 1
- self.prev_col = 0
- return "".join(self.tokens)
-
- def compat(self, token, iterable):
- # This import is here to avoid problems when the itertools
- # module is not built yet and tokenize is imported.
- from itertools import chain
- startline = False
- prevstring = False
- indents = []
- toks_append = self.tokens.append
-
- for tok in chain([token], iterable):
- toknum, tokval = tok[:2]
- if toknum == ENCODING:
- self.encoding = tokval
- continue
-
- if toknum in (NAME, NUMBER):
- tokval += ' '
-
- # Insert a space between two consecutive strings
- if toknum == STRING:
- if prevstring:
- tokval = ' ' + tokval
- prevstring = True
- else:
- prevstring = False
-
- if toknum == INDENT:
- indents.append(tokval)
- continue
- elif toknum == DEDENT:
- indents.pop()
- continue
- elif toknum in (NEWLINE, NL):
- startline = True
- elif startline and indents:
- toks_append(indents[-1])
- startline = False
- toks_append(tokval)
-
-
-def untokenize(tokens):
- """
- Convert ``tokens`` (an iterable) back into Python source code. Return
- a bytes object, encoded using the encoding specified by the last
- ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
-
- The result is guaranteed to tokenize back to match the input so that
- the conversion is lossless and round-trips are assured. The
- guarantee applies only to the token type and token string as the
- spacing between tokens (column positions) may change.
-
- :func:`untokenize` has two modes. If the input tokens are sequences
- of length 2 (``type``, ``string``) then spaces are added as necessary to
- preserve the round-trip property.
-
- If the input tokens are sequences of length 4 or more (``type``,
- ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
- spaces are added so that each token appears in the result at the
- position indicated by ``start`` and ``end``, if possible.
- """
- return Untokenizer().untokenize(tokens)
-
-
-def _get_normal_name(orig_enc):
- """Imitates get_normal_name in tokenizer.c."""
- # Only care about the first 12 characters.
- enc = orig_enc[:12].lower().replace("_", "-")
- if enc == "utf-8" or enc.startswith("utf-8-"):
- return "utf-8"
- if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
- enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
- return "iso-8859-1"
- return orig_enc
-
-def detect_encoding(readline):
- """
- The detect_encoding() function is used to detect the encoding that should
- be used to decode a Python source file. It requires one argment, readline,
- in the same way as the tokenize() generator.
-
- It will call readline a maximum of twice, and return the encoding used
- (as a string) and a list of any lines (left as bytes) it has read in.
-
- It detects the encoding from the presence of a utf-8 bom or an encoding
- cookie as specified in pep-0263. If both a bom and a cookie are present,
- but disagree, a SyntaxError will be raised. If the encoding cookie is an
- invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
- 'utf-8-sig' is returned.
-
- If no encoding is specified, then the default of 'utf-8' will be returned.
- """
- bom_found = False
- encoding = None
- default = 'utf-8'
- def read_or_stop():
- try:
- return readline()
- except StopIteration:
- return b''
-
- def find_cookie(line):
- try:
- # Decode as UTF-8. Either the line is an encoding declaration,
- # in which case it should be pure ASCII, or it must be UTF-8
- # per default encoding.
- line_string = line.decode('utf-8')
- except UnicodeDecodeError:
- raise SyntaxError("invalid or missing encoding declaration")
-
- matches = cookie_re.findall(line_string)
- if not matches:
- return None
- encoding = _get_normal_name(matches[0])
- try:
- codec = lookup(encoding)
- except LookupError:
- # This behaviour mimics the Python interpreter
- raise SyntaxError("unknown encoding: " + encoding)
-
- if bom_found:
- if encoding != 'utf-8':
- # This behaviour mimics the Python interpreter
- raise SyntaxError('encoding problem: utf-8')
- encoding += '-sig'
- return encoding
-
- first = read_or_stop()
- if first.startswith(BOM_UTF8):
- bom_found = True
- first = first[3:]
- default = 'utf-8-sig'
- if not first:
- return default, []
-
- encoding = find_cookie(first)
- if encoding:
- return encoding, [first]
-
- second = read_or_stop()
- if not second:
- return default, [first]
-
- encoding = find_cookie(second)
- if encoding:
- return encoding, [first, second]
-
- return default, [first, second]
-
-
-def open(filename):
- """Open a file in read only mode using the encoding detected by
- detect_encoding().
- """
- buffer = builtins.open(filename, 'rb')
- encoding, lines = detect_encoding(buffer.readline)
- buffer.seek(0)
- text = TextIOWrapper(buffer, encoding, line_buffering=True)
- text.mode = 'r'
- return text
-
-
-def tokenize(readline):
- """
- The tokenize() generator requires one argment, readline, which
- must be a callable object which provides the same interface as the
- readline() method of built-in file objects. Each call to the function
- should return one line of input as bytes. Alternately, readline
- can be a callable function terminating with StopIteration:
- readline = open(myfile, 'rb').__next__ # Example of alternate readline
-
- The generator produces 5-tuples with these members: the token type; the
- token string; a 2-tuple (srow, scol) of ints specifying the row and
- column where the token begins in the source; a 2-tuple (erow, ecol) of
- ints specifying the row and column where the token ends in the source;
- and the line on which the token was found. The line passed is the
- logical line; continuation lines are included.
-
- The first token sequence will always be an ENCODING token
- which tells you which encoding was used to decode the bytes stream.
- """
- # This import is here to avoid problems when the itertools module is not
- # built yet and tokenize is imported.
- from itertools import chain, repeat
- encoding, consumed = detect_encoding(readline)
- rl_gen = iter(readline, b"")
- empty = repeat(b"")
- return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
-
-
-def _tokenize(readline, encoding):
- lnum = parenlev = continued = 0
- numchars = '0123456789'
- contstr, needcont = '', 0
- contline = None
- indents = [0]
-
- if encoding is not None:
- if encoding == "utf-8-sig":
- # BOM will already have been stripped.
- encoding = "utf-8"
- yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
- while True: # loop over lines in stream
- try:
- line = readline()
- except StopIteration:
- line = b''
-
- if encoding is not None:
- line = line.decode(encoding)
- lnum += 1
- pos, max = 0, len(line)
-
- if contstr: # continued string
- if not line:
- raise TokenError("EOF in multi-line string", strstart)
- endmatch = endprog.match(line)
- if endmatch:
- pos = end = endmatch.end(0)
- yield TokenInfo(STRING, contstr + line[:end],
- strstart, (lnum, end), contline + line)
- contstr, needcont = '', 0
- contline = None
- elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
- yield TokenInfo(ERRORTOKEN, contstr + line,
- strstart, (lnum, len(line)), contline)
- contstr = ''
- contline = None
- continue
- else:
- contstr = contstr + line
- contline = contline + line
- continue
-
- elif parenlev == 0 and not continued: # new statement
- if not line: break
- column = 0
- while pos < max: # measure leading whitespace
- if line[pos] == ' ':
- column += 1
- elif line[pos] == '\t':
- column = (column//tabsize + 1)*tabsize
- elif line[pos] == '\f':
- column = 0
- else:
- break
- pos += 1
- if pos == max:
- break
-
- if line[pos] in '#\r\n': # skip comments or blank lines
- if line[pos] == '#':
- comment_token = line[pos:].rstrip('\r\n')
- nl_pos = pos + len(comment_token)
- yield TokenInfo(COMMENT, comment_token,
- (lnum, pos), (lnum, pos + len(comment_token)), line)
- yield TokenInfo(NEWLINE, line[nl_pos:],
- (lnum, nl_pos), (lnum, len(line)), line)
- else:
- yield TokenInfo(NEWLINE, line[pos:],
- (lnum, pos), (lnum, len(line)), line)
- continue
-
- if column > indents[-1]: # count indents or dedents
- indents.append(column)
- yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
- while column < indents[-1]:
- if column not in indents:
- raise IndentationError(
- "unindent does not match any outer indentation level",
- ("<tokenize>", lnum, pos, line))
- indents = indents[:-1]
- yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
-
- else: # continued statement
- if not line:
- raise TokenError("EOF in multi-line statement", (lnum, 0))
- continued = 0
-
- while pos < max:
- pseudomatch = pseudoprog.match(line, pos)
- if pseudomatch: # scan for tokens
- start, end = pseudomatch.span(1)
- spos, epos, pos = (lnum, start), (lnum, end), end
- token, initial = line[start:end], line[start]
-
- if (initial in numchars or # ordinary number
- (initial == '.' and token != '.' and token != '...')):
- yield TokenInfo(NUMBER, token, spos, epos, line)
- elif initial in '\r\n':
- yield TokenInfo(NL if parenlev > 0 else NEWLINE,
- token, spos, epos, line)
- elif initial == '#':
- assert not token.endswith("\n")
- yield TokenInfo(COMMENT, token, spos, epos, line)
- elif token in triple_quoted:
- endprog = endprogs[token]
- endmatch = endprog.match(line, pos)
- if endmatch: # all on one line
- pos = endmatch.end(0)
- token = line[start:pos]
- yield TokenInfo(STRING, token, spos, (lnum, pos), line)
- else:
- strstart = (lnum, start) # multiple lines
- contstr = line[start:]
- contline = line
- break
- elif initial in single_quoted or \
- token[:2] in single_quoted or \
- token[:3] in single_quoted:
- if token[-1] == '\n': # continued string
- strstart = (lnum, start)
- endprog = (endprogs[initial] or endprogs[token[1]] or
- endprogs[token[2]])
- contstr, needcont = line[start:], 1
- contline = line
- break
- else: # ordinary string
- yield TokenInfo(STRING, token, spos, epos, line)
- elif initial.isidentifier(): # ordinary name
- yield TokenInfo(NAME, token, spos, epos, line)
- elif initial == '\\': # continued stmt
- continued = 1
- else:
- if initial in '([{':
- parenlev += 1
- elif initial in ')]}':
- parenlev -= 1
- yield TokenInfo(OP, token, spos, epos, line)
- else:
- yield TokenInfo(ERRORTOKEN, line[pos],
- (lnum, pos), (lnum, pos+1), line)
- pos += 1
-
- for indent in indents[1:]: # pop remaining indent levels
- yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
- yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
-
-
-# An undocumented, backwards compatible, API for all the places in the standard
-# library that expect to be able to use tokenize with strings
-def generate_tokens(readline):
- return _tokenize(readline, None)
-
-if __name__ == "__main__":
- # Quick sanity check
- s = b'''def parseline(self, line):
- """Parse the line into a command name and a string containing
- the arguments. Returns a tuple containing (command, args, line).
- 'command' and 'args' may be None if the line couldn't be parsed.
- """
- line = line.strip()
- if not line:
- return None, None, line
- elif line[0] == '?':
- line = 'help ' + line[1:]
- elif line[0] == '!':
- if hasattr(self, 'do_shell'):
- line = 'shell ' + line[1:]
- else:
- return None, None, line
- i, n = 0, len(line)
- while i < n and line[i] in self.identchars: i = i+1
- cmd, arg = line[:i], line[i:].strip()
- return cmd, arg, line
- '''
- for tok in tokenize(iter(s.splitlines()).__next__):
- print(tok)
+"""Patched version of standard library tokenize, to deal with various bugs.
+
+Based on Python 3.2 code.
+
+Patches:
+
+- Gareth Rees' patch for Python issue #12691 (untokenizing)
+ - Except we don't encode the output of untokenize
+ - Python 2 compatible syntax, so that it can be byte-compiled at installation
+- Newlines in comments and blank lines should be either NL or NEWLINE, depending
+ on whether they are in a multi-line statement. Filed as Python issue #17061.
+- Export generate_tokens & TokenError
+- u and rb literals are allowed under Python 3.3 and above.
+
+------------------------------------------------------------------------------
+Tokenization help for Python programs.
+
+tokenize(readline) is a generator that breaks a stream of bytes into
+Python tokens. It decodes the bytes according to PEP-0263 for
+determining source file encoding.
+
+It accepts a readline-like method which is called repeatedly to get the
+next line of input (or b"" for EOF). It generates 5-tuples with these
+members:
+
+ the token type (see token.py)
+ the token (a string)
+ the starting (row, column) indices of the token (a 2-tuple of ints)
+ the ending (row, column) indices of the token (a 2-tuple of ints)
+ the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators. Additionally, all token lists start with an ENCODING token
+which tells you which encoding was used to decode the bytes stream.
+"""
+from __future__ import absolute_import
+
+__author__ = 'Ka-Ping Yee <ping@lfw.org>'
+__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
+ 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
+ 'Michael Foord')
+import builtins
+import re
+import sys
+from token import *
+from codecs import lookup, BOM_UTF8
+import collections
+from io import TextIOWrapper
+cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+
+import token
+__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
+ "NL", "untokenize", "ENCODING", "TokenInfo"]
+del token
+
+__all__ += ["generate_tokens", "TokenError"]
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+ENCODING = N_TOKENS + 2
+tok_name[ENCODING] = 'ENCODING'
+N_TOKENS += 3
+
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
+ def __repr__(self):
+ annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
+ return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
+ self._replace(type=annotated_type))
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'\w+'
+
+Hexnumber = r'0[xX][0-9a-fA-F]+'
+Binnumber = r'0[bB][01]+'
+Octnumber = r'0[oO][0-7]+'
+Decnumber = r'(?:0+|[1-9][0-9]*)'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?[0-9]+'
+Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+Expfloat = r'[0-9]+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+if sys.version_info.minor >= 3:
+ StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
+else:
+ StringPrefix = r'(?:[bB]?[rR]?)?'
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group(StringPrefix + "'''", StringPrefix + '"""')
+# Single-line ' or " string.
+String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+ StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
+ r"//=?", r"->",
+ r"[+\-*/%&|^=<>]=?",
+ r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+ group("'", r'\\\r?\n'),
+ StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+ group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+def _compile(expr):
+ return re.compile(expr, re.UNICODE)
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+ _compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": _compile(Single), '"': _compile(Double),
+ "'''": single3prog, '"""': double3prog,
+ "r'''": single3prog, 'r"""': double3prog,
+ "b'''": single3prog, 'b"""': double3prog,
+ "R'''": single3prog, 'R"""': double3prog,
+ "B'''": single3prog, 'B"""': double3prog,
+ "br'''": single3prog, 'br"""': double3prog,
+ "bR'''": single3prog, 'bR"""': double3prog,
+ "Br'''": single3prog, 'Br"""': double3prog,
+ "BR'''": single3prog, 'BR"""': double3prog,
+ 'r': None, 'R': None, 'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+ "r'''", 'r"""', "R'''", 'R"""',
+ "b'''", 'b"""', "B'''", 'B"""',
+ "br'''", 'br"""', "Br'''", 'Br"""',
+ "bR'''", 'bR"""', "BR'''", 'BR"""'):
+ triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+ "r'", 'r"', "R'", 'R"',
+ "b'", 'b"', "B'", 'B"',
+ "br'", 'br"', "Br'", 'Br"',
+ "bR'", 'bR"', "BR'", 'BR"' ):
+ single_quoted[t] = t
+
+if sys.version_info.minor >= 3:
+ # Python 3.3
+ for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
+ _t2 = _prefix+'"""'
+ endprogs[_t2] = double3prog
+ triple_quoted[_t2] = _t2
+ _t1 = _prefix + "'''"
+ endprogs[_t1] = single3prog
+ triple_quoted[_t1] = _t1
+ single_quoted[_prefix+'"'] = _prefix+'"'
+ single_quoted[_prefix+"'"] = _prefix+"'"
+ del _prefix, _t2, _t1
+ endprogs['u'] = None
+ endprogs['U'] = None
+
+del _compile
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+class StopTokenizing(Exception): pass
+
+
+class Untokenizer:
+
+ def __init__(self):
+ self.tokens = []
+ self.prev_row = 1
+ self.prev_col = 0
+ self.encoding = 'utf-8'
+
+ def add_whitespace(self, tok_type, start):
+ row, col = start
+ assert row >= self.prev_row
+ col_offset = col - self.prev_col
+ if col_offset > 0:
+ self.tokens.append(" " * col_offset)
+ elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
+ # Line was backslash-continued.
+ self.tokens.append(" ")
+
+ def untokenize(self, tokens):
+ iterable = iter(tokens)
+ for t in iterable:
+ if len(t) == 2:
+ self.compat(t, iterable)
+ break
+ tok_type, token, start, end = t[:4]
+ if tok_type == ENCODING:
+ self.encoding = token
+ continue
+ self.add_whitespace(tok_type, start)
+ self.tokens.append(token)
+ self.prev_row, self.prev_col = end
+ if tok_type in (NEWLINE, NL):
+ self.prev_row += 1
+ self.prev_col = 0
+ return "".join(self.tokens)
+
+ def compat(self, token, iterable):
+ # This import is here to avoid problems when the itertools
+ # module is not built yet and tokenize is imported.
+ from itertools import chain
+ startline = False
+ prevstring = False
+ indents = []
+ toks_append = self.tokens.append
+
+ for tok in chain([token], iterable):
+ toknum, tokval = tok[:2]
+ if toknum == ENCODING:
+ self.encoding = tokval
+ continue
+
+ if toknum in (NAME, NUMBER):
+ tokval += ' '
+
+ # Insert a space between two consecutive strings
+ if toknum == STRING:
+ if prevstring:
+ tokval = ' ' + tokval
+ prevstring = True
+ else:
+ prevstring = False
+
+ if toknum == INDENT:
+ indents.append(tokval)
+ continue
+ elif toknum == DEDENT:
+ indents.pop()
+ continue
+ elif toknum in (NEWLINE, NL):
+ startline = True
+ elif startline and indents:
+ toks_append(indents[-1])
+ startline = False
+ toks_append(tokval)
+
+
+def untokenize(tokens):
+ """
+ Convert ``tokens`` (an iterable) back into Python source code. Return
+ a bytes object, encoded using the encoding specified by the last
+ ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
+
+ The result is guaranteed to tokenize back to match the input so that
+ the conversion is lossless and round-trips are assured. The
+ guarantee applies only to the token type and token string as the
+ spacing between tokens (column positions) may change.
+
+ :func:`untokenize` has two modes. If the input tokens are sequences
+ of length 2 (``type``, ``string``) then spaces are added as necessary to
+ preserve the round-trip property.
+
+ If the input tokens are sequences of length 4 or more (``type``,
+ ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
+ spaces are added so that each token appears in the result at the
+ position indicated by ``start`` and ``end``, if possible.
+ """
+ return Untokenizer().untokenize(tokens)
+
+
+def _get_normal_name(orig_enc):
+ """Imitates get_normal_name in tokenizer.c."""
+ # Only care about the first 12 characters.
+ enc = orig_enc[:12].lower().replace("_", "-")
+ if enc == "utf-8" or enc.startswith("utf-8-"):
+ return "utf-8"
+ if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
+ enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
+ return "iso-8859-1"
+ return orig_enc
+
+def detect_encoding(readline):
+ """
+ The detect_encoding() function is used to detect the encoding that should
+ be used to decode a Python source file. It requires one argment, readline,
+ in the same way as the tokenize() generator.
+
+ It will call readline a maximum of twice, and return the encoding used
+ (as a string) and a list of any lines (left as bytes) it has read in.
+
+ It detects the encoding from the presence of a utf-8 bom or an encoding
+ cookie as specified in pep-0263. If both a bom and a cookie are present,
+ but disagree, a SyntaxError will be raised. If the encoding cookie is an
+ invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
+ 'utf-8-sig' is returned.
+
+ If no encoding is specified, then the default of 'utf-8' will be returned.
+ """
+ bom_found = False
+ encoding = None
+ default = 'utf-8'
+ def read_or_stop():
+ try:
+ return readline()
+ except StopIteration:
+ return b''
+
+ def find_cookie(line):
+ try:
+ # Decode as UTF-8. Either the line is an encoding declaration,
+ # in which case it should be pure ASCII, or it must be UTF-8
+ # per default encoding.
+ line_string = line.decode('utf-8')
+ except UnicodeDecodeError:
+ raise SyntaxError("invalid or missing encoding declaration")
+
+ matches = cookie_re.findall(line_string)
+ if not matches:
+ return None
+ encoding = _get_normal_name(matches[0])
+ try:
+ codec = lookup(encoding)
+ except LookupError:
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError("unknown encoding: " + encoding)
+
+ if bom_found:
+ if encoding != 'utf-8':
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError('encoding problem: utf-8')
+ encoding += '-sig'
+ return encoding
+
+ first = read_or_stop()
+ if first.startswith(BOM_UTF8):
+ bom_found = True
+ first = first[3:]
+ default = 'utf-8-sig'
+ if not first:
+ return default, []
+
+ encoding = find_cookie(first)
+ if encoding:
+ return encoding, [first]
+
+ second = read_or_stop()
+ if not second:
+ return default, [first]
+
+ encoding = find_cookie(second)
+ if encoding:
+ return encoding, [first, second]
+
+ return default, [first, second]
+
+
+def open(filename):
+ """Open a file in read only mode using the encoding detected by
+ detect_encoding().
+ """
+ buffer = builtins.open(filename, 'rb')
+ encoding, lines = detect_encoding(buffer.readline)
+ buffer.seek(0)
+ text = TextIOWrapper(buffer, encoding, line_buffering=True)
+ text.mode = 'r'
+ return text
+
+
+def tokenize(readline):
+ """
+ The tokenize() generator requires one argment, readline, which
+ must be a callable object which provides the same interface as the
+ readline() method of built-in file objects. Each call to the function
+ should return one line of input as bytes. Alternately, readline
+ can be a callable function terminating with StopIteration:
+ readline = open(myfile, 'rb').__next__ # Example of alternate readline
+
+ The generator produces 5-tuples with these members: the token type; the
+ token string; a 2-tuple (srow, scol) of ints specifying the row and
+ column where the token begins in the source; a 2-tuple (erow, ecol) of
+ ints specifying the row and column where the token ends in the source;
+ and the line on which the token was found. The line passed is the
+ logical line; continuation lines are included.
+
+ The first token sequence will always be an ENCODING token
+ which tells you which encoding was used to decode the bytes stream.
+ """
+ # This import is here to avoid problems when the itertools module is not
+ # built yet and tokenize is imported.
+ from itertools import chain, repeat
+ encoding, consumed = detect_encoding(readline)
+ rl_gen = iter(readline, b"")
+ empty = repeat(b"")
+ return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
+
+
+def _tokenize(readline, encoding):
+ lnum = parenlev = continued = 0
+ numchars = '0123456789'
+ contstr, needcont = '', 0
+ contline = None
+ indents = [0]
+
+ if encoding is not None:
+ if encoding == "utf-8-sig":
+ # BOM will already have been stripped.
+ encoding = "utf-8"
+ yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
+ while True: # loop over lines in stream
+ try:
+ line = readline()
+ except StopIteration:
+ line = b''
+
+ if encoding is not None:
+ line = line.decode(encoding)
+ lnum += 1
+ pos, max = 0, len(line)
+
+ if contstr: # continued string
+ if not line:
+ raise TokenError("EOF in multi-line string", strstart)
+ endmatch = endprog.match(line)
+ if endmatch:
+ pos = end = endmatch.end(0)
+ yield TokenInfo(STRING, contstr + line[:end],
+ strstart, (lnum, end), contline + line)
+ contstr, needcont = '', 0
+ contline = None
+ elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+ yield TokenInfo(ERRORTOKEN, contstr + line,
+ strstart, (lnum, len(line)), contline)
+ contstr = ''
+ contline = None
+ continue
+ else:
+ contstr = contstr + line
+ contline = contline + line
+ continue
+
+ elif parenlev == 0 and not continued: # new statement
+ if not line: break
+ column = 0
+ while pos < max: # measure leading whitespace
+ if line[pos] == ' ':
+ column += 1
+ elif line[pos] == '\t':
+ column = (column//tabsize + 1)*tabsize
+ elif line[pos] == '\f':
+ column = 0
+ else:
+ break
+ pos += 1
+ if pos == max:
+ break
+
+ if line[pos] in '#\r\n': # skip comments or blank lines
+ if line[pos] == '#':
+ comment_token = line[pos:].rstrip('\r\n')
+ nl_pos = pos + len(comment_token)
+ yield TokenInfo(COMMENT, comment_token,
+ (lnum, pos), (lnum, pos + len(comment_token)), line)
+ yield TokenInfo(NEWLINE, line[nl_pos:],
+ (lnum, nl_pos), (lnum, len(line)), line)
+ else:
+ yield TokenInfo(NEWLINE, line[pos:],
+ (lnum, pos), (lnum, len(line)), line)
+ continue
+
+ if column > indents[-1]: # count indents or dedents
+ indents.append(column)
+ yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+ while column < indents[-1]:
+ if column not in indents:
+ raise IndentationError(
+ "unindent does not match any outer indentation level",
+ ("<tokenize>", lnum, pos, line))
+ indents = indents[:-1]
+ yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+ else: # continued statement
+ if not line:
+ raise TokenError("EOF in multi-line statement", (lnum, 0))
+ continued = 0
+
+ while pos < max:
+ pseudomatch = pseudoprog.match(line, pos)
+ if pseudomatch: # scan for tokens
+ start, end = pseudomatch.span(1)
+ spos, epos, pos = (lnum, start), (lnum, end), end
+ token, initial = line[start:end], line[start]
+
+ if (initial in numchars or # ordinary number
+ (initial == '.' and token != '.' and token != '...')):
+ yield TokenInfo(NUMBER, token, spos, epos, line)
+ elif initial in '\r\n':
+ yield TokenInfo(NL if parenlev > 0 else NEWLINE,
+ token, spos, epos, line)
+ elif initial == '#':
+ assert not token.endswith("\n")
+ yield TokenInfo(COMMENT, token, spos, epos, line)
+ elif token in triple_quoted:
+ endprog = endprogs[token]
+ endmatch = endprog.match(line, pos)
+ if endmatch: # all on one line
+ pos = endmatch.end(0)
+ token = line[start:pos]
+ yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+ else:
+ strstart = (lnum, start) # multiple lines
+ contstr = line[start:]
+ contline = line
+ break
+ elif initial in single_quoted or \
+ token[:2] in single_quoted or \
+ token[:3] in single_quoted:
+ if token[-1] == '\n': # continued string
+ strstart = (lnum, start)
+ endprog = (endprogs[initial] or endprogs[token[1]] or
+ endprogs[token[2]])
+ contstr, needcont = line[start:], 1
+ contline = line
+ break
+ else: # ordinary string
+ yield TokenInfo(STRING, token, spos, epos, line)
+ elif initial.isidentifier(): # ordinary name
+ yield TokenInfo(NAME, token, spos, epos, line)
+ elif initial == '\\': # continued stmt
+ continued = 1
+ else:
+ if initial in '([{':
+ parenlev += 1
+ elif initial in ')]}':
+ parenlev -= 1
+ yield TokenInfo(OP, token, spos, epos, line)
+ else:
+ yield TokenInfo(ERRORTOKEN, line[pos],
+ (lnum, pos), (lnum, pos+1), line)
+ pos += 1
+
+ for indent in indents[1:]: # pop remaining indent levels
+ yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
+ yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+
+
+# An undocumented, backwards compatible, API for all the places in the standard
+# library that expect to be able to use tokenize with strings
+def generate_tokens(readline):
+ return _tokenize(readline, None)
+
+if __name__ == "__main__":
+ # Quick sanity check
+ s = b'''def parseline(self, line):
+ """Parse the line into a command name and a string containing
+ the arguments. Returns a tuple containing (command, args, line).
+ 'command' and 'args' may be None if the line couldn't be parsed.
+ """
+ line = line.strip()
+ if not line:
+ return None, None, line
+ elif line[0] == '?':
+ line = 'help ' + line[1:]
+ elif line[0] == '!':
+ if hasattr(self, 'do_shell'):
+ line = 'shell ' + line[1:]
+ else:
+ return None, None, line
+ i, n = 0, len(line)
+ while i < n and line[i] in self.identchars: i = i+1
+ cmd, arg = line[:i], line[i:].strip()
+ return cmd, arg, line
+ '''
+ for tok in tokenize(iter(s.splitlines()).__next__):
+ print(tok)
diff --git a/contrib/python/ipython/py2/IPython/utils/capture.py b/contrib/python/ipython/py2/IPython/utils/capture.py
index d8f919568c..bb241b0fad 100644
--- a/contrib/python/ipython/py2/IPython/utils/capture.py
+++ b/contrib/python/ipython/py2/IPython/utils/capture.py
@@ -1,176 +1,176 @@
-# encoding: utf-8
-"""IO capturing utilities."""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-from __future__ import print_function, absolute_import
-
-import sys
-
-from IPython.utils.py3compat import PY3
-
-if PY3:
- from io import StringIO
-else:
- from StringIO import StringIO
-
-#-----------------------------------------------------------------------------
-# Classes and functions
-#-----------------------------------------------------------------------------
-
-
-class RichOutput(object):
+# encoding: utf-8
+"""IO capturing utilities."""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from __future__ import print_function, absolute_import
+
+import sys
+
+from IPython.utils.py3compat import PY3
+
+if PY3:
+ from io import StringIO
+else:
+ from StringIO import StringIO
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+
+class RichOutput(object):
def __init__(self, data=None, metadata=None, transient=None, update=False):
- self.data = data or {}
- self.metadata = metadata or {}
+ self.data = data or {}
+ self.metadata = metadata or {}
self.transient = transient or {}
self.update = update
- def display(self):
- from IPython.display import publish_display_data
+ def display(self):
+ from IPython.display import publish_display_data
publish_display_data(data=self.data, metadata=self.metadata,
transient=self.transient, update=self.update)
- def _repr_mime_(self, mime):
- if mime not in self.data:
- return
- data = self.data[mime]
- if mime in self.metadata:
- return data, self.metadata[mime]
- else:
- return data
-
- def _repr_html_(self):
- return self._repr_mime_("text/html")
-
- def _repr_latex_(self):
- return self._repr_mime_("text/latex")
-
- def _repr_json_(self):
- return self._repr_mime_("application/json")
-
- def _repr_javascript_(self):
- return self._repr_mime_("application/javascript")
-
- def _repr_png_(self):
- return self._repr_mime_("image/png")
-
- def _repr_jpeg_(self):
- return self._repr_mime_("image/jpeg")
-
- def _repr_svg_(self):
- return self._repr_mime_("image/svg+xml")
-
-
-class CapturedIO(object):
- """Simple object for containing captured stdout/err and rich display StringIO objects
-
- Each instance `c` has three attributes:
-
- - ``c.stdout`` : standard output as a string
- - ``c.stderr`` : standard error as a string
- - ``c.outputs``: a list of rich display outputs
-
- Additionally, there's a ``c.show()`` method which will print all of the
- above in the same order, and can be invoked simply via ``c()``.
- """
-
- def __init__(self, stdout, stderr, outputs=None):
- self._stdout = stdout
- self._stderr = stderr
- if outputs is None:
- outputs = []
- self._outputs = outputs
-
- def __str__(self):
- return self.stdout
-
- @property
- def stdout(self):
- "Captured standard output"
- if not self._stdout:
- return ''
- return self._stdout.getvalue()
-
- @property
- def stderr(self):
- "Captured standard error"
- if not self._stderr:
- return ''
- return self._stderr.getvalue()
-
- @property
- def outputs(self):
- """A list of the captured rich display outputs, if any.
-
- If you have a CapturedIO object ``c``, these can be displayed in IPython
- using::
-
- from IPython.display import display
- for o in c.outputs:
- display(o)
- """
+ def _repr_mime_(self, mime):
+ if mime not in self.data:
+ return
+ data = self.data[mime]
+ if mime in self.metadata:
+ return data, self.metadata[mime]
+ else:
+ return data
+
+ def _repr_html_(self):
+ return self._repr_mime_("text/html")
+
+ def _repr_latex_(self):
+ return self._repr_mime_("text/latex")
+
+ def _repr_json_(self):
+ return self._repr_mime_("application/json")
+
+ def _repr_javascript_(self):
+ return self._repr_mime_("application/javascript")
+
+ def _repr_png_(self):
+ return self._repr_mime_("image/png")
+
+ def _repr_jpeg_(self):
+ return self._repr_mime_("image/jpeg")
+
+ def _repr_svg_(self):
+ return self._repr_mime_("image/svg+xml")
+
+
+class CapturedIO(object):
+ """Simple object for containing captured stdout/err and rich display StringIO objects
+
+ Each instance `c` has three attributes:
+
+ - ``c.stdout`` : standard output as a string
+ - ``c.stderr`` : standard error as a string
+ - ``c.outputs``: a list of rich display outputs
+
+ Additionally, there's a ``c.show()`` method which will print all of the
+ above in the same order, and can be invoked simply via ``c()``.
+ """
+
+ def __init__(self, stdout, stderr, outputs=None):
+ self._stdout = stdout
+ self._stderr = stderr
+ if outputs is None:
+ outputs = []
+ self._outputs = outputs
+
+ def __str__(self):
+ return self.stdout
+
+ @property
+ def stdout(self):
+ "Captured standard output"
+ if not self._stdout:
+ return ''
+ return self._stdout.getvalue()
+
+ @property
+ def stderr(self):
+ "Captured standard error"
+ if not self._stderr:
+ return ''
+ return self._stderr.getvalue()
+
+ @property
+ def outputs(self):
+ """A list of the captured rich display outputs, if any.
+
+ If you have a CapturedIO object ``c``, these can be displayed in IPython
+ using::
+
+ from IPython.display import display
+ for o in c.outputs:
+ display(o)
+ """
return [ RichOutput(**kargs) for kargs in self._outputs ]
- def show(self):
- """write my output to sys.stdout/err as appropriate"""
- sys.stdout.write(self.stdout)
- sys.stderr.write(self.stderr)
- sys.stdout.flush()
- sys.stderr.flush()
+ def show(self):
+ """write my output to sys.stdout/err as appropriate"""
+ sys.stdout.write(self.stdout)
+ sys.stderr.write(self.stderr)
+ sys.stdout.flush()
+ sys.stderr.flush()
for kargs in self._outputs:
RichOutput(**kargs).display()
- __call__ = show
-
-
-class capture_output(object):
- """context manager for capturing stdout/err"""
- stdout = True
- stderr = True
- display = True
-
- def __init__(self, stdout=True, stderr=True, display=True):
- self.stdout = stdout
- self.stderr = stderr
- self.display = display
- self.shell = None
-
- def __enter__(self):
- from IPython.core.getipython import get_ipython
- from IPython.core.displaypub import CapturingDisplayPublisher
+ __call__ = show
+
+
+class capture_output(object):
+ """context manager for capturing stdout/err"""
+ stdout = True
+ stderr = True
+ display = True
+
+ def __init__(self, stdout=True, stderr=True, display=True):
+ self.stdout = stdout
+ self.stderr = stderr
+ self.display = display
+ self.shell = None
+
+ def __enter__(self):
+ from IPython.core.getipython import get_ipython
+ from IPython.core.displaypub import CapturingDisplayPublisher
from IPython.core.displayhook import CapturingDisplayHook
- self.sys_stdout = sys.stdout
- self.sys_stderr = sys.stderr
-
- if self.display:
- self.shell = get_ipython()
- if self.shell is None:
- self.save_display_pub = None
- self.display = False
-
- stdout = stderr = outputs = None
- if self.stdout:
- stdout = sys.stdout = StringIO()
- if self.stderr:
- stderr = sys.stderr = StringIO()
- if self.display:
- self.save_display_pub = self.shell.display_pub
- self.shell.display_pub = CapturingDisplayPublisher()
- outputs = self.shell.display_pub.outputs
+ self.sys_stdout = sys.stdout
+ self.sys_stderr = sys.stderr
+
+ if self.display:
+ self.shell = get_ipython()
+ if self.shell is None:
+ self.save_display_pub = None
+ self.display = False
+
+ stdout = stderr = outputs = None
+ if self.stdout:
+ stdout = sys.stdout = StringIO()
+ if self.stderr:
+ stderr = sys.stderr = StringIO()
+ if self.display:
+ self.save_display_pub = self.shell.display_pub
+ self.shell.display_pub = CapturingDisplayPublisher()
+ outputs = self.shell.display_pub.outputs
self.save_display_hook = sys.displayhook
sys.displayhook = CapturingDisplayHook(shell=self.shell,
outputs=outputs)
- return CapturedIO(stdout, stderr, outputs)
+ return CapturedIO(stdout, stderr, outputs)
- def __exit__(self, exc_type, exc_value, traceback):
- sys.stdout = self.sys_stdout
- sys.stderr = self.sys_stderr
- if self.display and self.shell:
- self.shell.display_pub = self.save_display_pub
+ def __exit__(self, exc_type, exc_value, traceback):
+ sys.stdout = self.sys_stdout
+ sys.stderr = self.sys_stderr
+ if self.display and self.shell:
+ self.shell.display_pub = self.save_display_pub
sys.displayhook = self.save_display_hook
-
-
+
+
diff --git a/contrib/python/ipython/py2/IPython/utils/coloransi.py b/contrib/python/ipython/py2/IPython/utils/coloransi.py
index bc8e8377f7..597c69fe11 100644
--- a/contrib/python/ipython/py2/IPython/utils/coloransi.py
+++ b/contrib/python/ipython/py2/IPython/utils/coloransi.py
@@ -1,187 +1,187 @@
-# -*- coding: utf-8 -*-
-"""Tools for coloring text in ANSI terminals.
-"""
-
-#*****************************************************************************
-# Copyright (C) 2002-2006 Fernando Perez. <fperez@colorado.edu>
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#*****************************************************************************
-
-__all__ = ['TermColors','InputTermColors','ColorScheme','ColorSchemeTable']
-
-import os
-
-from IPython.utils.ipstruct import Struct
-
-color_templates = (
- # Dark colors
- ("Black" , "0;30"),
- ("Red" , "0;31"),
- ("Green" , "0;32"),
- ("Brown" , "0;33"),
- ("Blue" , "0;34"),
- ("Purple" , "0;35"),
- ("Cyan" , "0;36"),
- ("LightGray" , "0;37"),
- # Light colors
- ("DarkGray" , "1;30"),
- ("LightRed" , "1;31"),
- ("LightGreen" , "1;32"),
- ("Yellow" , "1;33"),
- ("LightBlue" , "1;34"),
- ("LightPurple" , "1;35"),
- ("LightCyan" , "1;36"),
- ("White" , "1;37"),
- # Blinking colors. Probably should not be used in anything serious.
- ("BlinkBlack" , "5;30"),
- ("BlinkRed" , "5;31"),
- ("BlinkGreen" , "5;32"),
- ("BlinkYellow" , "5;33"),
- ("BlinkBlue" , "5;34"),
- ("BlinkPurple" , "5;35"),
- ("BlinkCyan" , "5;36"),
- ("BlinkLightGray", "5;37"),
- )
-
-def make_color_table(in_class):
- """Build a set of color attributes in a class.
-
- Helper function for building the :class:`TermColors` and
- :class`InputTermColors`.
- """
- for name,value in color_templates:
- setattr(in_class,name,in_class._base % value)
-
-class TermColors:
- """Color escape sequences.
-
- This class defines the escape sequences for all the standard (ANSI?)
- colors in terminals. Also defines a NoColor escape which is just the null
- string, suitable for defining 'dummy' color schemes in terminals which get
- confused by color escapes.
-
- This class should be used as a mixin for building color schemes."""
-
- NoColor = '' # for color schemes in color-less terminals.
- Normal = '\033[0m' # Reset normal coloring
- _base = '\033[%sm' # Template for all other colors
-
-# Build the actual color table as a set of class attributes:
-make_color_table(TermColors)
-
-class InputTermColors:
- """Color escape sequences for input prompts.
-
- This class is similar to TermColors, but the escapes are wrapped in \001
- and \002 so that readline can properly know the length of each line and
- can wrap lines accordingly. Use this class for any colored text which
- needs to be used in input prompts, such as in calls to raw_input().
-
- This class defines the escape sequences for all the standard (ANSI?)
- colors in terminals. Also defines a NoColor escape which is just the null
- string, suitable for defining 'dummy' color schemes in terminals which get
- confused by color escapes.
-
- This class should be used as a mixin for building color schemes."""
-
- NoColor = '' # for color schemes in color-less terminals.
-
- if os.name == 'nt' and os.environ.get('TERM','dumb') == 'emacs':
- # (X)emacs on W32 gets confused with \001 and \002 so we remove them
- Normal = '\033[0m' # Reset normal coloring
- _base = '\033[%sm' # Template for all other colors
- else:
- Normal = '\001\033[0m\002' # Reset normal coloring
- _base = '\001\033[%sm\002' # Template for all other colors
-
-# Build the actual color table as a set of class attributes:
-make_color_table(InputTermColors)
-
-class NoColors:
- """This defines all the same names as the colour classes, but maps them to
- empty strings, so it can easily be substituted to turn off colours."""
- NoColor = ''
- Normal = ''
-
-for name, value in color_templates:
- setattr(NoColors, name, '')
-
-class ColorScheme:
- """Generic color scheme class. Just a name and a Struct."""
- def __init__(self,__scheme_name_,colordict=None,**colormap):
- self.name = __scheme_name_
- if colordict is None:
- self.colors = Struct(**colormap)
- else:
- self.colors = Struct(colordict)
-
- def copy(self,name=None):
- """Return a full copy of the object, optionally renaming it."""
- if name is None:
- name = self.name
- return ColorScheme(name, self.colors.dict())
-
-class ColorSchemeTable(dict):
- """General class to handle tables of color schemes.
-
- It's basically a dict of color schemes with a couple of shorthand
- attributes and some convenient methods.
-
- active_scheme_name -> obvious
- active_colors -> actual color table of the active scheme"""
-
- def __init__(self, scheme_list=None, default_scheme=''):
- """Create a table of color schemes.
-
- The table can be created empty and manually filled or it can be
- created with a list of valid color schemes AND the specification for
- the default active scheme.
- """
-
- # create object attributes to be set later
- self.active_scheme_name = ''
- self.active_colors = None
-
- if scheme_list:
- if default_scheme == '':
- raise ValueError('you must specify the default color scheme')
- for scheme in scheme_list:
- self.add_scheme(scheme)
- self.set_active_scheme(default_scheme)
-
- def copy(self):
- """Return full copy of object"""
- return ColorSchemeTable(self.values(),self.active_scheme_name)
-
- def add_scheme(self,new_scheme):
- """Add a new color scheme to the table."""
- if not isinstance(new_scheme,ColorScheme):
- raise ValueError('ColorSchemeTable only accepts ColorScheme instances')
- self[new_scheme.name] = new_scheme
-
- def set_active_scheme(self,scheme,case_sensitive=0):
- """Set the currently active scheme.
-
- Names are by default compared in a case-insensitive way, but this can
- be changed by setting the parameter case_sensitive to true."""
-
- scheme_names = list(self.keys())
- if case_sensitive:
- valid_schemes = scheme_names
- scheme_test = scheme
- else:
- valid_schemes = [s.lower() for s in scheme_names]
- scheme_test = scheme.lower()
- try:
- scheme_idx = valid_schemes.index(scheme_test)
- except ValueError:
- raise ValueError('Unrecognized color scheme: ' + scheme + \
- '\nValid schemes: '+str(scheme_names).replace("'', ",''))
- else:
- active = scheme_names[scheme_idx]
- self.active_scheme_name = active
- self.active_colors = self[active].colors
- # Now allow using '' as an index for the current active scheme
- self[''] = self[active]
+# -*- coding: utf-8 -*-
+"""Tools for coloring text in ANSI terminals.
+"""
+
+#*****************************************************************************
+# Copyright (C) 2002-2006 Fernando Perez. <fperez@colorado.edu>
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+__all__ = ['TermColors','InputTermColors','ColorScheme','ColorSchemeTable']
+
+import os
+
+from IPython.utils.ipstruct import Struct
+
+color_templates = (
+ # Dark colors
+ ("Black" , "0;30"),
+ ("Red" , "0;31"),
+ ("Green" , "0;32"),
+ ("Brown" , "0;33"),
+ ("Blue" , "0;34"),
+ ("Purple" , "0;35"),
+ ("Cyan" , "0;36"),
+ ("LightGray" , "0;37"),
+ # Light colors
+ ("DarkGray" , "1;30"),
+ ("LightRed" , "1;31"),
+ ("LightGreen" , "1;32"),
+ ("Yellow" , "1;33"),
+ ("LightBlue" , "1;34"),
+ ("LightPurple" , "1;35"),
+ ("LightCyan" , "1;36"),
+ ("White" , "1;37"),
+ # Blinking colors. Probably should not be used in anything serious.
+ ("BlinkBlack" , "5;30"),
+ ("BlinkRed" , "5;31"),
+ ("BlinkGreen" , "5;32"),
+ ("BlinkYellow" , "5;33"),
+ ("BlinkBlue" , "5;34"),
+ ("BlinkPurple" , "5;35"),
+ ("BlinkCyan" , "5;36"),
+ ("BlinkLightGray", "5;37"),
+ )
+
+def make_color_table(in_class):
+ """Build a set of color attributes in a class.
+
+ Helper function for building the :class:`TermColors` and
+ :class`InputTermColors`.
+ """
+ for name,value in color_templates:
+ setattr(in_class,name,in_class._base % value)
+
+class TermColors:
+ """Color escape sequences.
+
+ This class defines the escape sequences for all the standard (ANSI?)
+ colors in terminals. Also defines a NoColor escape which is just the null
+ string, suitable for defining 'dummy' color schemes in terminals which get
+ confused by color escapes.
+
+ This class should be used as a mixin for building color schemes."""
+
+ NoColor = '' # for color schemes in color-less terminals.
+ Normal = '\033[0m' # Reset normal coloring
+ _base = '\033[%sm' # Template for all other colors
+
+# Build the actual color table as a set of class attributes:
+make_color_table(TermColors)
+
+class InputTermColors:
+ """Color escape sequences for input prompts.
+
+ This class is similar to TermColors, but the escapes are wrapped in \001
+ and \002 so that readline can properly know the length of each line and
+ can wrap lines accordingly. Use this class for any colored text which
+ needs to be used in input prompts, such as in calls to raw_input().
+
+ This class defines the escape sequences for all the standard (ANSI?)
+ colors in terminals. Also defines a NoColor escape which is just the null
+ string, suitable for defining 'dummy' color schemes in terminals which get
+ confused by color escapes.
+
+ This class should be used as a mixin for building color schemes."""
+
+ NoColor = '' # for color schemes in color-less terminals.
+
+ if os.name == 'nt' and os.environ.get('TERM','dumb') == 'emacs':
+ # (X)emacs on W32 gets confused with \001 and \002 so we remove them
+ Normal = '\033[0m' # Reset normal coloring
+ _base = '\033[%sm' # Template for all other colors
+ else:
+ Normal = '\001\033[0m\002' # Reset normal coloring
+ _base = '\001\033[%sm\002' # Template for all other colors
+
+# Build the actual color table as a set of class attributes:
+make_color_table(InputTermColors)
+
+class NoColors:
+ """This defines all the same names as the colour classes, but maps them to
+ empty strings, so it can easily be substituted to turn off colours."""
+ NoColor = ''
+ Normal = ''
+
+for name, value in color_templates:
+ setattr(NoColors, name, '')
+
+class ColorScheme:
+ """Generic color scheme class. Just a name and a Struct."""
+ def __init__(self,__scheme_name_,colordict=None,**colormap):
+ self.name = __scheme_name_
+ if colordict is None:
+ self.colors = Struct(**colormap)
+ else:
+ self.colors = Struct(colordict)
+
+ def copy(self,name=None):
+ """Return a full copy of the object, optionally renaming it."""
+ if name is None:
+ name = self.name
+ return ColorScheme(name, self.colors.dict())
+
+class ColorSchemeTable(dict):
+ """General class to handle tables of color schemes.
+
+ It's basically a dict of color schemes with a couple of shorthand
+ attributes and some convenient methods.
+
+ active_scheme_name -> obvious
+ active_colors -> actual color table of the active scheme"""
+
+ def __init__(self, scheme_list=None, default_scheme=''):
+ """Create a table of color schemes.
+
+ The table can be created empty and manually filled or it can be
+ created with a list of valid color schemes AND the specification for
+ the default active scheme.
+ """
+
+ # create object attributes to be set later
+ self.active_scheme_name = ''
+ self.active_colors = None
+
+ if scheme_list:
+ if default_scheme == '':
+ raise ValueError('you must specify the default color scheme')
+ for scheme in scheme_list:
+ self.add_scheme(scheme)
+ self.set_active_scheme(default_scheme)
+
+ def copy(self):
+ """Return full copy of object"""
+ return ColorSchemeTable(self.values(),self.active_scheme_name)
+
+ def add_scheme(self,new_scheme):
+ """Add a new color scheme to the table."""
+ if not isinstance(new_scheme,ColorScheme):
+ raise ValueError('ColorSchemeTable only accepts ColorScheme instances')
+ self[new_scheme.name] = new_scheme
+
+ def set_active_scheme(self,scheme,case_sensitive=0):
+ """Set the currently active scheme.
+
+ Names are by default compared in a case-insensitive way, but this can
+ be changed by setting the parameter case_sensitive to true."""
+
+ scheme_names = list(self.keys())
+ if case_sensitive:
+ valid_schemes = scheme_names
+ scheme_test = scheme
+ else:
+ valid_schemes = [s.lower() for s in scheme_names]
+ scheme_test = scheme.lower()
+ try:
+ scheme_idx = valid_schemes.index(scheme_test)
+ except ValueError:
+ raise ValueError('Unrecognized color scheme: ' + scheme + \
+ '\nValid schemes: '+str(scheme_names).replace("'', ",''))
+ else:
+ active = scheme_names[scheme_idx]
+ self.active_scheme_name = active
+ self.active_colors = self[active].colors
+ # Now allow using '' as an index for the current active scheme
+ self[''] = self[active]
diff --git a/contrib/python/ipython/py2/IPython/utils/contexts.py b/contrib/python/ipython/py2/IPython/utils/contexts.py
index 4d379b0eda..358dfe8b29 100644
--- a/contrib/python/ipython/py2/IPython/utils/contexts.py
+++ b/contrib/python/ipython/py2/IPython/utils/contexts.py
@@ -1,66 +1,66 @@
-# encoding: utf-8
-"""Miscellaneous context managers.
-"""
-
+# encoding: utf-8
+"""Miscellaneous context managers.
+"""
+
import warnings
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-class preserve_keys(object):
- """Preserve a set of keys in a dictionary.
-
- Upon entering the context manager the current values of the keys
- will be saved. Upon exiting, the dictionary will be updated to
- restore the original value of the preserved keys. Preserved keys
- which did not exist when entering the context manager will be
- deleted.
-
- Examples
- --------
-
- >>> d = {'a': 1, 'b': 2, 'c': 3}
- >>> with preserve_keys(d, 'b', 'c', 'd'):
- ... del d['a']
- ... del d['b'] # will be reset to 2
- ... d['c'] = None # will be reset to 3
- ... d['d'] = 4 # will be deleted
- ... d['e'] = 5
- ... print(sorted(d.items()))
- ...
- [('c', None), ('d', 4), ('e', 5)]
- >>> print(sorted(d.items()))
- [('b', 2), ('c', 3), ('e', 5)]
- """
-
- def __init__(self, dictionary, *keys):
- self.dictionary = dictionary
- self.keys = keys
-
- def __enter__(self):
- # Actions to perform upon exiting.
- to_delete = []
- to_update = {}
-
- d = self.dictionary
- for k in self.keys:
- if k in d:
- to_update[k] = d[k]
- else:
- to_delete.append(k)
-
- self.to_delete = to_delete
- self.to_update = to_update
-
- def __exit__(self, *exc_info):
- d = self.dictionary
-
- for k in self.to_delete:
- d.pop(k, None)
- d.update(self.to_update)
-
-
-class NoOpContext(object):
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+class preserve_keys(object):
+ """Preserve a set of keys in a dictionary.
+
+ Upon entering the context manager the current values of the keys
+ will be saved. Upon exiting, the dictionary will be updated to
+ restore the original value of the preserved keys. Preserved keys
+ which did not exist when entering the context manager will be
+ deleted.
+
+ Examples
+ --------
+
+ >>> d = {'a': 1, 'b': 2, 'c': 3}
+ >>> with preserve_keys(d, 'b', 'c', 'd'):
+ ... del d['a']
+ ... del d['b'] # will be reset to 2
+ ... d['c'] = None # will be reset to 3
+ ... d['d'] = 4 # will be deleted
+ ... d['e'] = 5
+ ... print(sorted(d.items()))
+ ...
+ [('c', None), ('d', 4), ('e', 5)]
+ >>> print(sorted(d.items()))
+ [('b', 2), ('c', 3), ('e', 5)]
+ """
+
+ def __init__(self, dictionary, *keys):
+ self.dictionary = dictionary
+ self.keys = keys
+
+ def __enter__(self):
+ # Actions to perform upon exiting.
+ to_delete = []
+ to_update = {}
+
+ d = self.dictionary
+ for k in self.keys:
+ if k in d:
+ to_update[k] = d[k]
+ else:
+ to_delete.append(k)
+
+ self.to_delete = to_delete
+ self.to_update = to_update
+
+ def __exit__(self, *exc_info):
+ d = self.dictionary
+
+ for k in self.to_delete:
+ d.pop(k, None)
+ d.update(self.to_update)
+
+
+class NoOpContext(object):
"""
Deprecated
@@ -70,5 +70,5 @@ class NoOpContext(object):
warnings.warn("""NoOpContext is deprecated since IPython 5.0 """,
DeprecationWarning, stacklevel=2)
- def __enter__(self): pass
- def __exit__(self, type, value, traceback): pass
+ def __enter__(self): pass
+ def __exit__(self, type, value, traceback): pass
diff --git a/contrib/python/ipython/py2/IPython/utils/daemonize.py b/contrib/python/ipython/py2/IPython/utils/daemonize.py
index a1bfaa193b..f093cf67cb 100644
--- a/contrib/python/ipython/py2/IPython/utils/daemonize.py
+++ b/contrib/python/ipython/py2/IPython/utils/daemonize.py
@@ -1,4 +1,4 @@
-from warnings import warn
-
-warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize")
-from ipyparallel.apps.daemonize import daemonize
+from warnings import warn
+
+warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize")
+from ipyparallel.apps.daemonize import daemonize
diff --git a/contrib/python/ipython/py2/IPython/utils/data.py b/contrib/python/ipython/py2/IPython/utils/data.py
index 308a692559..36a8aabd95 100644
--- a/contrib/python/ipython/py2/IPython/utils/data.py
+++ b/contrib/python/ipython/py2/IPython/utils/data.py
@@ -1,37 +1,37 @@
-# encoding: utf-8
-"""Utilities for working with data structures like lists, dicts and tuples.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-from .py3compat import xrange
-
-def uniq_stable(elems):
- """uniq_stable(elems) -> list
-
- Return from an iterable, a list of all the unique elements in the input,
- but maintaining the order in which they first appear.
-
- Note: All elements in the input must be hashable for this routine
- to work, as it internally uses a set for efficiency reasons.
- """
- seen = set()
- return [x for x in elems if x not in seen and not seen.add(x)]
-
-
-def flatten(seq):
- """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
-
- return [x for subseq in seq for x in subseq]
-
-
-def chop(seq, size):
- """Chop a sequence into chunks of the given size."""
- return [seq[i:i+size] for i in xrange(0,len(seq),size)]
-
-
+# encoding: utf-8
+"""Utilities for working with data structures like lists, dicts and tuples.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from .py3compat import xrange
+
+def uniq_stable(elems):
+ """uniq_stable(elems) -> list
+
+ Return from an iterable, a list of all the unique elements in the input,
+ but maintaining the order in which they first appear.
+
+ Note: All elements in the input must be hashable for this routine
+ to work, as it internally uses a set for efficiency reasons.
+ """
+ seen = set()
+ return [x for x in elems if x not in seen and not seen.add(x)]
+
+
+def flatten(seq):
+ """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
+
+ return [x for subseq in seq for x in subseq]
+
+
+def chop(seq, size):
+ """Chop a sequence into chunks of the given size."""
+ return [seq[i:i+size] for i in xrange(0,len(seq),size)]
+
+
diff --git a/contrib/python/ipython/py2/IPython/utils/decorators.py b/contrib/python/ipython/py2/IPython/utils/decorators.py
index c26485553c..79be8ca1e6 100644
--- a/contrib/python/ipython/py2/IPython/utils/decorators.py
+++ b/contrib/python/ipython/py2/IPython/utils/decorators.py
@@ -1,58 +1,58 @@
-# encoding: utf-8
-"""Decorators that don't go anywhere else.
-
-This module contains misc. decorators that don't really go with another module
-in :mod:`IPython.utils`. Beore putting something here please see if it should
-go into another topical module in :mod:`IPython.utils`.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-def flag_calls(func):
- """Wrap a function to detect and flag when it gets called.
-
- This is a decorator which takes a function and wraps it in a function with
- a 'called' attribute. wrapper.called is initialized to False.
-
- The wrapper.called attribute is set to False right before each call to the
- wrapped function, so if the call fails it remains False. After the call
- completes, wrapper.called is set to True and the output is returned.
-
- Testing for truth in wrapper.called allows you to determine if a call to
- func() was attempted and succeeded."""
-
- # don't wrap twice
- if hasattr(func, 'called'):
- return func
-
- def wrapper(*args,**kw):
- wrapper.called = False
- out = func(*args,**kw)
- wrapper.called = True
- return out
-
- wrapper.called = False
- wrapper.__doc__ = func.__doc__
- return wrapper
-
-def undoc(func):
- """Mark a function or class as undocumented.
-
- This is found by inspecting the AST, so for now it must be used directly
- as @undoc, not as e.g. @decorators.undoc
- """
- return func
-
+# encoding: utf-8
+"""Decorators that don't go anywhere else.
+
+This module contains misc. decorators that don't really go with another module
+in :mod:`IPython.utils`. Beore putting something here please see if it should
+go into another topical module in :mod:`IPython.utils`.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def flag_calls(func):
+ """Wrap a function to detect and flag when it gets called.
+
+ This is a decorator which takes a function and wraps it in a function with
+ a 'called' attribute. wrapper.called is initialized to False.
+
+ The wrapper.called attribute is set to False right before each call to the
+ wrapped function, so if the call fails it remains False. After the call
+ completes, wrapper.called is set to True and the output is returned.
+
+ Testing for truth in wrapper.called allows you to determine if a call to
+ func() was attempted and succeeded."""
+
+ # don't wrap twice
+ if hasattr(func, 'called'):
+ return func
+
+ def wrapper(*args,**kw):
+ wrapper.called = False
+ out = func(*args,**kw)
+ wrapper.called = True
+ return out
+
+ wrapper.called = False
+ wrapper.__doc__ = func.__doc__
+ return wrapper
+
+def undoc(func):
+ """Mark a function or class as undocumented.
+
+ This is found by inspecting the AST, so for now it must be used directly
+ as @undoc, not as e.g. @decorators.undoc
+ """
+ return func
+
diff --git a/contrib/python/ipython/py2/IPython/utils/dir2.py b/contrib/python/ipython/py2/IPython/utils/dir2.py
index f6f164f9b1..fb0cd719ef 100644
--- a/contrib/python/ipython/py2/IPython/utils/dir2.py
+++ b/contrib/python/ipython/py2/IPython/utils/dir2.py
@@ -1,51 +1,51 @@
-# encoding: utf-8
-"""A fancy version of Python's builtin :func:`dir` function.
-"""
-
+# encoding: utf-8
+"""A fancy version of Python's builtin :func:`dir` function.
+"""
+
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
-
+
import inspect
-from .py3compat import string_types
-
-
-def safe_hasattr(obj, attr):
- """In recent versions of Python, hasattr() only catches AttributeError.
- This catches all errors.
- """
- try:
- getattr(obj, attr)
- return True
- except:
- return False
-
-
-def dir2(obj):
- """dir2(obj) -> list of strings
-
- Extended version of the Python builtin dir(), which does a few extra
- checks.
-
- This version is guaranteed to return only a list of true strings, whereas
- dir() returns anything that objects inject into themselves, even if they
- are later not really valid for attribute access (many extension libraries
- have such bugs).
- """
-
- # Start building the attribute list via dir(), and then complete it
- # with a few extra special-purpose calls.
-
- try:
- words = set(dir(obj))
- except Exception:
- # TypeError: dir(obj) does not return a list
- words = set()
-
- # filter out non-string attributes which may be stuffed by dir() calls
- # and poor coding in third-party modules
-
- words = [w for w in words if isinstance(w, string_types)]
- return sorted(words)
+from .py3compat import string_types
+
+
+def safe_hasattr(obj, attr):
+ """In recent versions of Python, hasattr() only catches AttributeError.
+ This catches all errors.
+ """
+ try:
+ getattr(obj, attr)
+ return True
+ except:
+ return False
+
+
+def dir2(obj):
+ """dir2(obj) -> list of strings
+
+ Extended version of the Python builtin dir(), which does a few extra
+ checks.
+
+ This version is guaranteed to return only a list of true strings, whereas
+ dir() returns anything that objects inject into themselves, even if they
+ are later not really valid for attribute access (many extension libraries
+ have such bugs).
+ """
+
+ # Start building the attribute list via dir(), and then complete it
+ # with a few extra special-purpose calls.
+
+ try:
+ words = set(dir(obj))
+ except Exception:
+ # TypeError: dir(obj) does not return a list
+ words = set()
+
+ # filter out non-string attributes which may be stuffed by dir() calls
+ # and poor coding in third-party modules
+
+ words = [w for w in words if isinstance(w, string_types)]
+ return sorted(words)
def get_real_method(obj, name):
diff --git a/contrib/python/ipython/py2/IPython/utils/encoding.py b/contrib/python/ipython/py2/IPython/utils/encoding.py
index 387a24700c..ba8ca09534 100644
--- a/contrib/python/ipython/py2/IPython/utils/encoding.py
+++ b/contrib/python/ipython/py2/IPython/utils/encoding.py
@@ -1,71 +1,71 @@
-# coding: utf-8
-"""
-Utilities for dealing with text encodings
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2012 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-import sys
-import locale
-import warnings
-
-# to deal with the possibility of sys.std* not being a stream at all
-def get_stream_enc(stream, default=None):
- """Return the given stream's encoding or a default.
-
- There are cases where ``sys.std*`` might not actually be a stream, so
- check for the encoding attribute prior to returning it, and return
- a default if it doesn't exist or evaluates as False. ``default``
- is None if not provided.
- """
- if not hasattr(stream, 'encoding') or not stream.encoding:
- return default
- else:
- return stream.encoding
-
-# Less conservative replacement for sys.getdefaultencoding, that will try
-# to match the environment.
-# Defined here as central function, so if we find better choices, we
-# won't need to make changes all over IPython.
-def getdefaultencoding(prefer_stream=True):
- """Return IPython's guess for the default encoding for bytes as text.
-
- If prefer_stream is True (default), asks for stdin.encoding first,
- to match the calling Terminal, but that is often None for subprocesses.
-
- Then fall back on locale.getpreferredencoding(),
- which should be a sensible platform default (that respects LANG environment),
- and finally to sys.getdefaultencoding() which is the most conservative option,
- and usually ASCII on Python 2 or UTF8 on Python 3.
- """
- enc = None
- if prefer_stream:
- enc = get_stream_enc(sys.stdin)
- if not enc or enc=='ascii':
- try:
- # There are reports of getpreferredencoding raising errors
- # in some cases, which may well be fixed, but let's be conservative here.
- enc = locale.getpreferredencoding()
- except Exception:
- pass
- enc = enc or sys.getdefaultencoding()
- # On windows `cp0` can be returned to indicate that there is no code page.
- # Since cp0 is an invalid encoding return instead cp1252 which is the
- # Western European default.
- if enc == 'cp0':
- warnings.warn(
- "Invalid code page cp0 detected - using cp1252 instead."
- "If cp1252 is incorrect please ensure a valid code page "
- "is defined for the process.", RuntimeWarning)
- return 'cp1252'
- return enc
-
-DEFAULT_ENCODING = getdefaultencoding()
+# coding: utf-8
+"""
+Utilities for dealing with text encodings
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2012 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import sys
+import locale
+import warnings
+
+# to deal with the possibility of sys.std* not being a stream at all
+def get_stream_enc(stream, default=None):
+ """Return the given stream's encoding or a default.
+
+ There are cases where ``sys.std*`` might not actually be a stream, so
+ check for the encoding attribute prior to returning it, and return
+ a default if it doesn't exist or evaluates as False. ``default``
+ is None if not provided.
+ """
+ if not hasattr(stream, 'encoding') or not stream.encoding:
+ return default
+ else:
+ return stream.encoding
+
+# Less conservative replacement for sys.getdefaultencoding, that will try
+# to match the environment.
+# Defined here as central function, so if we find better choices, we
+# won't need to make changes all over IPython.
+def getdefaultencoding(prefer_stream=True):
+ """Return IPython's guess for the default encoding for bytes as text.
+
+ If prefer_stream is True (default), asks for stdin.encoding first,
+ to match the calling Terminal, but that is often None for subprocesses.
+
+ Then fall back on locale.getpreferredencoding(),
+ which should be a sensible platform default (that respects LANG environment),
+ and finally to sys.getdefaultencoding() which is the most conservative option,
+ and usually ASCII on Python 2 or UTF8 on Python 3.
+ """
+ enc = None
+ if prefer_stream:
+ enc = get_stream_enc(sys.stdin)
+ if not enc or enc=='ascii':
+ try:
+ # There are reports of getpreferredencoding raising errors
+ # in some cases, which may well be fixed, but let's be conservative here.
+ enc = locale.getpreferredencoding()
+ except Exception:
+ pass
+ enc = enc or sys.getdefaultencoding()
+ # On windows `cp0` can be returned to indicate that there is no code page.
+ # Since cp0 is an invalid encoding return instead cp1252 which is the
+ # Western European default.
+ if enc == 'cp0':
+ warnings.warn(
+ "Invalid code page cp0 detected - using cp1252 instead."
+ "If cp1252 is incorrect please ensure a valid code page "
+ "is defined for the process.", RuntimeWarning)
+ return 'cp1252'
+ return enc
+
+DEFAULT_ENCODING = getdefaultencoding()
diff --git a/contrib/python/ipython/py2/IPython/utils/eventful.py b/contrib/python/ipython/py2/IPython/utils/eventful.py
index fc0f7aee4f..e954a45e0a 100644
--- a/contrib/python/ipython/py2/IPython/utils/eventful.py
+++ b/contrib/python/ipython/py2/IPython/utils/eventful.py
@@ -1,7 +1,7 @@
-from __future__ import absolute_import
-
-from warnings import warn
-
-warn("IPython.utils.eventful has moved to traitlets.eventful")
-
-from traitlets.eventful import *
+from __future__ import absolute_import
+
+from warnings import warn
+
+warn("IPython.utils.eventful has moved to traitlets.eventful")
+
+from traitlets.eventful import *
diff --git a/contrib/python/ipython/py2/IPython/utils/frame.py b/contrib/python/ipython/py2/IPython/utils/frame.py
index 76ccc71c44..ebf9e47bf9 100644
--- a/contrib/python/ipython/py2/IPython/utils/frame.py
+++ b/contrib/python/ipython/py2/IPython/utils/frame.py
@@ -1,98 +1,98 @@
-# encoding: utf-8
-"""
-Utilities for working with stack frames.
-"""
-from __future__ import print_function
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import sys
-from IPython.utils import py3compat
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-@py3compat.doctest_refactor_print
-def extract_vars(*names,**kw):
- """Extract a set of variables by name from another frame.
-
- Parameters
- ----------
- *names : str
- One or more variable names which will be extracted from the caller's
- frame.
-
- depth : integer, optional
- How many frames in the stack to walk when looking for your variables.
- The default is 0, which will use the frame where the call was made.
-
-
- Examples
- --------
- ::
-
- In [2]: def func(x):
- ...: y = 1
- ...: print(sorted(extract_vars('x','y').items()))
- ...:
-
- In [3]: func('hello')
- [('x', 'hello'), ('y', 1)]
- """
-
- depth = kw.get('depth',0)
-
- callerNS = sys._getframe(depth+1).f_locals
- return dict((k,callerNS[k]) for k in names)
-
-
-def extract_vars_above(*names):
- """Extract a set of variables by name from another frame.
-
- Similar to extractVars(), but with a specified depth of 1, so that names
- are exctracted exactly from above the caller.
-
- This is simply a convenience function so that the very common case (for us)
- of skipping exactly 1 frame doesn't have to construct a special dict for
- keyword passing."""
-
- callerNS = sys._getframe(2).f_locals
- return dict((k,callerNS[k]) for k in names)
-
-
-def debugx(expr,pre_msg=''):
- """Print the value of an expression from the caller's frame.
-
- Takes an expression, evaluates it in the caller's frame and prints both
- the given expression and the resulting value (as well as a debug mark
- indicating the name of the calling function. The input must be of a form
- suitable for eval().
-
- An optional message can be passed, which will be prepended to the printed
- expr->value pair."""
-
- cf = sys._getframe(1)
- print('[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
- eval(expr,cf.f_globals,cf.f_locals)))
-
-
-# deactivate it by uncommenting the following line, which makes it a no-op
-#def debugx(expr,pre_msg=''): pass
-
-def extract_module_locals(depth=0):
- """Returns (module, locals) of the function `depth` frames away from the caller"""
- f = sys._getframe(depth + 1)
- global_ns = f.f_globals
- module = sys.modules[global_ns['__name__']]
- return (module, f.f_locals)
-
+# encoding: utf-8
+"""
+Utilities for working with stack frames.
+"""
+from __future__ import print_function
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+from IPython.utils import py3compat
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+@py3compat.doctest_refactor_print
+def extract_vars(*names,**kw):
+ """Extract a set of variables by name from another frame.
+
+ Parameters
+ ----------
+ *names : str
+ One or more variable names which will be extracted from the caller's
+ frame.
+
+ depth : integer, optional
+ How many frames in the stack to walk when looking for your variables.
+ The default is 0, which will use the frame where the call was made.
+
+
+ Examples
+ --------
+ ::
+
+ In [2]: def func(x):
+ ...: y = 1
+ ...: print(sorted(extract_vars('x','y').items()))
+ ...:
+
+ In [3]: func('hello')
+ [('x', 'hello'), ('y', 1)]
+ """
+
+ depth = kw.get('depth',0)
+
+ callerNS = sys._getframe(depth+1).f_locals
+ return dict((k,callerNS[k]) for k in names)
+
+
+def extract_vars_above(*names):
+ """Extract a set of variables by name from another frame.
+
+ Similar to extractVars(), but with a specified depth of 1, so that names
+ are exctracted exactly from above the caller.
+
+ This is simply a convenience function so that the very common case (for us)
+ of skipping exactly 1 frame doesn't have to construct a special dict for
+ keyword passing."""
+
+ callerNS = sys._getframe(2).f_locals
+ return dict((k,callerNS[k]) for k in names)
+
+
+def debugx(expr,pre_msg=''):
+ """Print the value of an expression from the caller's frame.
+
+ Takes an expression, evaluates it in the caller's frame and prints both
+ the given expression and the resulting value (as well as a debug mark
+ indicating the name of the calling function. The input must be of a form
+ suitable for eval().
+
+ An optional message can be passed, which will be prepended to the printed
+ expr->value pair."""
+
+ cf = sys._getframe(1)
+ print('[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
+ eval(expr,cf.f_globals,cf.f_locals)))
+
+
+# deactivate it by uncommenting the following line, which makes it a no-op
+#def debugx(expr,pre_msg=''): pass
+
+def extract_module_locals(depth=0):
+ """Returns (module, locals) of the function `depth` frames away from the caller"""
+ f = sys._getframe(depth + 1)
+ global_ns = f.f_globals
+ module = sys.modules[global_ns['__name__']]
+ return (module, f.f_locals)
+
diff --git a/contrib/python/ipython/py2/IPython/utils/generics.py b/contrib/python/ipython/py2/IPython/utils/generics.py
index 5ffdc86ebd..ff856a7e55 100644
--- a/contrib/python/ipython/py2/IPython/utils/generics.py
+++ b/contrib/python/ipython/py2/IPython/utils/generics.py
@@ -1,34 +1,34 @@
-# encoding: utf-8
-"""Generic functions for extending IPython.
-
-See http://pypi.python.org/pypi/simplegeneric.
-"""
-
-from IPython.core.error import TryNext
-from simplegeneric import generic
-
-
-@generic
-def inspect_object(obj):
- """Called when you do obj?"""
- raise TryNext
-
-
-@generic
-def complete_object(obj, prev_completions):
- """Custom completer dispatching for python objects.
-
- Parameters
- ----------
- obj : object
- The object to complete.
- prev_completions : list
- List of attributes discovered so far.
-
- This should return the list of attributes in obj. If you only wish to
- add to the attributes already discovered normally, return
- own_attrs + prev_completions.
- """
- raise TryNext
-
-
+# encoding: utf-8
+"""Generic functions for extending IPython.
+
+See http://pypi.python.org/pypi/simplegeneric.
+"""
+
+from IPython.core.error import TryNext
+from simplegeneric import generic
+
+
+@generic
+def inspect_object(obj):
+ """Called when you do obj?"""
+ raise TryNext
+
+
+@generic
+def complete_object(obj, prev_completions):
+ """Custom completer dispatching for python objects.
+
+ Parameters
+ ----------
+ obj : object
+ The object to complete.
+ prev_completions : list
+ List of attributes discovered so far.
+
+ This should return the list of attributes in obj. If you only wish to
+ add to the attributes already discovered normally, return
+ own_attrs + prev_completions.
+ """
+ raise TryNext
+
+
diff --git a/contrib/python/ipython/py2/IPython/utils/importstring.py b/contrib/python/ipython/py2/IPython/utils/importstring.py
index c8e1840eb3..2c7a2a167e 100644
--- a/contrib/python/ipython/py2/IPython/utils/importstring.py
+++ b/contrib/python/ipython/py2/IPython/utils/importstring.py
@@ -1,39 +1,39 @@
-# encoding: utf-8
-"""
-A simple utility to import something by its string name.
-"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-
-def import_item(name):
- """Import and return ``bar`` given the string ``foo.bar``.
-
- Calling ``bar = import_item("foo.bar")`` is the functional equivalent of
- executing the code ``from foo import bar``.
-
- Parameters
- ----------
- name : string
- The fully qualified name of the module/package being imported.
-
- Returns
- -------
- mod : module object
- The module that was imported.
- """
-
- parts = name.rsplit('.', 1)
- if len(parts) == 2:
- # called with 'foo.bar....'
- package, obj = parts
- module = __import__(package, fromlist=[obj])
- try:
- pak = getattr(module, obj)
- except AttributeError:
- raise ImportError('No module named %s' % obj)
- return pak
- else:
- # called with un-dotted string
- return __import__(parts[0])
+# encoding: utf-8
+"""
+A simple utility to import something by its string name.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+
+def import_item(name):
+ """Import and return ``bar`` given the string ``foo.bar``.
+
+ Calling ``bar = import_item("foo.bar")`` is the functional equivalent of
+ executing the code ``from foo import bar``.
+
+ Parameters
+ ----------
+ name : string
+ The fully qualified name of the module/package being imported.
+
+ Returns
+ -------
+ mod : module object
+ The module that was imported.
+ """
+
+ parts = name.rsplit('.', 1)
+ if len(parts) == 2:
+ # called with 'foo.bar....'
+ package, obj = parts
+ module = __import__(package, fromlist=[obj])
+ try:
+ pak = getattr(module, obj)
+ except AttributeError:
+ raise ImportError('No module named %s' % obj)
+ return pak
+ else:
+ # called with un-dotted string
+ return __import__(parts[0])
diff --git a/contrib/python/ipython/py2/IPython/utils/io.py b/contrib/python/ipython/py2/IPython/utils/io.py
index 036d6e3926..c316c73bcd 100644
--- a/contrib/python/ipython/py2/IPython/utils/io.py
+++ b/contrib/python/ipython/py2/IPython/utils/io.py
@@ -1,95 +1,95 @@
-# encoding: utf-8
-"""
-IO related utilities.
-"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-
-import atexit
-import os
-import sys
-import tempfile
+# encoding: utf-8
+"""
+IO related utilities.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+
+import atexit
+import os
+import sys
+import tempfile
import warnings
-from warnings import warn
+from warnings import warn
from IPython.utils.decorators import undoc
-from .capture import CapturedIO, capture_output
-from .py3compat import string_types, input, PY3
-
+from .capture import CapturedIO, capture_output
+from .py3compat import string_types, input, PY3
+
@undoc
-class IOStream:
-
+class IOStream:
+
def __init__(self, stream, fallback=None):
warn('IOStream is deprecated since IPython 5.0, use sys.{stdin,stdout,stderr} instead',
DeprecationWarning, stacklevel=2)
- if not hasattr(stream,'write') or not hasattr(stream,'flush'):
- if fallback is not None:
- stream = fallback
- else:
- raise ValueError("fallback required, but not specified")
- self.stream = stream
- self._swrite = stream.write
-
- # clone all methods not overridden:
- def clone(meth):
- return not hasattr(self, meth) and not meth.startswith('_')
- for meth in filter(clone, dir(stream)):
+ if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+ if fallback is not None:
+ stream = fallback
+ else:
+ raise ValueError("fallback required, but not specified")
+ self.stream = stream
+ self._swrite = stream.write
+
+ # clone all methods not overridden:
+ def clone(meth):
+ return not hasattr(self, meth) and not meth.startswith('_')
+ for meth in filter(clone, dir(stream)):
try:
val = getattr(stream, meth)
except AttributeError:
pass
else:
setattr(self, meth, val)
-
- def __repr__(self):
- cls = self.__class__
- tpl = '{mod}.{cls}({args})'
- return tpl.format(mod=cls.__module__, cls=cls.__name__, args=self.stream)
-
- def write(self,data):
+
+ def __repr__(self):
+ cls = self.__class__
+ tpl = '{mod}.{cls}({args})'
+ return tpl.format(mod=cls.__module__, cls=cls.__name__, args=self.stream)
+
+ def write(self,data):
warn('IOStream is deprecated since IPython 5.0, use sys.{stdin,stdout,stderr} instead',
DeprecationWarning, stacklevel=2)
- try:
- self._swrite(data)
- except:
- try:
- # print handles some unicode issues which may trip a plain
- # write() call. Emulate write() by using an empty end
- # argument.
- print(data, end='', file=self.stream)
- except:
- # if we get here, something is seriously broken.
- print('ERROR - failed to write data to stream:', self.stream,
- file=sys.stderr)
-
- def writelines(self, lines):
+ try:
+ self._swrite(data)
+ except:
+ try:
+ # print handles some unicode issues which may trip a plain
+ # write() call. Emulate write() by using an empty end
+ # argument.
+ print(data, end='', file=self.stream)
+ except:
+ # if we get here, something is seriously broken.
+ print('ERROR - failed to write data to stream:', self.stream,
+ file=sys.stderr)
+
+ def writelines(self, lines):
warn('IOStream is deprecated since IPython 5.0, use sys.{stdin,stdout,stderr} instead',
DeprecationWarning, stacklevel=2)
- if isinstance(lines, string_types):
- lines = [lines]
- for line in lines:
- self.write(line)
-
- # This class used to have a writeln method, but regular files and streams
- # in Python don't have this method. We need to keep this completely
- # compatible so we removed it.
-
- @property
- def closed(self):
- return self.stream.closed
-
- def close(self):
- pass
-
-# setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
+ if isinstance(lines, string_types):
+ lines = [lines]
+ for line in lines:
+ self.write(line)
+
+ # This class used to have a writeln method, but regular files and streams
+ # in Python don't have this method. We need to keep this completely
+ # compatible so we removed it.
+
+ @property
+ def closed(self):
+ return self.stream.closed
+
+ def close(self):
+ pass
+
+# setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
devnull = open(os.devnull, 'w')
-atexit.register(devnull.close)
-
+atexit.register(devnull.close)
+
# io.std* are deprecated, but don't show our own deprecation warnings
# during initialization of the deprecated API.
with warnings.catch_warnings():
@@ -98,149 +98,149 @@ with warnings.catch_warnings():
stdout = IOStream(sys.stdout, fallback=devnull)
stderr = IOStream(sys.stderr, fallback=devnull)
-class Tee(object):
- """A class to duplicate an output stream to stdout/err.
-
- This works in a manner very similar to the Unix 'tee' command.
-
- When the object is closed or deleted, it closes the original file given to
- it for duplication.
- """
- # Inspired by:
- # http://mail.python.org/pipermail/python-list/2007-May/442737.html
-
- def __init__(self, file_or_name, mode="w", channel='stdout'):
- """Construct a new Tee object.
-
- Parameters
- ----------
- file_or_name : filename or open filehandle (writable)
- File that will be duplicated
-
- mode : optional, valid mode for open().
- If a filename was give, open with this mode.
-
- channel : str, one of ['stdout', 'stderr']
- """
- if channel not in ['stdout', 'stderr']:
- raise ValueError('Invalid channel spec %s' % channel)
-
- if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
- self.file = file_or_name
- else:
- self.file = open(file_or_name, mode)
- self.channel = channel
- self.ostream = getattr(sys, channel)
- setattr(sys, channel, self)
- self._closed = False
-
- def close(self):
- """Close the file and restore the channel."""
- self.flush()
- setattr(sys, self.channel, self.ostream)
- self.file.close()
- self._closed = True
-
- def write(self, data):
- """Write data to both channels."""
- self.file.write(data)
- self.ostream.write(data)
- self.ostream.flush()
-
- def flush(self):
- """Flush both channels."""
- self.file.flush()
- self.ostream.flush()
-
- def __del__(self):
- if not self._closed:
- self.close()
-
-
-def ask_yes_no(prompt, default=None, interrupt=None):
- """Asks a question and returns a boolean (y/n) answer.
-
- If default is given (one of 'y','n'), it is used if the user input is
- empty. If interrupt is given (one of 'y','n'), it is used if the user
- presses Ctrl-C. Otherwise the question is repeated until an answer is
- given.
-
- An EOF is treated as the default answer. If there is no default, an
- exception is raised to prevent infinite loops.
-
- Valid answers are: y/yes/n/no (match is not case sensitive)."""
-
- answers = {'y':True,'n':False,'yes':True,'no':False}
- ans = None
- while ans not in answers.keys():
- try:
- ans = input(prompt+' ').lower()
- if not ans: # response was an empty string
- ans = default
- except KeyboardInterrupt:
- if interrupt:
- ans = interrupt
- except EOFError:
- if default in answers.keys():
- ans = default
- print()
- else:
- raise
-
- return answers[ans]
-
-
-def temp_pyfile(src, ext='.py'):
- """Make a temporary python file, return filename and filehandle.
-
- Parameters
- ----------
- src : string or list of strings (no need for ending newlines if list)
- Source code to be written to the file.
-
- ext : optional, string
- Extension for the generated file.
-
- Returns
- -------
- (filename, open filehandle)
- It is the caller's responsibility to close the open file and unlink it.
- """
- fname = tempfile.mkstemp(ext)[1]
- f = open(fname,'w')
- f.write(src)
- f.flush()
- return fname, f
-
-def atomic_writing(*args, **kwargs):
- """DEPRECATED: moved to notebook.services.contents.fileio"""
- warn("IPython.utils.io.atomic_writing has moved to notebook.services.contents.fileio")
- from notebook.services.contents.fileio import atomic_writing
- return atomic_writing(*args, **kwargs)
-
-def raw_print(*args, **kw):
- """Raw print to sys.__stdout__, otherwise identical interface to print()."""
-
- print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
- file=sys.__stdout__)
- sys.__stdout__.flush()
-
-
-def raw_print_err(*args, **kw):
- """Raw print to sys.__stderr__, otherwise identical interface to print()."""
-
- print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
- file=sys.__stderr__)
- sys.__stderr__.flush()
-
-
-# Short aliases for quick debugging, do NOT use these in production code.
-rprint = raw_print
-rprinte = raw_print_err
-
-
-def unicode_std_stream(stream='stdout'):
- """DEPRECATED, moved to nbconvert.utils.io"""
- warn("IPython.utils.io.unicode_std_stream has moved to nbconvert.utils.io")
- from nbconvert.utils.io import unicode_std_stream
- return unicode_std_stream(stream)
+class Tee(object):
+ """A class to duplicate an output stream to stdout/err.
+
+ This works in a manner very similar to the Unix 'tee' command.
+
+ When the object is closed or deleted, it closes the original file given to
+ it for duplication.
+ """
+ # Inspired by:
+ # http://mail.python.org/pipermail/python-list/2007-May/442737.html
+
+ def __init__(self, file_or_name, mode="w", channel='stdout'):
+ """Construct a new Tee object.
+
+ Parameters
+ ----------
+ file_or_name : filename or open filehandle (writable)
+ File that will be duplicated
+
+ mode : optional, valid mode for open().
+ If a filename was give, open with this mode.
+
+ channel : str, one of ['stdout', 'stderr']
+ """
+ if channel not in ['stdout', 'stderr']:
+ raise ValueError('Invalid channel spec %s' % channel)
+
+ if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
+ self.file = file_or_name
+ else:
+ self.file = open(file_or_name, mode)
+ self.channel = channel
+ self.ostream = getattr(sys, channel)
+ setattr(sys, channel, self)
+ self._closed = False
+
+ def close(self):
+ """Close the file and restore the channel."""
+ self.flush()
+ setattr(sys, self.channel, self.ostream)
+ self.file.close()
+ self._closed = True
+
+ def write(self, data):
+ """Write data to both channels."""
+ self.file.write(data)
+ self.ostream.write(data)
+ self.ostream.flush()
+
+ def flush(self):
+ """Flush both channels."""
+ self.file.flush()
+ self.ostream.flush()
+
+ def __del__(self):
+ if not self._closed:
+ self.close()
+
+
+def ask_yes_no(prompt, default=None, interrupt=None):
+ """Asks a question and returns a boolean (y/n) answer.
+
+ If default is given (one of 'y','n'), it is used if the user input is
+ empty. If interrupt is given (one of 'y','n'), it is used if the user
+ presses Ctrl-C. Otherwise the question is repeated until an answer is
+ given.
+
+ An EOF is treated as the default answer. If there is no default, an
+ exception is raised to prevent infinite loops.
+
+ Valid answers are: y/yes/n/no (match is not case sensitive)."""
+
+ answers = {'y':True,'n':False,'yes':True,'no':False}
+ ans = None
+ while ans not in answers.keys():
+ try:
+ ans = input(prompt+' ').lower()
+ if not ans: # response was an empty string
+ ans = default
+ except KeyboardInterrupt:
+ if interrupt:
+ ans = interrupt
+ except EOFError:
+ if default in answers.keys():
+ ans = default
+ print()
+ else:
+ raise
+
+ return answers[ans]
+
+
+def temp_pyfile(src, ext='.py'):
+ """Make a temporary python file, return filename and filehandle.
+
+ Parameters
+ ----------
+ src : string or list of strings (no need for ending newlines if list)
+ Source code to be written to the file.
+
+ ext : optional, string
+ Extension for the generated file.
+
+ Returns
+ -------
+ (filename, open filehandle)
+ It is the caller's responsibility to close the open file and unlink it.
+ """
+ fname = tempfile.mkstemp(ext)[1]
+ f = open(fname,'w')
+ f.write(src)
+ f.flush()
+ return fname, f
+
+def atomic_writing(*args, **kwargs):
+ """DEPRECATED: moved to notebook.services.contents.fileio"""
+ warn("IPython.utils.io.atomic_writing has moved to notebook.services.contents.fileio")
+ from notebook.services.contents.fileio import atomic_writing
+ return atomic_writing(*args, **kwargs)
+
+def raw_print(*args, **kw):
+ """Raw print to sys.__stdout__, otherwise identical interface to print()."""
+
+ print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
+ file=sys.__stdout__)
+ sys.__stdout__.flush()
+
+
+def raw_print_err(*args, **kw):
+ """Raw print to sys.__stderr__, otherwise identical interface to print()."""
+
+ print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
+ file=sys.__stderr__)
+ sys.__stderr__.flush()
+
+
+# Short aliases for quick debugging, do NOT use these in production code.
+rprint = raw_print
+rprinte = raw_print_err
+
+
+def unicode_std_stream(stream='stdout'):
+ """DEPRECATED, moved to nbconvert.utils.io"""
+ warn("IPython.utils.io.unicode_std_stream has moved to nbconvert.utils.io")
+ from nbconvert.utils.io import unicode_std_stream
+ return unicode_std_stream(stream)
diff --git a/contrib/python/ipython/py2/IPython/utils/ipstruct.py b/contrib/python/ipython/py2/IPython/utils/ipstruct.py
index e2b3e8fa4c..e17760b4f9 100644
--- a/contrib/python/ipython/py2/IPython/utils/ipstruct.py
+++ b/contrib/python/ipython/py2/IPython/utils/ipstruct.py
@@ -1,391 +1,391 @@
-# encoding: utf-8
-"""A dict subclass that supports attribute style access.
-
-Authors:
-
-* Fernando Perez (original)
-* Brian Granger (refactoring to a dict subclass)
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-__all__ = ['Struct']
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-
-class Struct(dict):
- """A dict subclass with attribute style access.
-
- This dict subclass has a a few extra features:
-
- * Attribute style access.
- * Protection of class members (like keys, items) when using attribute
- style access.
- * The ability to restrict assignment to only existing keys.
- * Intelligent merging.
- * Overloaded operators.
- """
- _allownew = True
- def __init__(self, *args, **kw):
- """Initialize with a dictionary, another Struct, or data.
-
- Parameters
- ----------
- args : dict, Struct
- Initialize with one dict or Struct
- kw : dict
- Initialize with key, value pairs.
-
- Examples
- --------
-
- >>> s = Struct(a=10,b=30)
- >>> s.a
- 10
- >>> s.b
- 30
- >>> s2 = Struct(s,c=30)
- >>> sorted(s2.keys())
- ['a', 'b', 'c']
- """
- object.__setattr__(self, '_allownew', True)
- dict.__init__(self, *args, **kw)
-
- def __setitem__(self, key, value):
- """Set an item with check for allownew.
-
- Examples
- --------
-
- >>> s = Struct()
- >>> s['a'] = 10
- >>> s.allow_new_attr(False)
- >>> s['a'] = 10
- >>> s['a']
- 10
- >>> try:
- ... s['b'] = 20
- ... except KeyError:
- ... print('this is not allowed')
- ...
- this is not allowed
- """
- if not self._allownew and key not in self:
- raise KeyError(
- "can't create new attribute %s when allow_new_attr(False)" % key)
- dict.__setitem__(self, key, value)
-
- def __setattr__(self, key, value):
- """Set an attr with protection of class members.
-
- This calls :meth:`self.__setitem__` but convert :exc:`KeyError` to
- :exc:`AttributeError`.
-
- Examples
- --------
-
- >>> s = Struct()
- >>> s.a = 10
- >>> s.a
- 10
- >>> try:
- ... s.get = 10
- ... except AttributeError:
- ... print("you can't set a class member")
- ...
- you can't set a class member
- """
- # If key is an str it might be a class member or instance var
- if isinstance(key, str):
- # I can't simply call hasattr here because it calls getattr, which
- # calls self.__getattr__, which returns True for keys in
- # self._data. But I only want keys in the class and in
- # self.__dict__
- if key in self.__dict__ or hasattr(Struct, key):
- raise AttributeError(
- 'attr %s is a protected member of class Struct.' % key
- )
- try:
- self.__setitem__(key, value)
- except KeyError as e:
- raise AttributeError(e)
-
- def __getattr__(self, key):
- """Get an attr by calling :meth:`dict.__getitem__`.
-
- Like :meth:`__setattr__`, this method converts :exc:`KeyError` to
- :exc:`AttributeError`.
-
- Examples
- --------
-
- >>> s = Struct(a=10)
- >>> s.a
- 10
- >>> type(s.get)
- <... 'builtin_function_or_method'>
- >>> try:
- ... s.b
- ... except AttributeError:
- ... print("I don't have that key")
- ...
- I don't have that key
- """
- try:
- result = self[key]
- except KeyError:
- raise AttributeError(key)
- else:
- return result
-
- def __iadd__(self, other):
- """s += s2 is a shorthand for s.merge(s2).
-
- Examples
- --------
-
- >>> s = Struct(a=10,b=30)
- >>> s2 = Struct(a=20,c=40)
- >>> s += s2
- >>> sorted(s.keys())
- ['a', 'b', 'c']
- """
- self.merge(other)
- return self
-
- def __add__(self,other):
- """s + s2 -> New Struct made from s.merge(s2).
-
- Examples
- --------
-
- >>> s1 = Struct(a=10,b=30)
- >>> s2 = Struct(a=20,c=40)
- >>> s = s1 + s2
- >>> sorted(s.keys())
- ['a', 'b', 'c']
- """
- sout = self.copy()
- sout.merge(other)
- return sout
-
- def __sub__(self,other):
- """s1 - s2 -> remove keys in s2 from s1.
-
- Examples
- --------
-
- >>> s1 = Struct(a=10,b=30)
- >>> s2 = Struct(a=40)
- >>> s = s1 - s2
- >>> s
- {'b': 30}
- """
- sout = self.copy()
- sout -= other
- return sout
-
- def __isub__(self,other):
- """Inplace remove keys from self that are in other.
-
- Examples
- --------
-
- >>> s1 = Struct(a=10,b=30)
- >>> s2 = Struct(a=40)
- >>> s1 -= s2
- >>> s1
- {'b': 30}
- """
- for k in other.keys():
- if k in self:
- del self[k]
- return self
-
- def __dict_invert(self, data):
- """Helper function for merge.
-
- Takes a dictionary whose values are lists and returns a dict with
- the elements of each list as keys and the original keys as values.
- """
- outdict = {}
- for k,lst in data.items():
- if isinstance(lst, str):
- lst = lst.split()
- for entry in lst:
- outdict[entry] = k
- return outdict
-
- def dict(self):
- return self
-
- def copy(self):
- """Return a copy as a Struct.
-
- Examples
- --------
-
- >>> s = Struct(a=10,b=30)
- >>> s2 = s.copy()
- >>> type(s2) is Struct
- True
- """
- return Struct(dict.copy(self))
-
- def hasattr(self, key):
- """hasattr function available as a method.
-
- Implemented like has_key.
-
- Examples
- --------
-
- >>> s = Struct(a=10)
- >>> s.hasattr('a')
- True
- >>> s.hasattr('b')
- False
- >>> s.hasattr('get')
- False
- """
- return key in self
-
- def allow_new_attr(self, allow = True):
- """Set whether new attributes can be created in this Struct.
-
- This can be used to catch typos by verifying that the attribute user
- tries to change already exists in this Struct.
- """
- object.__setattr__(self, '_allownew', allow)
-
- def merge(self, __loc_data__=None, __conflict_solve=None, **kw):
- """Merge two Structs with customizable conflict resolution.
-
- This is similar to :meth:`update`, but much more flexible. First, a
- dict is made from data+key=value pairs. When merging this dict with
- the Struct S, the optional dictionary 'conflict' is used to decide
- what to do.
-
- If conflict is not given, the default behavior is to preserve any keys
- with their current value (the opposite of the :meth:`update` method's
- behavior).
-
- Parameters
- ----------
- __loc_data : dict, Struct
- The data to merge into self
- __conflict_solve : dict
- The conflict policy dict. The keys are binary functions used to
- resolve the conflict and the values are lists of strings naming
- the keys the conflict resolution function applies to. Instead of
- a list of strings a space separated string can be used, like
- 'a b c'.
- kw : dict
- Additional key, value pairs to merge in
-
- Notes
- -----
-
- The `__conflict_solve` dict is a dictionary of binary functions which will be used to
- solve key conflicts. Here is an example::
-
- __conflict_solve = dict(
- func1=['a','b','c'],
- func2=['d','e']
- )
-
- In this case, the function :func:`func1` will be used to resolve
- keys 'a', 'b' and 'c' and the function :func:`func2` will be used for
- keys 'd' and 'e'. This could also be written as::
-
- __conflict_solve = dict(func1='a b c',func2='d e')
-
- These functions will be called for each key they apply to with the
- form::
-
- func1(self['a'], other['a'])
-
- The return value is used as the final merged value.
-
- As a convenience, merge() provides five (the most commonly needed)
- pre-defined policies: preserve, update, add, add_flip and add_s. The
- easiest explanation is their implementation::
-
- preserve = lambda old,new: old
- update = lambda old,new: new
- add = lambda old,new: old + new
- add_flip = lambda old,new: new + old # note change of order!
- add_s = lambda old,new: old + ' ' + new # only for str!
-
- You can use those four words (as strings) as keys instead
- of defining them as functions, and the merge method will substitute
- the appropriate functions for you.
-
- For more complicated conflict resolution policies, you still need to
- construct your own functions.
-
- Examples
- --------
-
- This show the default policy:
-
- >>> s = Struct(a=10,b=30)
- >>> s2 = Struct(a=20,c=40)
- >>> s.merge(s2)
- >>> sorted(s.items())
- [('a', 10), ('b', 30), ('c', 40)]
-
- Now, show how to specify a conflict dict:
-
- >>> s = Struct(a=10,b=30)
- >>> s2 = Struct(a=20,b=40)
- >>> conflict = {'update':'a','add':'b'}
- >>> s.merge(s2,conflict)
- >>> sorted(s.items())
- [('a', 20), ('b', 70)]
- """
-
- data_dict = dict(__loc_data__,**kw)
-
- # policies for conflict resolution: two argument functions which return
- # the value that will go in the new struct
- preserve = lambda old,new: old
- update = lambda old,new: new
- add = lambda old,new: old + new
- add_flip = lambda old,new: new + old # note change of order!
- add_s = lambda old,new: old + ' ' + new
-
- # default policy is to keep current keys when there's a conflict
- conflict_solve = dict.fromkeys(self, preserve)
-
- # the conflict_solve dictionary is given by the user 'inverted': we
- # need a name-function mapping, it comes as a function -> names
- # dict. Make a local copy (b/c we'll make changes), replace user
- # strings for the three builtin policies and invert it.
- if __conflict_solve:
- inv_conflict_solve_user = __conflict_solve.copy()
- for name, func in [('preserve',preserve), ('update',update),
- ('add',add), ('add_flip',add_flip),
- ('add_s',add_s)]:
- if name in inv_conflict_solve_user.keys():
- inv_conflict_solve_user[func] = inv_conflict_solve_user[name]
- del inv_conflict_solve_user[name]
- conflict_solve.update(self.__dict_invert(inv_conflict_solve_user))
- for key in data_dict:
- if key not in self:
- self[key] = data_dict[key]
- else:
- self[key] = conflict_solve[key](self[key],data_dict[key])
-
+# encoding: utf-8
+"""A dict subclass that supports attribute style access.
+
+Authors:
+
+* Fernando Perez (original)
+* Brian Granger (refactoring to a dict subclass)
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+__all__ = ['Struct']
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class Struct(dict):
+ """A dict subclass with attribute style access.
+
+ This dict subclass has a a few extra features:
+
+ * Attribute style access.
+ * Protection of class members (like keys, items) when using attribute
+ style access.
+ * The ability to restrict assignment to only existing keys.
+ * Intelligent merging.
+ * Overloaded operators.
+ """
+ _allownew = True
+ def __init__(self, *args, **kw):
+ """Initialize with a dictionary, another Struct, or data.
+
+ Parameters
+ ----------
+ args : dict, Struct
+ Initialize with one dict or Struct
+ kw : dict
+ Initialize with key, value pairs.
+
+ Examples
+ --------
+
+ >>> s = Struct(a=10,b=30)
+ >>> s.a
+ 10
+ >>> s.b
+ 30
+ >>> s2 = Struct(s,c=30)
+ >>> sorted(s2.keys())
+ ['a', 'b', 'c']
+ """
+ object.__setattr__(self, '_allownew', True)
+ dict.__init__(self, *args, **kw)
+
+ def __setitem__(self, key, value):
+ """Set an item with check for allownew.
+
+ Examples
+ --------
+
+ >>> s = Struct()
+ >>> s['a'] = 10
+ >>> s.allow_new_attr(False)
+ >>> s['a'] = 10
+ >>> s['a']
+ 10
+ >>> try:
+ ... s['b'] = 20
+ ... except KeyError:
+ ... print('this is not allowed')
+ ...
+ this is not allowed
+ """
+ if not self._allownew and key not in self:
+ raise KeyError(
+ "can't create new attribute %s when allow_new_attr(False)" % key)
+ dict.__setitem__(self, key, value)
+
+ def __setattr__(self, key, value):
+ """Set an attr with protection of class members.
+
+ This calls :meth:`self.__setitem__` but convert :exc:`KeyError` to
+ :exc:`AttributeError`.
+
+ Examples
+ --------
+
+ >>> s = Struct()
+ >>> s.a = 10
+ >>> s.a
+ 10
+ >>> try:
+ ... s.get = 10
+ ... except AttributeError:
+ ... print("you can't set a class member")
+ ...
+ you can't set a class member
+ """
+ # If key is an str it might be a class member or instance var
+ if isinstance(key, str):
+ # I can't simply call hasattr here because it calls getattr, which
+ # calls self.__getattr__, which returns True for keys in
+ # self._data. But I only want keys in the class and in
+ # self.__dict__
+ if key in self.__dict__ or hasattr(Struct, key):
+ raise AttributeError(
+ 'attr %s is a protected member of class Struct.' % key
+ )
+ try:
+ self.__setitem__(key, value)
+ except KeyError as e:
+ raise AttributeError(e)
+
+ def __getattr__(self, key):
+ """Get an attr by calling :meth:`dict.__getitem__`.
+
+ Like :meth:`__setattr__`, this method converts :exc:`KeyError` to
+ :exc:`AttributeError`.
+
+ Examples
+ --------
+
+ >>> s = Struct(a=10)
+ >>> s.a
+ 10
+ >>> type(s.get)
+ <... 'builtin_function_or_method'>
+ >>> try:
+ ... s.b
+ ... except AttributeError:
+ ... print("I don't have that key")
+ ...
+ I don't have that key
+ """
+ try:
+ result = self[key]
+ except KeyError:
+ raise AttributeError(key)
+ else:
+ return result
+
+ def __iadd__(self, other):
+ """s += s2 is a shorthand for s.merge(s2).
+
+ Examples
+ --------
+
+ >>> s = Struct(a=10,b=30)
+ >>> s2 = Struct(a=20,c=40)
+ >>> s += s2
+ >>> sorted(s.keys())
+ ['a', 'b', 'c']
+ """
+ self.merge(other)
+ return self
+
+ def __add__(self,other):
+ """s + s2 -> New Struct made from s.merge(s2).
+
+ Examples
+ --------
+
+ >>> s1 = Struct(a=10,b=30)
+ >>> s2 = Struct(a=20,c=40)
+ >>> s = s1 + s2
+ >>> sorted(s.keys())
+ ['a', 'b', 'c']
+ """
+ sout = self.copy()
+ sout.merge(other)
+ return sout
+
+ def __sub__(self,other):
+ """s1 - s2 -> remove keys in s2 from s1.
+
+ Examples
+ --------
+
+ >>> s1 = Struct(a=10,b=30)
+ >>> s2 = Struct(a=40)
+ >>> s = s1 - s2
+ >>> s
+ {'b': 30}
+ """
+ sout = self.copy()
+ sout -= other
+ return sout
+
+ def __isub__(self,other):
+ """Inplace remove keys from self that are in other.
+
+ Examples
+ --------
+
+ >>> s1 = Struct(a=10,b=30)
+ >>> s2 = Struct(a=40)
+ >>> s1 -= s2
+ >>> s1
+ {'b': 30}
+ """
+ for k in other.keys():
+ if k in self:
+ del self[k]
+ return self
+
+ def __dict_invert(self, data):
+ """Helper function for merge.
+
+ Takes a dictionary whose values are lists and returns a dict with
+ the elements of each list as keys and the original keys as values.
+ """
+ outdict = {}
+ for k,lst in data.items():
+ if isinstance(lst, str):
+ lst = lst.split()
+ for entry in lst:
+ outdict[entry] = k
+ return outdict
+
+ def dict(self):
+ return self
+
+ def copy(self):
+ """Return a copy as a Struct.
+
+ Examples
+ --------
+
+ >>> s = Struct(a=10,b=30)
+ >>> s2 = s.copy()
+ >>> type(s2) is Struct
+ True
+ """
+ return Struct(dict.copy(self))
+
+ def hasattr(self, key):
+ """hasattr function available as a method.
+
+ Implemented like has_key.
+
+ Examples
+ --------
+
+ >>> s = Struct(a=10)
+ >>> s.hasattr('a')
+ True
+ >>> s.hasattr('b')
+ False
+ >>> s.hasattr('get')
+ False
+ """
+ return key in self
+
+ def allow_new_attr(self, allow = True):
+ """Set whether new attributes can be created in this Struct.
+
+ This can be used to catch typos by verifying that the attribute user
+ tries to change already exists in this Struct.
+ """
+ object.__setattr__(self, '_allownew', allow)
+
+ def merge(self, __loc_data__=None, __conflict_solve=None, **kw):
+ """Merge two Structs with customizable conflict resolution.
+
+ This is similar to :meth:`update`, but much more flexible. First, a
+ dict is made from data+key=value pairs. When merging this dict with
+ the Struct S, the optional dictionary 'conflict' is used to decide
+ what to do.
+
+ If conflict is not given, the default behavior is to preserve any keys
+ with their current value (the opposite of the :meth:`update` method's
+ behavior).
+
+ Parameters
+ ----------
+ __loc_data : dict, Struct
+ The data to merge into self
+ __conflict_solve : dict
+ The conflict policy dict. The keys are binary functions used to
+ resolve the conflict and the values are lists of strings naming
+ the keys the conflict resolution function applies to. Instead of
+ a list of strings a space separated string can be used, like
+ 'a b c'.
+ kw : dict
+ Additional key, value pairs to merge in
+
+ Notes
+ -----
+
+ The `__conflict_solve` dict is a dictionary of binary functions which will be used to
+ solve key conflicts. Here is an example::
+
+ __conflict_solve = dict(
+ func1=['a','b','c'],
+ func2=['d','e']
+ )
+
+ In this case, the function :func:`func1` will be used to resolve
+ keys 'a', 'b' and 'c' and the function :func:`func2` will be used for
+ keys 'd' and 'e'. This could also be written as::
+
+ __conflict_solve = dict(func1='a b c',func2='d e')
+
+ These functions will be called for each key they apply to with the
+ form::
+
+ func1(self['a'], other['a'])
+
+ The return value is used as the final merged value.
+
+ As a convenience, merge() provides five (the most commonly needed)
+ pre-defined policies: preserve, update, add, add_flip and add_s. The
+ easiest explanation is their implementation::
+
+ preserve = lambda old,new: old
+ update = lambda old,new: new
+ add = lambda old,new: old + new
+ add_flip = lambda old,new: new + old # note change of order!
+ add_s = lambda old,new: old + ' ' + new # only for str!
+
+ You can use those four words (as strings) as keys instead
+ of defining them as functions, and the merge method will substitute
+ the appropriate functions for you.
+
+ For more complicated conflict resolution policies, you still need to
+ construct your own functions.
+
+ Examples
+ --------
+
+ This show the default policy:
+
+ >>> s = Struct(a=10,b=30)
+ >>> s2 = Struct(a=20,c=40)
+ >>> s.merge(s2)
+ >>> sorted(s.items())
+ [('a', 10), ('b', 30), ('c', 40)]
+
+ Now, show how to specify a conflict dict:
+
+ >>> s = Struct(a=10,b=30)
+ >>> s2 = Struct(a=20,b=40)
+ >>> conflict = {'update':'a','add':'b'}
+ >>> s.merge(s2,conflict)
+ >>> sorted(s.items())
+ [('a', 20), ('b', 70)]
+ """
+
+ data_dict = dict(__loc_data__,**kw)
+
+ # policies for conflict resolution: two argument functions which return
+ # the value that will go in the new struct
+ preserve = lambda old,new: old
+ update = lambda old,new: new
+ add = lambda old,new: old + new
+ add_flip = lambda old,new: new + old # note change of order!
+ add_s = lambda old,new: old + ' ' + new
+
+ # default policy is to keep current keys when there's a conflict
+ conflict_solve = dict.fromkeys(self, preserve)
+
+ # the conflict_solve dictionary is given by the user 'inverted': we
+ # need a name-function mapping, it comes as a function -> names
+ # dict. Make a local copy (b/c we'll make changes), replace user
+ # strings for the three builtin policies and invert it.
+ if __conflict_solve:
+ inv_conflict_solve_user = __conflict_solve.copy()
+ for name, func in [('preserve',preserve), ('update',update),
+ ('add',add), ('add_flip',add_flip),
+ ('add_s',add_s)]:
+ if name in inv_conflict_solve_user.keys():
+ inv_conflict_solve_user[func] = inv_conflict_solve_user[name]
+ del inv_conflict_solve_user[name]
+ conflict_solve.update(self.__dict_invert(inv_conflict_solve_user))
+ for key in data_dict:
+ if key not in self:
+ self[key] = data_dict[key]
+ else:
+ self[key] = conflict_solve[key](self[key],data_dict[key])
+
diff --git a/contrib/python/ipython/py2/IPython/utils/jsonutil.py b/contrib/python/ipython/py2/IPython/utils/jsonutil.py
index c3ee93859e..4bb400ca1e 100644
--- a/contrib/python/ipython/py2/IPython/utils/jsonutil.py
+++ b/contrib/python/ipython/py2/IPython/utils/jsonutil.py
@@ -1,5 +1,5 @@
-from warnings import warn
-
-warn("IPython.utils.jsonutil has moved to jupyter_client.jsonutil")
-
-from jupyter_client.jsonutil import *
+from warnings import warn
+
+warn("IPython.utils.jsonutil has moved to jupyter_client.jsonutil")
+
+from jupyter_client.jsonutil import *
diff --git a/contrib/python/ipython/py2/IPython/utils/localinterfaces.py b/contrib/python/ipython/py2/IPython/utils/localinterfaces.py
index 89b8fdeb54..f90564def5 100644
--- a/contrib/python/ipython/py2/IPython/utils/localinterfaces.py
+++ b/contrib/python/ipython/py2/IPython/utils/localinterfaces.py
@@ -1,5 +1,5 @@
-from warnings import warn
-
-warn("IPython.utils.localinterfaces has moved to jupyter_client.localinterfaces")
-
-from jupyter_client.localinterfaces import *
+from warnings import warn
+
+warn("IPython.utils.localinterfaces has moved to jupyter_client.localinterfaces")
+
+from jupyter_client.localinterfaces import *
diff --git a/contrib/python/ipython/py2/IPython/utils/log.py b/contrib/python/ipython/py2/IPython/utils/log.py
index 3eb9bdadd8..422bb9b343 100644
--- a/contrib/python/ipython/py2/IPython/utils/log.py
+++ b/contrib/python/ipython/py2/IPython/utils/log.py
@@ -1,7 +1,7 @@
-from __future__ import absolute_import
-
-from warnings import warn
-
-warn("IPython.utils.log has moved to traitlets.log")
-
-from traitlets.log import *
+from __future__ import absolute_import
+
+from warnings import warn
+
+warn("IPython.utils.log has moved to traitlets.log")
+
+from traitlets.log import *
diff --git a/contrib/python/ipython/py2/IPython/utils/module_paths.py b/contrib/python/ipython/py2/IPython/utils/module_paths.py
index 45a711c0b4..fc2c7f07c0 100644
--- a/contrib/python/ipython/py2/IPython/utils/module_paths.py
+++ b/contrib/python/ipython/py2/IPython/utils/module_paths.py
@@ -1,125 +1,125 @@
-"""Utility functions for finding modules
-
-Utility functions for finding modules on sys.path.
-
-`find_mod` finds named module on sys.path.
-
-`get_init` helper function that finds __init__ file in a directory.
-
-`find_module` variant of imp.find_module in std_lib that only returns
-path to module and not an open file object as well.
-
-
-
-"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2011, the IPython Development Team.
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
-
-# Stdlib imports
-import imp
-import os
-
-# Third-party imports
-
-# Our own imports
-
-
-#-----------------------------------------------------------------------------
-# Globals and constants
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Local utilities
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Classes and functions
-#-----------------------------------------------------------------------------
-def find_module(name, path=None):
- """imp.find_module variant that only return path of module.
-
- The `imp.find_module` returns a filehandle that we are not interested in.
- Also we ignore any bytecode files that `imp.find_module` finds.
-
- Parameters
- ----------
- name : str
- name of module to locate
- path : list of str
- list of paths to search for `name`. If path=None then search sys.path
-
- Returns
- -------
- filename : str
- Return full path of module or None if module is missing or does not have
- .py or .pyw extension
- """
- if name is None:
- return None
- try:
- file, filename, _ = imp.find_module(name, path)
- except ImportError:
- return None
- if file is None:
- return filename
- else:
- file.close()
- if os.path.splitext(filename)[1] in [".py", ".pyc"]:
- return filename
- else:
- return None
-
-def get_init(dirname):
- """Get __init__ file path for module directory
-
- Parameters
- ----------
- dirname : str
- Find the __init__ file in directory `dirname`
-
- Returns
- -------
- init_path : str
- Path to __init__ file
- """
- fbase = os.path.join(dirname, "__init__")
- for ext in [".py", ".pyw"]:
- fname = fbase + ext
- if os.path.isfile(fname):
- return fname
-
-
-def find_mod(module_name):
- """Find module `module_name` on sys.path
-
- Return the path to module `module_name`. If `module_name` refers to
- a module directory then return path to __init__ file. Return full
- path of module or None if module is missing or does not have .py or .pyw
- extension. We are not interested in running bytecode.
-
- Parameters
- ----------
- module_name : str
-
- Returns
- -------
- modulepath : str
- Path to module `module_name`.
- """
- parts = module_name.split(".")
- basepath = find_module(parts[0])
- for submodname in parts[1:]:
- basepath = find_module(submodname, [basepath])
- if basepath and os.path.isdir(basepath):
- basepath = get_init(basepath)
- return basepath
+"""Utility functions for finding modules
+
+Utility functions for finding modules on sys.path.
+
+`find_mod` finds named module on sys.path.
+
+`get_init` helper function that finds __init__ file in a directory.
+
+`find_module` variant of imp.find_module in std_lib that only returns
+path to module and not an open file object as well.
+
+
+
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2011, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib imports
+import imp
+import os
+
+# Third-party imports
+
+# Our own imports
+
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Local utilities
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+def find_module(name, path=None):
+ """imp.find_module variant that only return path of module.
+
+ The `imp.find_module` returns a filehandle that we are not interested in.
+ Also we ignore any bytecode files that `imp.find_module` finds.
+
+ Parameters
+ ----------
+ name : str
+ name of module to locate
+ path : list of str
+ list of paths to search for `name`. If path=None then search sys.path
+
+ Returns
+ -------
+ filename : str
+ Return full path of module or None if module is missing or does not have
+ .py or .pyw extension
+ """
+ if name is None:
+ return None
+ try:
+ file, filename, _ = imp.find_module(name, path)
+ except ImportError:
+ return None
+ if file is None:
+ return filename
+ else:
+ file.close()
+ if os.path.splitext(filename)[1] in [".py", ".pyc"]:
+ return filename
+ else:
+ return None
+
+def get_init(dirname):
+ """Get __init__ file path for module directory
+
+ Parameters
+ ----------
+ dirname : str
+ Find the __init__ file in directory `dirname`
+
+ Returns
+ -------
+ init_path : str
+ Path to __init__ file
+ """
+ fbase = os.path.join(dirname, "__init__")
+ for ext in [".py", ".pyw"]:
+ fname = fbase + ext
+ if os.path.isfile(fname):
+ return fname
+
+
+def find_mod(module_name):
+ """Find module `module_name` on sys.path
+
+ Return the path to module `module_name`. If `module_name` refers to
+ a module directory then return path to __init__ file. Return full
+ path of module or None if module is missing or does not have .py or .pyw
+ extension. We are not interested in running bytecode.
+
+ Parameters
+ ----------
+ module_name : str
+
+ Returns
+ -------
+ modulepath : str
+ Path to module `module_name`.
+ """
+ parts = module_name.split(".")
+ basepath = find_module(parts[0])
+ for submodname in parts[1:]:
+ basepath = find_module(submodname, [basepath])
+ if basepath and os.path.isdir(basepath):
+ basepath = get_init(basepath)
+ return basepath
diff --git a/contrib/python/ipython/py2/IPython/utils/openpy.py b/contrib/python/ipython/py2/IPython/utils/openpy.py
index 0a7cc0f00e..f55f254bc1 100644
--- a/contrib/python/ipython/py2/IPython/utils/openpy.py
+++ b/contrib/python/ipython/py2/IPython/utils/openpy.py
@@ -1,249 +1,249 @@
-"""
-Tools to open .py files as Unicode, using the encoding specified within the file,
-as per PEP 263.
-
-Much of the code is taken from the tokenize module in Python 3.2.
-"""
-from __future__ import absolute_import
-
-import io
-from io import TextIOWrapper, BytesIO
-import os.path
-import re
-
-from .py3compat import unicode_type
-
-cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE)
-cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)
-
-try:
- # Available in Python 3
- from tokenize import detect_encoding
-except ImportError:
- from codecs import lookup, BOM_UTF8
-
- # Copied from Python 3.2 tokenize
- def _get_normal_name(orig_enc):
- """Imitates get_normal_name in tokenizer.c."""
- # Only care about the first 12 characters.
- enc = orig_enc[:12].lower().replace("_", "-")
- if enc == "utf-8" or enc.startswith("utf-8-"):
- return "utf-8"
- if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
- enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
- return "iso-8859-1"
- return orig_enc
-
- # Copied from Python 3.2 tokenize
- def detect_encoding(readline):
- """
- The detect_encoding() function is used to detect the encoding that should
- be used to decode a Python source file. It requires one argment, readline,
- in the same way as the tokenize() generator.
-
- It will call readline a maximum of twice, and return the encoding used
- (as a string) and a list of any lines (left as bytes) it has read in.
-
- It detects the encoding from the presence of a utf-8 bom or an encoding
- cookie as specified in pep-0263. If both a bom and a cookie are present,
- but disagree, a SyntaxError will be raised. If the encoding cookie is an
- invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
- 'utf-8-sig' is returned.
-
- If no encoding is specified, then the default of 'utf-8' will be returned.
- """
- bom_found = False
- encoding = None
- default = 'utf-8'
- def read_or_stop():
- try:
- return readline()
- except StopIteration:
- return b''
-
- def find_cookie(line):
- try:
- line_string = line.decode('ascii')
- except UnicodeDecodeError:
- return None
-
- matches = cookie_re.findall(line_string)
- if not matches:
- return None
- encoding = _get_normal_name(matches[0])
- try:
- codec = lookup(encoding)
- except LookupError:
- # This behaviour mimics the Python interpreter
- raise SyntaxError("unknown encoding: " + encoding)
-
- if bom_found:
- if codec.name != 'utf-8':
- # This behaviour mimics the Python interpreter
- raise SyntaxError('encoding problem: utf-8')
- encoding += '-sig'
- return encoding
-
- first = read_or_stop()
- if first.startswith(BOM_UTF8):
- bom_found = True
- first = first[3:]
- default = 'utf-8-sig'
- if not first:
- return default, []
-
- encoding = find_cookie(first)
- if encoding:
- return encoding, [first]
-
- second = read_or_stop()
- if not second:
- return default, [first]
-
- encoding = find_cookie(second)
- if encoding:
- return encoding, [first, second]
-
- return default, [first, second]
-
-try:
- # Available in Python 3.2 and above.
- from tokenize import open
-except ImportError:
- # Copied from Python 3.2 tokenize
- def open(filename):
- """Open a file in read only mode using the encoding detected by
- detect_encoding().
- """
- buffer = io.open(filename, 'rb') # Tweaked to use io.open for Python 2
- encoding, lines = detect_encoding(buffer.readline)
- buffer.seek(0)
- text = TextIOWrapper(buffer, encoding, line_buffering=True)
- text.mode = 'r'
- return text
-
-def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
- """Converts a bytes string with python source code to unicode.
-
- Unicode strings are passed through unchanged. Byte strings are checked
- for the python source file encoding cookie to determine encoding.
- txt can be either a bytes buffer or a string containing the source
- code.
- """
- if isinstance(txt, unicode_type):
- return txt
- if isinstance(txt, bytes):
- buffer = BytesIO(txt)
- else:
- buffer = txt
- try:
- encoding, _ = detect_encoding(buffer.readline)
- except SyntaxError:
- encoding = "ascii"
- buffer.seek(0)
- text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
- text.mode = 'r'
- if skip_encoding_cookie:
- return u"".join(strip_encoding_cookie(text))
- else:
- return text.read()
-
-def strip_encoding_cookie(filelike):
- """Generator to pull lines from a text-mode file, skipping the encoding
- cookie if it is found in the first two lines.
- """
- it = iter(filelike)
- try:
- first = next(it)
- if not cookie_comment_re.match(first):
- yield first
- second = next(it)
- if not cookie_comment_re.match(second):
- yield second
- except StopIteration:
- return
-
- for line in it:
- yield line
-
-def read_py_file(filename, skip_encoding_cookie=True):
- """Read a Python file, using the encoding declared inside the file.
-
- Parameters
- ----------
- filename : str
- The path to the file to read.
- skip_encoding_cookie : bool
- If True (the default), and the encoding declaration is found in the first
- two lines, that line will be excluded from the output - compiling a
- unicode string with an encoding declaration is a SyntaxError in Python 2.
-
- Returns
- -------
- A unicode string containing the contents of the file.
- """
- with open(filename) as f: # the open function defined in this module.
- if skip_encoding_cookie:
- return "".join(strip_encoding_cookie(f))
- else:
- return f.read()
-
-def read_py_url(url, errors='replace', skip_encoding_cookie=True):
- """Read a Python file from a URL, using the encoding declared inside the file.
-
- Parameters
- ----------
- url : str
- The URL from which to fetch the file.
- errors : str
- How to handle decoding errors in the file. Options are the same as for
- bytes.decode(), but here 'replace' is the default.
- skip_encoding_cookie : bool
- If True (the default), and the encoding declaration is found in the first
- two lines, that line will be excluded from the output - compiling a
- unicode string with an encoding declaration is a SyntaxError in Python 2.
-
- Returns
- -------
- A unicode string containing the contents of the file.
- """
- # Deferred import for faster start
- try:
- from urllib.request import urlopen # Py 3
- except ImportError:
- from urllib import urlopen
- response = urlopen(url)
- buffer = io.BytesIO(response.read())
- return source_to_unicode(buffer, errors, skip_encoding_cookie)
-
-def _list_readline(x):
- """Given a list, returns a readline() function that returns the next element
- with each call.
- """
- x = iter(x)
- def readline():
- return next(x)
- return readline
-
-# Code for going between .py files and cached .pyc files ----------------------
-
-try: # Python 3.2, see PEP 3147
+"""
+Tools to open .py files as Unicode, using the encoding specified within the file,
+as per PEP 263.
+
+Much of the code is taken from the tokenize module in Python 3.2.
+"""
+from __future__ import absolute_import
+
+import io
+from io import TextIOWrapper, BytesIO
+import os.path
+import re
+
+from .py3compat import unicode_type
+
+cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE)
+cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)
+
+try:
+ # Available in Python 3
+ from tokenize import detect_encoding
+except ImportError:
+ from codecs import lookup, BOM_UTF8
+
+ # Copied from Python 3.2 tokenize
+ def _get_normal_name(orig_enc):
+ """Imitates get_normal_name in tokenizer.c."""
+ # Only care about the first 12 characters.
+ enc = orig_enc[:12].lower().replace("_", "-")
+ if enc == "utf-8" or enc.startswith("utf-8-"):
+ return "utf-8"
+ if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
+ enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
+ return "iso-8859-1"
+ return orig_enc
+
+ # Copied from Python 3.2 tokenize
+ def detect_encoding(readline):
+ """
+ The detect_encoding() function is used to detect the encoding that should
+ be used to decode a Python source file. It requires one argment, readline,
+ in the same way as the tokenize() generator.
+
+ It will call readline a maximum of twice, and return the encoding used
+ (as a string) and a list of any lines (left as bytes) it has read in.
+
+ It detects the encoding from the presence of a utf-8 bom or an encoding
+ cookie as specified in pep-0263. If both a bom and a cookie are present,
+ but disagree, a SyntaxError will be raised. If the encoding cookie is an
+ invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
+ 'utf-8-sig' is returned.
+
+ If no encoding is specified, then the default of 'utf-8' will be returned.
+ """
+ bom_found = False
+ encoding = None
+ default = 'utf-8'
+ def read_or_stop():
+ try:
+ return readline()
+ except StopIteration:
+ return b''
+
+ def find_cookie(line):
+ try:
+ line_string = line.decode('ascii')
+ except UnicodeDecodeError:
+ return None
+
+ matches = cookie_re.findall(line_string)
+ if not matches:
+ return None
+ encoding = _get_normal_name(matches[0])
+ try:
+ codec = lookup(encoding)
+ except LookupError:
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError("unknown encoding: " + encoding)
+
+ if bom_found:
+ if codec.name != 'utf-8':
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError('encoding problem: utf-8')
+ encoding += '-sig'
+ return encoding
+
+ first = read_or_stop()
+ if first.startswith(BOM_UTF8):
+ bom_found = True
+ first = first[3:]
+ default = 'utf-8-sig'
+ if not first:
+ return default, []
+
+ encoding = find_cookie(first)
+ if encoding:
+ return encoding, [first]
+
+ second = read_or_stop()
+ if not second:
+ return default, [first]
+
+ encoding = find_cookie(second)
+ if encoding:
+ return encoding, [first, second]
+
+ return default, [first, second]
+
+try:
+ # Available in Python 3.2 and above.
+ from tokenize import open
+except ImportError:
+ # Copied from Python 3.2 tokenize
+ def open(filename):
+ """Open a file in read only mode using the encoding detected by
+ detect_encoding().
+ """
+ buffer = io.open(filename, 'rb') # Tweaked to use io.open for Python 2
+ encoding, lines = detect_encoding(buffer.readline)
+ buffer.seek(0)
+ text = TextIOWrapper(buffer, encoding, line_buffering=True)
+ text.mode = 'r'
+ return text
+
+def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
+ """Converts a bytes string with python source code to unicode.
+
+ Unicode strings are passed through unchanged. Byte strings are checked
+ for the python source file encoding cookie to determine encoding.
+ txt can be either a bytes buffer or a string containing the source
+ code.
+ """
+ if isinstance(txt, unicode_type):
+ return txt
+ if isinstance(txt, bytes):
+ buffer = BytesIO(txt)
+ else:
+ buffer = txt
try:
- from importlib.util import source_from_cache, cache_from_source
- except ImportError :
- ## deprecated since 3.4
- from imp import source_from_cache, cache_from_source
-except ImportError:
- # Python <= 3.1: .pyc files go next to .py
- def source_from_cache(path):
- basename, ext = os.path.splitext(path)
- if ext not in ('.pyc', '.pyo'):
- raise ValueError('Not a cached Python file extension', ext)
- # Should we look for .pyw files?
- return basename + '.py'
-
- def cache_from_source(path, debug_override=None):
- if debug_override is None:
- debug_override = __debug__
- basename, ext = os.path.splitext(path)
- return basename + '.pyc' if debug_override else '.pyo'
+ encoding, _ = detect_encoding(buffer.readline)
+ except SyntaxError:
+ encoding = "ascii"
+ buffer.seek(0)
+ text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
+ text.mode = 'r'
+ if skip_encoding_cookie:
+ return u"".join(strip_encoding_cookie(text))
+ else:
+ return text.read()
+
+def strip_encoding_cookie(filelike):
+ """Generator to pull lines from a text-mode file, skipping the encoding
+ cookie if it is found in the first two lines.
+ """
+ it = iter(filelike)
+ try:
+ first = next(it)
+ if not cookie_comment_re.match(first):
+ yield first
+ second = next(it)
+ if not cookie_comment_re.match(second):
+ yield second
+ except StopIteration:
+ return
+
+ for line in it:
+ yield line
+
+def read_py_file(filename, skip_encoding_cookie=True):
+ """Read a Python file, using the encoding declared inside the file.
+
+ Parameters
+ ----------
+ filename : str
+ The path to the file to read.
+ skip_encoding_cookie : bool
+ If True (the default), and the encoding declaration is found in the first
+ two lines, that line will be excluded from the output - compiling a
+ unicode string with an encoding declaration is a SyntaxError in Python 2.
+
+ Returns
+ -------
+ A unicode string containing the contents of the file.
+ """
+ with open(filename) as f: # the open function defined in this module.
+ if skip_encoding_cookie:
+ return "".join(strip_encoding_cookie(f))
+ else:
+ return f.read()
+
+def read_py_url(url, errors='replace', skip_encoding_cookie=True):
+ """Read a Python file from a URL, using the encoding declared inside the file.
+
+ Parameters
+ ----------
+ url : str
+ The URL from which to fetch the file.
+ errors : str
+ How to handle decoding errors in the file. Options are the same as for
+ bytes.decode(), but here 'replace' is the default.
+ skip_encoding_cookie : bool
+ If True (the default), and the encoding declaration is found in the first
+ two lines, that line will be excluded from the output - compiling a
+ unicode string with an encoding declaration is a SyntaxError in Python 2.
+
+ Returns
+ -------
+ A unicode string containing the contents of the file.
+ """
+ # Deferred import for faster start
+ try:
+ from urllib.request import urlopen # Py 3
+ except ImportError:
+ from urllib import urlopen
+ response = urlopen(url)
+ buffer = io.BytesIO(response.read())
+ return source_to_unicode(buffer, errors, skip_encoding_cookie)
+
+def _list_readline(x):
+ """Given a list, returns a readline() function that returns the next element
+ with each call.
+ """
+ x = iter(x)
+ def readline():
+ return next(x)
+ return readline
+
+# Code for going between .py files and cached .pyc files ----------------------
+
+try: # Python 3.2, see PEP 3147
+ try:
+ from importlib.util import source_from_cache, cache_from_source
+ except ImportError :
+ ## deprecated since 3.4
+ from imp import source_from_cache, cache_from_source
+except ImportError:
+ # Python <= 3.1: .pyc files go next to .py
+ def source_from_cache(path):
+ basename, ext = os.path.splitext(path)
+ if ext not in ('.pyc', '.pyo'):
+ raise ValueError('Not a cached Python file extension', ext)
+ # Should we look for .pyw files?
+ return basename + '.py'
+
+ def cache_from_source(path, debug_override=None):
+ if debug_override is None:
+ debug_override = __debug__
+ basename, ext = os.path.splitext(path)
+ return basename + '.pyc' if debug_override else '.pyo'
diff --git a/contrib/python/ipython/py2/IPython/utils/path.py b/contrib/python/ipython/py2/IPython/utils/path.py
index fa850812c7..800e0e13ee 100644
--- a/contrib/python/ipython/py2/IPython/utils/path.py
+++ b/contrib/python/ipython/py2/IPython/utils/path.py
@@ -1,447 +1,447 @@
-# encoding: utf-8
-"""
-Utilities for path handling.
-"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-import os
-import sys
-import errno
-import shutil
-import random
-import glob
-from warnings import warn
-from hashlib import md5
-
-from IPython.utils.process import system
-from IPython.utils import py3compat
-from IPython.utils.decorators import undoc
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-fs_encoding = sys.getfilesystemencoding()
-
-def _writable_dir(path):
- """Whether `path` is a directory, to which the user has write access."""
- return os.path.isdir(path) and os.access(path, os.W_OK)
-
-if sys.platform == 'win32':
- def _get_long_path_name(path):
- """Get a long path name (expand ~) on Windows using ctypes.
-
- Examples
- --------
-
- >>> get_long_path_name('c:\\docume~1')
- u'c:\\\\Documents and Settings'
-
- """
- try:
- import ctypes
- except ImportError:
- raise ImportError('you need to have ctypes installed for this to work')
- _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
- _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
- ctypes.c_uint ]
-
- buf = ctypes.create_unicode_buffer(260)
- rv = _GetLongPathName(path, buf, 260)
- if rv == 0 or rv > 260:
- return path
- else:
- return buf.value
-else:
- def _get_long_path_name(path):
- """Dummy no-op."""
- return path
-
-
-
-def get_long_path_name(path):
- """Expand a path into its long form.
-
- On Windows this expands any ~ in the paths. On other platforms, it is
- a null operation.
- """
- return _get_long_path_name(path)
-
-
-def unquote_filename(name, win32=(sys.platform=='win32')):
- """ On Windows, remove leading and trailing quotes from filenames.
+# encoding: utf-8
+"""
+Utilities for path handling.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import os
+import sys
+import errno
+import shutil
+import random
+import glob
+from warnings import warn
+from hashlib import md5
+
+from IPython.utils.process import system
+from IPython.utils import py3compat
+from IPython.utils.decorators import undoc
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+fs_encoding = sys.getfilesystemencoding()
+
+def _writable_dir(path):
+ """Whether `path` is a directory, to which the user has write access."""
+ return os.path.isdir(path) and os.access(path, os.W_OK)
+
+if sys.platform == 'win32':
+ def _get_long_path_name(path):
+ """Get a long path name (expand ~) on Windows using ctypes.
+
+ Examples
+ --------
+
+ >>> get_long_path_name('c:\\docume~1')
+ u'c:\\\\Documents and Settings'
+
+ """
+ try:
+ import ctypes
+ except ImportError:
+ raise ImportError('you need to have ctypes installed for this to work')
+ _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
+ _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
+ ctypes.c_uint ]
+
+ buf = ctypes.create_unicode_buffer(260)
+ rv = _GetLongPathName(path, buf, 260)
+ if rv == 0 or rv > 260:
+ return path
+ else:
+ return buf.value
+else:
+ def _get_long_path_name(path):
+ """Dummy no-op."""
+ return path
+
+
+
+def get_long_path_name(path):
+ """Expand a path into its long form.
+
+ On Windows this expands any ~ in the paths. On other platforms, it is
+ a null operation.
+ """
+ return _get_long_path_name(path)
+
+
+def unquote_filename(name, win32=(sys.platform=='win32')):
+ """ On Windows, remove leading and trailing quotes from filenames.
This function has been deprecated and should not be used any more:
unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
- """
+ """
warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
"be used anymore", DeprecationWarning, stacklevel=2)
- if win32:
- if name.startswith(("'", '"')) and name.endswith(("'", '"')):
- name = name[1:-1]
- return name
-
-
-def compress_user(path):
- """Reverse of :func:`os.path.expanduser`
+ if win32:
+ if name.startswith(("'", '"')) and name.endswith(("'", '"')):
+ name = name[1:-1]
+ return name
+
+
+def compress_user(path):
+ """Reverse of :func:`os.path.expanduser`
"""
- path = py3compat.unicode_to_str(path, sys.getfilesystemencoding())
- home = os.path.expanduser('~')
- if path.startswith(home):
- path = "~" + path[len(home):]
- return path
-
-def get_py_filename(name, force_win32=None):
- """Return a valid python filename in the current directory.
-
- If the given name is not a file, it adds '.py' and searches again.
- Raises IOError with an informative message if the file isn't found.
- """
-
- name = os.path.expanduser(name)
+ path = py3compat.unicode_to_str(path, sys.getfilesystemencoding())
+ home = os.path.expanduser('~')
+ if path.startswith(home):
+ path = "~" + path[len(home):]
+ return path
+
+def get_py_filename(name, force_win32=None):
+ """Return a valid python filename in the current directory.
+
+ If the given name is not a file, it adds '.py' and searches again.
+ Raises IOError with an informative message if the file isn't found.
+ """
+
+ name = os.path.expanduser(name)
if force_win32 is not None:
warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
"since IPython 5.0 and should not be used anymore",
DeprecationWarning, stacklevel=2)
- if not os.path.isfile(name) and not name.endswith('.py'):
- name += '.py'
- if os.path.isfile(name):
- return name
- else:
- raise IOError('File `%r` not found.' % name)
-
-
-def filefind(filename, path_dirs=None):
- """Find a file by looking through a sequence of paths.
-
- This iterates through a sequence of paths looking for a file and returns
- the full, absolute path of the first occurence of the file. If no set of
- path dirs is given, the filename is tested as is, after running through
- :func:`expandvars` and :func:`expanduser`. Thus a simple call::
-
- filefind('myfile.txt')
-
- will find the file in the current working dir, but::
-
- filefind('~/myfile.txt')
-
- Will find the file in the users home directory. This function does not
- automatically try any paths, such as the cwd or the user's home directory.
-
- Parameters
- ----------
- filename : str
- The filename to look for.
- path_dirs : str, None or sequence of str
- The sequence of paths to look for the file in. If None, the filename
- need to be absolute or be in the cwd. If a string, the string is
- put into a sequence and the searched. If a sequence, walk through
- each element and join with ``filename``, calling :func:`expandvars`
- and :func:`expanduser` before testing for existence.
-
- Returns
- -------
- Raises :exc:`IOError` or returns absolute path to file.
- """
-
- # If paths are quoted, abspath gets confused, strip them...
- filename = filename.strip('"').strip("'")
- # If the input is an absolute path, just check it exists
- if os.path.isabs(filename) and os.path.isfile(filename):
- return filename
-
- if path_dirs is None:
- path_dirs = ("",)
- elif isinstance(path_dirs, py3compat.string_types):
- path_dirs = (path_dirs,)
-
- for path in path_dirs:
- if path == '.': path = py3compat.getcwd()
- testname = expand_path(os.path.join(path, filename))
- if os.path.isfile(testname):
- return os.path.abspath(testname)
-
- raise IOError("File %r does not exist in any of the search paths: %r" %
- (filename, path_dirs) )
-
-
-class HomeDirError(Exception):
- pass
-
-
-def get_home_dir(require_writable=False):
- """Return the 'home' directory, as a unicode string.
-
- Uses os.path.expanduser('~'), and checks for writability.
-
- See stdlib docs for how this is determined.
- $HOME is first priority on *ALL* platforms.
-
- Parameters
- ----------
-
- require_writable : bool [default: False]
- if True:
- guarantees the return value is a writable directory, otherwise
- raises HomeDirError
- if False:
- The path is resolved, but it is not guaranteed to exist or be writable.
- """
-
- homedir = os.path.expanduser('~')
- # Next line will make things work even when /home/ is a symlink to
- # /usr/home as it is on FreeBSD, for example
- homedir = os.path.realpath(homedir)
-
- if not _writable_dir(homedir) and os.name == 'nt':
- # expanduser failed, use the registry to get the 'My Documents' folder.
- try:
- try:
- import winreg as wreg # Py 3
- except ImportError:
- import _winreg as wreg # Py 2
- key = wreg.OpenKey(
- wreg.HKEY_CURRENT_USER,
- "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
- )
- homedir = wreg.QueryValueEx(key,'Personal')[0]
- key.Close()
- except:
- pass
-
- if (not require_writable) or _writable_dir(homedir):
- return py3compat.cast_unicode(homedir, fs_encoding)
- else:
- raise HomeDirError('%s is not a writable dir, '
- 'set $HOME environment variable to override' % homedir)
-
-def get_xdg_dir():
- """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
-
- This is only for non-OS X posix (Linux,Unix,etc.) systems.
- """
-
- env = os.environ
-
- if os.name == 'posix' and sys.platform != 'darwin':
- # Linux, Unix, AIX, etc.
- # use ~/.config if empty OR not set
- xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
- if xdg and _writable_dir(xdg):
- return py3compat.cast_unicode(xdg, fs_encoding)
-
- return None
-
-
-def get_xdg_cache_dir():
- """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
-
- This is only for non-OS X posix (Linux,Unix,etc.) systems.
- """
-
- env = os.environ
-
- if os.name == 'posix' and sys.platform != 'darwin':
- # Linux, Unix, AIX, etc.
- # use ~/.cache if empty OR not set
- xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
- if xdg and _writable_dir(xdg):
- return py3compat.cast_unicode(xdg, fs_encoding)
-
- return None
-
-
-@undoc
-def get_ipython_dir():
+ if not os.path.isfile(name) and not name.endswith('.py'):
+ name += '.py'
+ if os.path.isfile(name):
+ return name
+ else:
+ raise IOError('File `%r` not found.' % name)
+
+
+def filefind(filename, path_dirs=None):
+ """Find a file by looking through a sequence of paths.
+
+ This iterates through a sequence of paths looking for a file and returns
+ the full, absolute path of the first occurence of the file. If no set of
+ path dirs is given, the filename is tested as is, after running through
+ :func:`expandvars` and :func:`expanduser`. Thus a simple call::
+
+ filefind('myfile.txt')
+
+ will find the file in the current working dir, but::
+
+ filefind('~/myfile.txt')
+
+ Will find the file in the users home directory. This function does not
+ automatically try any paths, such as the cwd or the user's home directory.
+
+ Parameters
+ ----------
+ filename : str
+ The filename to look for.
+ path_dirs : str, None or sequence of str
+ The sequence of paths to look for the file in. If None, the filename
+ need to be absolute or be in the cwd. If a string, the string is
+ put into a sequence and the searched. If a sequence, walk through
+ each element and join with ``filename``, calling :func:`expandvars`
+ and :func:`expanduser` before testing for existence.
+
+ Returns
+ -------
+ Raises :exc:`IOError` or returns absolute path to file.
+ """
+
+ # If paths are quoted, abspath gets confused, strip them...
+ filename = filename.strip('"').strip("'")
+ # If the input is an absolute path, just check it exists
+ if os.path.isabs(filename) and os.path.isfile(filename):
+ return filename
+
+ if path_dirs is None:
+ path_dirs = ("",)
+ elif isinstance(path_dirs, py3compat.string_types):
+ path_dirs = (path_dirs,)
+
+ for path in path_dirs:
+ if path == '.': path = py3compat.getcwd()
+ testname = expand_path(os.path.join(path, filename))
+ if os.path.isfile(testname):
+ return os.path.abspath(testname)
+
+ raise IOError("File %r does not exist in any of the search paths: %r" %
+ (filename, path_dirs) )
+
+
+class HomeDirError(Exception):
+ pass
+
+
+def get_home_dir(require_writable=False):
+ """Return the 'home' directory, as a unicode string.
+
+ Uses os.path.expanduser('~'), and checks for writability.
+
+ See stdlib docs for how this is determined.
+ $HOME is first priority on *ALL* platforms.
+
+ Parameters
+ ----------
+
+ require_writable : bool [default: False]
+ if True:
+ guarantees the return value is a writable directory, otherwise
+ raises HomeDirError
+ if False:
+ The path is resolved, but it is not guaranteed to exist or be writable.
+ """
+
+ homedir = os.path.expanduser('~')
+ # Next line will make things work even when /home/ is a symlink to
+ # /usr/home as it is on FreeBSD, for example
+ homedir = os.path.realpath(homedir)
+
+ if not _writable_dir(homedir) and os.name == 'nt':
+ # expanduser failed, use the registry to get the 'My Documents' folder.
+ try:
+ try:
+ import winreg as wreg # Py 3
+ except ImportError:
+ import _winreg as wreg # Py 2
+ key = wreg.OpenKey(
+ wreg.HKEY_CURRENT_USER,
+ "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+ )
+ homedir = wreg.QueryValueEx(key,'Personal')[0]
+ key.Close()
+ except:
+ pass
+
+ if (not require_writable) or _writable_dir(homedir):
+ return py3compat.cast_unicode(homedir, fs_encoding)
+ else:
+ raise HomeDirError('%s is not a writable dir, '
+ 'set $HOME environment variable to override' % homedir)
+
+def get_xdg_dir():
+ """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
+
+ This is only for non-OS X posix (Linux,Unix,etc.) systems.
+ """
+
+ env = os.environ
+
+ if os.name == 'posix' and sys.platform != 'darwin':
+ # Linux, Unix, AIX, etc.
+ # use ~/.config if empty OR not set
+ xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
+ if xdg and _writable_dir(xdg):
+ return py3compat.cast_unicode(xdg, fs_encoding)
+
+ return None
+
+
+def get_xdg_cache_dir():
+ """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
+
+ This is only for non-OS X posix (Linux,Unix,etc.) systems.
+ """
+
+ env = os.environ
+
+ if os.name == 'posix' and sys.platform != 'darwin':
+ # Linux, Unix, AIX, etc.
+ # use ~/.cache if empty OR not set
+ xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
+ if xdg and _writable_dir(xdg):
+ return py3compat.cast_unicode(xdg, fs_encoding)
+
+ return None
+
+
+@undoc
+def get_ipython_dir():
warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
- from IPython.paths import get_ipython_dir
- return get_ipython_dir()
-
-@undoc
-def get_ipython_cache_dir():
+ from IPython.paths import get_ipython_dir
+ return get_ipython_dir()
+
+@undoc
+def get_ipython_cache_dir():
warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
- from IPython.paths import get_ipython_cache_dir
- return get_ipython_cache_dir()
-
-@undoc
-def get_ipython_package_dir():
+ from IPython.paths import get_ipython_cache_dir
+ return get_ipython_cache_dir()
+
+@undoc
+def get_ipython_package_dir():
warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
- from IPython.paths import get_ipython_package_dir
- return get_ipython_package_dir()
-
-@undoc
-def get_ipython_module_path(module_str):
+ from IPython.paths import get_ipython_package_dir
+ return get_ipython_package_dir()
+
+@undoc
+def get_ipython_module_path(module_str):
warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
- from IPython.paths import get_ipython_module_path
- return get_ipython_module_path(module_str)
-
-@undoc
-def locate_profile(profile='default'):
+ from IPython.paths import get_ipython_module_path
+ return get_ipython_module_path(module_str)
+
+@undoc
+def locate_profile(profile='default'):
warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
- from IPython.paths import locate_profile
- return locate_profile(profile=profile)
-
-def expand_path(s):
- """Expand $VARS and ~names in a string, like a shell
-
- :Examples:
-
- In [2]: os.environ['FOO']='test'
-
- In [3]: expand_path('variable FOO is $FOO')
- Out[3]: 'variable FOO is test'
- """
- # This is a pretty subtle hack. When expand user is given a UNC path
- # on Windows (\\server\share$\%username%), os.path.expandvars, removes
- # the $ to get (\\server\share\%username%). I think it considered $
- # alone an empty var. But, we need the $ to remains there (it indicates
- # a hidden share).
- if os.name=='nt':
- s = s.replace('$\\', 'IPYTHON_TEMP')
- s = os.path.expandvars(os.path.expanduser(s))
- if os.name=='nt':
- s = s.replace('IPYTHON_TEMP', '$\\')
- return s
-
-
-def unescape_glob(string):
- """Unescape glob pattern in `string`."""
- def unescape(s):
- for pattern in '*[]!?':
- s = s.replace(r'\{0}'.format(pattern), pattern)
- return s
- return '\\'.join(map(unescape, string.split('\\\\')))
-
-
-def shellglob(args):
- """
- Do glob expansion for each element in `args` and return a flattened list.
-
- Unmatched glob pattern will remain as-is in the returned list.
-
- """
- expanded = []
- # Do not unescape backslash in Windows as it is interpreted as
- # path separator:
- unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
- for a in args:
- expanded.extend(glob.glob(a) or [unescape(a)])
- return expanded
-
-
-def target_outdated(target,deps):
- """Determine whether a target is out of date.
-
- target_outdated(target,deps) -> 1/0
-
- deps: list of filenames which MUST exist.
- target: single filename which may or may not exist.
-
- If target doesn't exist or is older than any file listed in deps, return
- true, otherwise return false.
- """
- try:
- target_time = os.path.getmtime(target)
- except os.error:
- return 1
- for dep in deps:
- dep_time = os.path.getmtime(dep)
- if dep_time > target_time:
- #print "For target",target,"Dep failed:",dep # dbg
- #print "times (dep,tar):",dep_time,target_time # dbg
- return 1
- return 0
-
-
-def target_update(target,deps,cmd):
- """Update a target with a given command given a list of dependencies.
-
- target_update(target,deps,cmd) -> runs cmd if target is outdated.
-
- This is just a wrapper around target_outdated() which calls the given
- command if target is outdated."""
-
- if target_outdated(target,deps):
- system(cmd)
-
-@undoc
-def filehash(path):
- """Make an MD5 hash of a file, ignoring any differences in line
- ending characters."""
+ from IPython.paths import locate_profile
+ return locate_profile(profile=profile)
+
+def expand_path(s):
+ """Expand $VARS and ~names in a string, like a shell
+
+ :Examples:
+
+ In [2]: os.environ['FOO']='test'
+
+ In [3]: expand_path('variable FOO is $FOO')
+ Out[3]: 'variable FOO is test'
+ """
+ # This is a pretty subtle hack. When expand user is given a UNC path
+ # on Windows (\\server\share$\%username%), os.path.expandvars, removes
+ # the $ to get (\\server\share\%username%). I think it considered $
+ # alone an empty var. But, we need the $ to remains there (it indicates
+ # a hidden share).
+ if os.name=='nt':
+ s = s.replace('$\\', 'IPYTHON_TEMP')
+ s = os.path.expandvars(os.path.expanduser(s))
+ if os.name=='nt':
+ s = s.replace('IPYTHON_TEMP', '$\\')
+ return s
+
+
+def unescape_glob(string):
+ """Unescape glob pattern in `string`."""
+ def unescape(s):
+ for pattern in '*[]!?':
+ s = s.replace(r'\{0}'.format(pattern), pattern)
+ return s
+ return '\\'.join(map(unescape, string.split('\\\\')))
+
+
+def shellglob(args):
+ """
+ Do glob expansion for each element in `args` and return a flattened list.
+
+ Unmatched glob pattern will remain as-is in the returned list.
+
+ """
+ expanded = []
+ # Do not unescape backslash in Windows as it is interpreted as
+ # path separator:
+ unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
+ for a in args:
+ expanded.extend(glob.glob(a) or [unescape(a)])
+ return expanded
+
+
+def target_outdated(target,deps):
+ """Determine whether a target is out of date.
+
+ target_outdated(target,deps) -> 1/0
+
+ deps: list of filenames which MUST exist.
+ target: single filename which may or may not exist.
+
+ If target doesn't exist or is older than any file listed in deps, return
+ true, otherwise return false.
+ """
+ try:
+ target_time = os.path.getmtime(target)
+ except os.error:
+ return 1
+ for dep in deps:
+ dep_time = os.path.getmtime(dep)
+ if dep_time > target_time:
+ #print "For target",target,"Dep failed:",dep # dbg
+ #print "times (dep,tar):",dep_time,target_time # dbg
+ return 1
+ return 0
+
+
+def target_update(target,deps,cmd):
+ """Update a target with a given command given a list of dependencies.
+
+ target_update(target,deps,cmd) -> runs cmd if target is outdated.
+
+ This is just a wrapper around target_outdated() which calls the given
+ command if target is outdated."""
+
+ if target_outdated(target,deps):
+ system(cmd)
+
+@undoc
+def filehash(path):
+ """Make an MD5 hash of a file, ignoring any differences in line
+ ending characters."""
warn("filehash() is deprecated since IPython 4.0", DeprecationWarning, stacklevel=2)
- with open(path, "rU") as f:
- return md5(py3compat.str_to_bytes(f.read())).hexdigest()
-
-ENOLINK = 1998
-
-def link(src, dst):
- """Hard links ``src`` to ``dst``, returning 0 or errno.
-
- Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
- supported by the operating system.
- """
-
- if not hasattr(os, "link"):
- return ENOLINK
- link_errno = 0
- try:
- os.link(src, dst)
- except OSError as e:
- link_errno = e.errno
- return link_errno
-
-
-def link_or_copy(src, dst):
- """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
-
- Attempts to maintain the semantics of ``shutil.copy``.
-
- Because ``os.link`` does not overwrite files, a unique temporary file
- will be used if the target already exists, then that file will be moved
- into place.
- """
-
- if os.path.isdir(dst):
- dst = os.path.join(dst, os.path.basename(src))
-
- link_errno = link(src, dst)
- if link_errno == errno.EEXIST:
- if os.stat(src).st_ino == os.stat(dst).st_ino:
- # dst is already a hard link to the correct file, so we don't need
- # to do anything else. If we try to link and rename the file
- # anyway, we get duplicate files - see http://bugs.python.org/issue21876
- return
-
- new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
- try:
- link_or_copy(src, new_dst)
- except:
- try:
- os.remove(new_dst)
- except OSError:
- pass
- raise
- os.rename(new_dst, dst)
- elif link_errno != 0:
- # Either link isn't supported, or the filesystem doesn't support
- # linking, or 'src' and 'dst' are on different filesystems.
- shutil.copy(src, dst)
-
-def ensure_dir_exists(path, mode=0o755):
- """ensure that a directory exists
-
- If it doesn't exist, try to create it and protect against a race condition
- if another process is doing the same.
-
- The default permissions are 755, which differ from os.makedirs default of 777.
- """
- if not os.path.exists(path):
- try:
- os.makedirs(path, mode=mode)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- elif not os.path.isdir(path):
- raise IOError("%r exists but is not a directory" % path)
+ with open(path, "rU") as f:
+ return md5(py3compat.str_to_bytes(f.read())).hexdigest()
+
+ENOLINK = 1998
+
+def link(src, dst):
+ """Hard links ``src`` to ``dst``, returning 0 or errno.
+
+ Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
+ supported by the operating system.
+ """
+
+ if not hasattr(os, "link"):
+ return ENOLINK
+ link_errno = 0
+ try:
+ os.link(src, dst)
+ except OSError as e:
+ link_errno = e.errno
+ return link_errno
+
+
+def link_or_copy(src, dst):
+ """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
+
+ Attempts to maintain the semantics of ``shutil.copy``.
+
+ Because ``os.link`` does not overwrite files, a unique temporary file
+ will be used if the target already exists, then that file will be moved
+ into place.
+ """
+
+ if os.path.isdir(dst):
+ dst = os.path.join(dst, os.path.basename(src))
+
+ link_errno = link(src, dst)
+ if link_errno == errno.EEXIST:
+ if os.stat(src).st_ino == os.stat(dst).st_ino:
+ # dst is already a hard link to the correct file, so we don't need
+ # to do anything else. If we try to link and rename the file
+ # anyway, we get duplicate files - see http://bugs.python.org/issue21876
+ return
+
+ new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
+ try:
+ link_or_copy(src, new_dst)
+ except:
+ try:
+ os.remove(new_dst)
+ except OSError:
+ pass
+ raise
+ os.rename(new_dst, dst)
+ elif link_errno != 0:
+ # Either link isn't supported, or the filesystem doesn't support
+ # linking, or 'src' and 'dst' are on different filesystems.
+ shutil.copy(src, dst)
+
+def ensure_dir_exists(path, mode=0o755):
+ """ensure that a directory exists
+
+ If it doesn't exist, try to create it and protect against a race condition
+ if another process is doing the same.
+
+ The default permissions are 755, which differ from os.makedirs default of 777.
+ """
+ if not os.path.exists(path):
+ try:
+ os.makedirs(path, mode=mode)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ elif not os.path.isdir(path):
+ raise IOError("%r exists but is not a directory" % path)
diff --git a/contrib/python/ipython/py2/IPython/utils/pickleutil.py b/contrib/python/ipython/py2/IPython/utils/pickleutil.py
index 665ff09f2d..9111ad73c0 100644
--- a/contrib/python/ipython/py2/IPython/utils/pickleutil.py
+++ b/contrib/python/ipython/py2/IPython/utils/pickleutil.py
@@ -1,5 +1,5 @@
-from warnings import warn
-
-warn("IPython.utils.pickleutil has moved to ipykernel.pickleutil")
-
-from ipykernel.pickleutil import *
+from warnings import warn
+
+warn("IPython.utils.pickleutil has moved to ipykernel.pickleutil")
+
+from ipykernel.pickleutil import *
diff --git a/contrib/python/ipython/py2/IPython/utils/process.py b/contrib/python/ipython/py2/IPython/utils/process.py
index a274f43f3a..ca85a03b5c 100644
--- a/contrib/python/ipython/py2/IPython/utils/process.py
+++ b/contrib/python/ipython/py2/IPython/utils/process.py
@@ -1,106 +1,106 @@
-# encoding: utf-8
-"""
-Utilities for working with external processes.
-"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-from __future__ import print_function
-
-import os
-import sys
-
-if sys.platform == 'win32':
- from ._process_win32 import system, getoutput, arg_split, check_pid
-elif sys.platform == 'cli':
- from ._process_cli import system, getoutput, arg_split, check_pid
-else:
- from ._process_posix import system, getoutput, arg_split, check_pid
-
-from ._process_common import getoutputerror, get_output_error_code, process_handler
-from . import py3compat
-
-
-class FindCmdError(Exception):
- pass
-
-
-def find_cmd(cmd):
- """Find absolute path to executable cmd in a cross platform manner.
-
- This function tries to determine the full path to a command line program
- using `which` on Unix/Linux/OS X and `win32api` on Windows. Most of the
- time it will use the version that is first on the users `PATH`.
-
- Warning, don't use this to find IPython command line programs as there
- is a risk you will find the wrong one. Instead find those using the
- following code and looking for the application itself::
-
- from IPython.utils.path import get_ipython_module_path
- from IPython.utils.process import pycmd2argv
- argv = pycmd2argv(get_ipython_module_path('IPython.terminal.ipapp'))
-
- Parameters
- ----------
- cmd : str
- The command line program to look for.
- """
- path = py3compat.which(cmd)
- if path is None:
- raise FindCmdError('command could not be found: %s' % cmd)
- return path
-
-
-def is_cmd_found(cmd):
- """Check whether executable `cmd` exists or not and return a bool."""
- try:
- find_cmd(cmd)
- return True
- except FindCmdError:
- return False
-
-
-def pycmd2argv(cmd):
- r"""Take the path of a python command and return a list (argv-style).
-
- This only works on Python based command line programs and will find the
- location of the ``python`` executable using ``sys.executable`` to make
- sure the right version is used.
-
- For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
- .com or .bat, and [, cmd] otherwise.
-
- Parameters
- ----------
- cmd : string
- The path of the command.
-
- Returns
- -------
- argv-style list.
- """
- ext = os.path.splitext(cmd)[1]
- if ext in ['.exe', '.com', '.bat']:
- return [cmd]
- else:
- return [sys.executable, cmd]
-
-
-def abbrev_cwd():
- """ Return abbreviated version of cwd, e.g. d:mydir """
- cwd = py3compat.getcwd().replace('\\','/')
- drivepart = ''
- tail = cwd
- if sys.platform == 'win32':
- if len(cwd) < 4:
- return cwd
- drivepart,tail = os.path.splitdrive(cwd)
-
-
- parts = tail.split('/')
- if len(parts) > 2:
- tail = '/'.join(parts[-2:])
-
- return (drivepart + (
- cwd == '/' and '/' or tail))
+# encoding: utf-8
+"""
+Utilities for working with external processes.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from __future__ import print_function
+
+import os
+import sys
+
+if sys.platform == 'win32':
+ from ._process_win32 import system, getoutput, arg_split, check_pid
+elif sys.platform == 'cli':
+ from ._process_cli import system, getoutput, arg_split, check_pid
+else:
+ from ._process_posix import system, getoutput, arg_split, check_pid
+
+from ._process_common import getoutputerror, get_output_error_code, process_handler
+from . import py3compat
+
+
+class FindCmdError(Exception):
+ pass
+
+
+def find_cmd(cmd):
+ """Find absolute path to executable cmd in a cross platform manner.
+
+ This function tries to determine the full path to a command line program
+ using `which` on Unix/Linux/OS X and `win32api` on Windows. Most of the
+ time it will use the version that is first on the users `PATH`.
+
+ Warning, don't use this to find IPython command line programs as there
+ is a risk you will find the wrong one. Instead find those using the
+ following code and looking for the application itself::
+
+ from IPython.utils.path import get_ipython_module_path
+ from IPython.utils.process import pycmd2argv
+ argv = pycmd2argv(get_ipython_module_path('IPython.terminal.ipapp'))
+
+ Parameters
+ ----------
+ cmd : str
+ The command line program to look for.
+ """
+ path = py3compat.which(cmd)
+ if path is None:
+ raise FindCmdError('command could not be found: %s' % cmd)
+ return path
+
+
+def is_cmd_found(cmd):
+ """Check whether executable `cmd` exists or not and return a bool."""
+ try:
+ find_cmd(cmd)
+ return True
+ except FindCmdError:
+ return False
+
+
+def pycmd2argv(cmd):
+ r"""Take the path of a python command and return a list (argv-style).
+
+ This only works on Python based command line programs and will find the
+ location of the ``python`` executable using ``sys.executable`` to make
+ sure the right version is used.
+
+ For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
+ .com or .bat, and [, cmd] otherwise.
+
+ Parameters
+ ----------
+ cmd : string
+ The path of the command.
+
+ Returns
+ -------
+ argv-style list.
+ """
+ ext = os.path.splitext(cmd)[1]
+ if ext in ['.exe', '.com', '.bat']:
+ return [cmd]
+ else:
+ return [sys.executable, cmd]
+
+
+def abbrev_cwd():
+ """ Return abbreviated version of cwd, e.g. d:mydir """
+ cwd = py3compat.getcwd().replace('\\','/')
+ drivepart = ''
+ tail = cwd
+ if sys.platform == 'win32':
+ if len(cwd) < 4:
+ return cwd
+ drivepart,tail = os.path.splitdrive(cwd)
+
+
+ parts = tail.split('/')
+ if len(parts) > 2:
+ tail = '/'.join(parts[-2:])
+
+ return (drivepart + (
+ cwd == '/' and '/' or tail))
diff --git a/contrib/python/ipython/py2/IPython/utils/py3compat.py b/contrib/python/ipython/py2/IPython/utils/py3compat.py
index 88602e5342..adaac362a7 100644
--- a/contrib/python/ipython/py2/IPython/utils/py3compat.py
+++ b/contrib/python/ipython/py2/IPython/utils/py3compat.py
@@ -1,336 +1,336 @@
-# coding: utf-8
-"""Compatibility tricks for Python 3. Mainly to do with unicode."""
-import functools
-import os
-import sys
-import re
-import shutil
-import types
+# coding: utf-8
+"""Compatibility tricks for Python 3. Mainly to do with unicode."""
+import functools
+import os
+import sys
+import re
+import shutil
+import types
import platform
-
-from .encoding import DEFAULT_ENCODING
-
-def no_code(x, encoding=None):
- return x
-
-def decode(s, encoding=None):
- encoding = encoding or DEFAULT_ENCODING
- return s.decode(encoding, "replace")
-
-def encode(u, encoding=None):
- encoding = encoding or DEFAULT_ENCODING
- return u.encode(encoding, "replace")
-
-
-def cast_unicode(s, encoding=None):
- if isinstance(s, bytes):
- return decode(s, encoding)
- return s
-
-def cast_bytes(s, encoding=None):
- if not isinstance(s, bytes):
- return encode(s, encoding)
- return s
-
-def buffer_to_bytes(buf):
- """Cast a buffer object to bytes"""
- if not isinstance(buf, bytes):
- buf = bytes(buf)
- return buf
-
-def _modify_str_or_docstring(str_change_func):
- @functools.wraps(str_change_func)
- def wrapper(func_or_str):
- if isinstance(func_or_str, string_types):
- func = None
- doc = func_or_str
- else:
- func = func_or_str
- doc = func.__doc__
-
- # PYTHONOPTIMIZE=2 strips docstrings, so they can disappear unexpectedly
- if doc is not None:
- doc = str_change_func(doc)
-
- if func:
- func.__doc__ = doc
- return func
- return doc
- return wrapper
-
-def safe_unicode(e):
- """unicode(e) with various fallbacks. Used for exceptions, which may not be
- safe to call unicode() on.
- """
- try:
- return unicode_type(e)
- except UnicodeError:
- pass
-
- try:
- return str_to_unicode(str(e))
- except UnicodeError:
- pass
-
- try:
- return str_to_unicode(repr(e))
- except UnicodeError:
- pass
-
- return u'Unrecoverably corrupt evalue'
-
-# shutil.which from Python 3.4
-def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
- """Given a command, mode, and a PATH string, return the path which
- conforms to the given mode on the PATH, or None if there is no such
- file.
-
- `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
- of os.environ.get("PATH"), or can be overridden with a custom search
- path.
-
- This is a backport of shutil.which from Python 3.4
- """
- # Check that a given file can be accessed with the correct mode.
- # Additionally check that `file` is not a directory, as on Windows
- # directories pass the os.access check.
- def _access_check(fn, mode):
- return (os.path.exists(fn) and os.access(fn, mode)
- and not os.path.isdir(fn))
-
- # If we're given a path with a directory part, look it up directly rather
- # than referring to PATH directories. This includes checking relative to the
- # current directory, e.g. ./script
- if os.path.dirname(cmd):
- if _access_check(cmd, mode):
- return cmd
- return None
-
- if path is None:
- path = os.environ.get("PATH", os.defpath)
- if not path:
- return None
- path = path.split(os.pathsep)
-
- if sys.platform == "win32":
- # The current directory takes precedence on Windows.
- if not os.curdir in path:
- path.insert(0, os.curdir)
-
- # PATHEXT is necessary to check on Windows.
- pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
- # See if the given file matches any of the expected path extensions.
- # This will allow us to short circuit when given "python.exe".
- # If it does match, only test that one, otherwise we have to try
- # others.
- if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
- files = [cmd]
- else:
- files = [cmd + ext for ext in pathext]
- else:
- # On other platforms you don't have things like PATHEXT to tell you
- # what file suffixes are executable, so just pass on cmd as-is.
- files = [cmd]
-
- seen = set()
- for dir in path:
- normdir = os.path.normcase(dir)
- if not normdir in seen:
- seen.add(normdir)
- for thefile in files:
- name = os.path.join(dir, thefile)
- if _access_check(name, mode):
- return name
- return None
-
-if sys.version_info[0] >= 3:
- PY3 = True
-
- # keep reference to builtin_mod because the kernel overrides that value
- # to forward requests to a frontend.
- def input(prompt=''):
- return builtin_mod.input(prompt)
-
- builtin_mod_name = "builtins"
- import builtins as builtin_mod
-
- str_to_unicode = no_code
- unicode_to_str = no_code
- str_to_bytes = encode
- bytes_to_str = decode
- cast_bytes_py2 = no_code
- cast_unicode_py2 = no_code
- buffer_to_bytes_py2 = no_code
-
- string_types = (str,)
- unicode_type = str
-
- which = shutil.which
-
- def isidentifier(s, dotted=False):
- if dotted:
- return all(isidentifier(a) for a in s.split("."))
- return s.isidentifier()
-
- xrange = range
- def iteritems(d): return iter(d.items())
- def itervalues(d): return iter(d.values())
- getcwd = os.getcwd
-
- MethodType = types.MethodType
-
- def execfile(fname, glob, loc=None, compiler=None):
- loc = loc if (loc is not None) else glob
- with open(fname, 'rb') as f:
- compiler = compiler or compile
- exec(compiler(f.read(), fname, 'exec'), glob, loc)
-
- # Refactor print statements in doctests.
- _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
- def _print_statement_sub(match):
- expr = match.groups('expr')
- return "print(%s)" % expr
-
- @_modify_str_or_docstring
- def doctest_refactor_print(doc):
- """Refactor 'print x' statements in a doctest to print(x) style. 2to3
- unfortunately doesn't pick up on our doctests.
-
- Can accept a string or a function, so it can be used as a decorator."""
- return _print_statement_re.sub(_print_statement_sub, doc)
-
- # Abstract u'abc' syntax:
- @_modify_str_or_docstring
- def u_format(s):
- """"{u}'abc'" --> "'abc'" (Python 3)
-
- Accepts a string or a function, so it can be used as a decorator."""
- return s.format(u='')
-
- def get_closure(f):
- """Get a function's closure attribute"""
- return f.__closure__
-
-else:
- PY3 = False
-
- # keep reference to builtin_mod because the kernel overrides that value
- # to forward requests to a frontend.
- def input(prompt=''):
- return builtin_mod.raw_input(prompt)
-
- builtin_mod_name = "__builtin__"
- import __builtin__ as builtin_mod
-
- str_to_unicode = decode
- unicode_to_str = encode
- str_to_bytes = no_code
- bytes_to_str = no_code
- cast_bytes_py2 = cast_bytes
- cast_unicode_py2 = cast_unicode
- buffer_to_bytes_py2 = buffer_to_bytes
-
- string_types = (str, unicode)
- unicode_type = unicode
-
- import re
- _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
- def isidentifier(s, dotted=False):
- if dotted:
- return all(isidentifier(a) for a in s.split("."))
- return bool(_name_re.match(s))
-
- xrange = xrange
- def iteritems(d): return d.iteritems()
- def itervalues(d): return d.itervalues()
- getcwd = os.getcwdu
-
- def MethodType(func, instance):
- return types.MethodType(func, instance, type(instance))
-
- def doctest_refactor_print(func_or_str):
- return func_or_str
-
- def get_closure(f):
- """Get a function's closure attribute"""
- return f.func_closure
-
- which = _shutil_which
-
- # Abstract u'abc' syntax:
- @_modify_str_or_docstring
- def u_format(s):
- """"{u}'abc'" --> "u'abc'" (Python 2)
-
- Accepts a string or a function, so it can be used as a decorator."""
- return s.format(u='u')
-
- if sys.platform == 'win32':
- def execfile(fname, glob=None, loc=None, compiler=None):
- loc = loc if (loc is not None) else glob
- scripttext = builtin_mod.open(fname).read()+ '\n'
- # compile converts unicode filename to str assuming
- # ascii. Let's do the conversion before calling compile
- if isinstance(fname, unicode):
- filename = unicode_to_str(fname)
- else:
- filename = fname
- compiler = compiler or compile
- exec(compiler(scripttext, filename, 'exec'), glob, loc)
-
- else:
- def execfile(fname, glob=None, loc=None, compiler=None):
- if isinstance(fname, unicode):
- filename = fname.encode(sys.getfilesystemencoding())
- else:
- filename = fname
- where = [ns for ns in [glob, loc] if ns is not None]
- if compiler is None:
- builtin_mod.execfile(filename, *where)
- else:
- scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
- exec(compiler(scripttext, filename, 'exec'), glob, loc)
-
-
-PY2 = not PY3
+
+from .encoding import DEFAULT_ENCODING
+
+def no_code(x, encoding=None):
+ return x
+
+def decode(s, encoding=None):
+ encoding = encoding or DEFAULT_ENCODING
+ return s.decode(encoding, "replace")
+
+def encode(u, encoding=None):
+ encoding = encoding or DEFAULT_ENCODING
+ return u.encode(encoding, "replace")
+
+
+def cast_unicode(s, encoding=None):
+ if isinstance(s, bytes):
+ return decode(s, encoding)
+ return s
+
+def cast_bytes(s, encoding=None):
+ if not isinstance(s, bytes):
+ return encode(s, encoding)
+ return s
+
+def buffer_to_bytes(buf):
+ """Cast a buffer object to bytes"""
+ if not isinstance(buf, bytes):
+ buf = bytes(buf)
+ return buf
+
+def _modify_str_or_docstring(str_change_func):
+ @functools.wraps(str_change_func)
+ def wrapper(func_or_str):
+ if isinstance(func_or_str, string_types):
+ func = None
+ doc = func_or_str
+ else:
+ func = func_or_str
+ doc = func.__doc__
+
+ # PYTHONOPTIMIZE=2 strips docstrings, so they can disappear unexpectedly
+ if doc is not None:
+ doc = str_change_func(doc)
+
+ if func:
+ func.__doc__ = doc
+ return func
+ return doc
+ return wrapper
+
+def safe_unicode(e):
+ """unicode(e) with various fallbacks. Used for exceptions, which may not be
+ safe to call unicode() on.
+ """
+ try:
+ return unicode_type(e)
+ except UnicodeError:
+ pass
+
+ try:
+ return str_to_unicode(str(e))
+ except UnicodeError:
+ pass
+
+ try:
+ return str_to_unicode(repr(e))
+ except UnicodeError:
+ pass
+
+ return u'Unrecoverably corrupt evalue'
+
+# shutil.which from Python 3.4
+def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
+ """Given a command, mode, and a PATH string, return the path which
+ conforms to the given mode on the PATH, or None if there is no such
+ file.
+
+ `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+ of os.environ.get("PATH"), or can be overridden with a custom search
+ path.
+
+ This is a backport of shutil.which from Python 3.4
+ """
+ # Check that a given file can be accessed with the correct mode.
+ # Additionally check that `file` is not a directory, as on Windows
+ # directories pass the os.access check.
+ def _access_check(fn, mode):
+ return (os.path.exists(fn) and os.access(fn, mode)
+ and not os.path.isdir(fn))
+
+ # If we're given a path with a directory part, look it up directly rather
+ # than referring to PATH directories. This includes checking relative to the
+ # current directory, e.g. ./script
+ if os.path.dirname(cmd):
+ if _access_check(cmd, mode):
+ return cmd
+ return None
+
+ if path is None:
+ path = os.environ.get("PATH", os.defpath)
+ if not path:
+ return None
+ path = path.split(os.pathsep)
+
+ if sys.platform == "win32":
+ # The current directory takes precedence on Windows.
+ if not os.curdir in path:
+ path.insert(0, os.curdir)
+
+ # PATHEXT is necessary to check on Windows.
+ pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+ # See if the given file matches any of the expected path extensions.
+ # This will allow us to short circuit when given "python.exe".
+ # If it does match, only test that one, otherwise we have to try
+ # others.
+ if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+ files = [cmd]
+ else:
+ files = [cmd + ext for ext in pathext]
+ else:
+ # On other platforms you don't have things like PATHEXT to tell you
+ # what file suffixes are executable, so just pass on cmd as-is.
+ files = [cmd]
+
+ seen = set()
+ for dir in path:
+ normdir = os.path.normcase(dir)
+ if not normdir in seen:
+ seen.add(normdir)
+ for thefile in files:
+ name = os.path.join(dir, thefile)
+ if _access_check(name, mode):
+ return name
+ return None
+
+if sys.version_info[0] >= 3:
+ PY3 = True
+
+ # keep reference to builtin_mod because the kernel overrides that value
+ # to forward requests to a frontend.
+ def input(prompt=''):
+ return builtin_mod.input(prompt)
+
+ builtin_mod_name = "builtins"
+ import builtins as builtin_mod
+
+ str_to_unicode = no_code
+ unicode_to_str = no_code
+ str_to_bytes = encode
+ bytes_to_str = decode
+ cast_bytes_py2 = no_code
+ cast_unicode_py2 = no_code
+ buffer_to_bytes_py2 = no_code
+
+ string_types = (str,)
+ unicode_type = str
+
+ which = shutil.which
+
+ def isidentifier(s, dotted=False):
+ if dotted:
+ return all(isidentifier(a) for a in s.split("."))
+ return s.isidentifier()
+
+ xrange = range
+ def iteritems(d): return iter(d.items())
+ def itervalues(d): return iter(d.values())
+ getcwd = os.getcwd
+
+ MethodType = types.MethodType
+
+ def execfile(fname, glob, loc=None, compiler=None):
+ loc = loc if (loc is not None) else glob
+ with open(fname, 'rb') as f:
+ compiler = compiler or compile
+ exec(compiler(f.read(), fname, 'exec'), glob, loc)
+
+ # Refactor print statements in doctests.
+ _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
+ def _print_statement_sub(match):
+ expr = match.groups('expr')
+ return "print(%s)" % expr
+
+ @_modify_str_or_docstring
+ def doctest_refactor_print(doc):
+ """Refactor 'print x' statements in a doctest to print(x) style. 2to3
+ unfortunately doesn't pick up on our doctests.
+
+ Can accept a string or a function, so it can be used as a decorator."""
+ return _print_statement_re.sub(_print_statement_sub, doc)
+
+ # Abstract u'abc' syntax:
+ @_modify_str_or_docstring
+ def u_format(s):
+ """"{u}'abc'" --> "'abc'" (Python 3)
+
+ Accepts a string or a function, so it can be used as a decorator."""
+ return s.format(u='')
+
+ def get_closure(f):
+ """Get a function's closure attribute"""
+ return f.__closure__
+
+else:
+ PY3 = False
+
+ # keep reference to builtin_mod because the kernel overrides that value
+ # to forward requests to a frontend.
+ def input(prompt=''):
+ return builtin_mod.raw_input(prompt)
+
+ builtin_mod_name = "__builtin__"
+ import __builtin__ as builtin_mod
+
+ str_to_unicode = decode
+ unicode_to_str = encode
+ str_to_bytes = no_code
+ bytes_to_str = no_code
+ cast_bytes_py2 = cast_bytes
+ cast_unicode_py2 = cast_unicode
+ buffer_to_bytes_py2 = buffer_to_bytes
+
+ string_types = (str, unicode)
+ unicode_type = unicode
+
+ import re
+ _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
+ def isidentifier(s, dotted=False):
+ if dotted:
+ return all(isidentifier(a) for a in s.split("."))
+ return bool(_name_re.match(s))
+
+ xrange = xrange
+ def iteritems(d): return d.iteritems()
+ def itervalues(d): return d.itervalues()
+ getcwd = os.getcwdu
+
+ def MethodType(func, instance):
+ return types.MethodType(func, instance, type(instance))
+
+ def doctest_refactor_print(func_or_str):
+ return func_or_str
+
+ def get_closure(f):
+ """Get a function's closure attribute"""
+ return f.func_closure
+
+ which = _shutil_which
+
+ # Abstract u'abc' syntax:
+ @_modify_str_or_docstring
+ def u_format(s):
+ """"{u}'abc'" --> "u'abc'" (Python 2)
+
+ Accepts a string or a function, so it can be used as a decorator."""
+ return s.format(u='u')
+
+ if sys.platform == 'win32':
+ def execfile(fname, glob=None, loc=None, compiler=None):
+ loc = loc if (loc is not None) else glob
+ scripttext = builtin_mod.open(fname).read()+ '\n'
+ # compile converts unicode filename to str assuming
+ # ascii. Let's do the conversion before calling compile
+ if isinstance(fname, unicode):
+ filename = unicode_to_str(fname)
+ else:
+ filename = fname
+ compiler = compiler or compile
+ exec(compiler(scripttext, filename, 'exec'), glob, loc)
+
+ else:
+ def execfile(fname, glob=None, loc=None, compiler=None):
+ if isinstance(fname, unicode):
+ filename = fname.encode(sys.getfilesystemencoding())
+ else:
+ filename = fname
+ where = [ns for ns in [glob, loc] if ns is not None]
+ if compiler is None:
+ builtin_mod.execfile(filename, *where)
+ else:
+ scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
+ exec(compiler(scripttext, filename, 'exec'), glob, loc)
+
+
+PY2 = not PY3
PYPY = platform.python_implementation() == "PyPy"
-
-
-def annotate(**kwargs):
- """Python 3 compatible function annotation for Python 2."""
- if not kwargs:
- raise ValueError('annotations must be provided as keyword arguments')
- def dec(f):
- if hasattr(f, '__annotations__'):
- for k, v in kwargs.items():
- f.__annotations__[k] = v
- else:
- f.__annotations__ = kwargs
- return f
- return dec
-
-
-# Parts below taken from six:
-# Copyright (c) 2010-2013 Benjamin Peterson
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-def with_metaclass(meta, *bases):
- """Create a base class with a metaclass."""
- return meta("_NewBase", bases, {})
+
+
+def annotate(**kwargs):
+ """Python 3 compatible function annotation for Python 2."""
+ if not kwargs:
+ raise ValueError('annotations must be provided as keyword arguments')
+ def dec(f):
+ if hasattr(f, '__annotations__'):
+ for k, v in kwargs.items():
+ f.__annotations__[k] = v
+ else:
+ f.__annotations__ = kwargs
+ return f
+ return dec
+
+
+# Parts below taken from six:
+# Copyright (c) 2010-2013 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+def with_metaclass(meta, *bases):
+ """Create a base class with a metaclass."""
+ return meta("_NewBase", bases, {})
diff --git a/contrib/python/ipython/py2/IPython/utils/rlineimpl.py b/contrib/python/ipython/py2/IPython/utils/rlineimpl.py
index e1cf03942c..f8c4e84334 100644
--- a/contrib/python/ipython/py2/IPython/utils/rlineimpl.py
+++ b/contrib/python/ipython/py2/IPython/utils/rlineimpl.py
@@ -1,74 +1,74 @@
-# -*- coding: utf-8 -*-
-""" Imports and provides the 'correct' version of readline for the platform.
-
-Readline is used throughout IPython as::
-
- import IPython.utils.rlineimpl as readline
-
-In addition to normal readline stuff, this module provides have_readline
-boolean and _outputfile variable used in IPython.utils.
-"""
-
-import sys
-import warnings
-
-_rlmod_names = ['gnureadline', 'readline']
-
-have_readline = False
-for _rlmod_name in _rlmod_names:
- try:
- # import readline as _rl
- _rl = __import__(_rlmod_name)
- # from readline import *
- globals().update({k:v for k,v in _rl.__dict__.items() if not k.startswith('_')})
- except ImportError:
- pass
- else:
- have_readline = True
- break
-
-if have_readline and (sys.platform == 'win32' or sys.platform == 'cli'):
- try:
- _outputfile=_rl.GetOutputFile()
- except AttributeError:
- warnings.warn("Failed GetOutputFile")
- have_readline = False
-
-# Test to see if libedit is being used instead of GNU readline.
-# Thanks to Boyd Waters for the original patch.
-uses_libedit = False
-
-if have_readline:
- # Official Python docs state that 'libedit' is in the docstring for libedit readline:
- uses_libedit = _rl.__doc__ and 'libedit' in _rl.__doc__
- # Note that many non-System Pythons also do not use proper readline,
- # but do not report libedit at all, nor are they linked dynamically against libedit.
- # known culprits of this include: EPD, Fink
- # There is not much we can do to detect this, until we find a specific failure
- # case, rather than relying on the readline module to self-identify as broken.
-
-if uses_libedit and sys.platform == 'darwin':
- _rl.parse_and_bind("bind ^I rl_complete")
- warnings.warn('\n'.join(['', "*"*78,
- "libedit detected - readline will not be well behaved, including but not limited to:",
- " * crashes on tab completion",
- " * incorrect history navigation",
- " * corrupting long-lines",
- " * failure to wrap or indent lines properly",
- "It is highly recommended that you install gnureadline, which is installable with:",
- " pip install gnureadline",
- "*"*78]),
- RuntimeWarning)
-
-# the clear_history() function was only introduced in Python 2.4 and is
-# actually optional in the readline API, so we must explicitly check for its
-# existence. Some known platforms actually don't have it. This thread:
-# http://mail.python.org/pipermail/python-dev/2003-August/037845.html
-# has the original discussion.
-
-if have_readline:
- try:
- _rl.clear_history
- except AttributeError:
- def clear_history(): pass
- _rl.clear_history = clear_history
+# -*- coding: utf-8 -*-
+""" Imports and provides the 'correct' version of readline for the platform.
+
+Readline is used throughout IPython as::
+
+ import IPython.utils.rlineimpl as readline
+
+In addition to normal readline stuff, this module provides have_readline
+boolean and _outputfile variable used in IPython.utils.
+"""
+
+import sys
+import warnings
+
+_rlmod_names = ['gnureadline', 'readline']
+
+have_readline = False
+for _rlmod_name in _rlmod_names:
+ try:
+ # import readline as _rl
+ _rl = __import__(_rlmod_name)
+ # from readline import *
+ globals().update({k:v for k,v in _rl.__dict__.items() if not k.startswith('_')})
+ except ImportError:
+ pass
+ else:
+ have_readline = True
+ break
+
+if have_readline and (sys.platform == 'win32' or sys.platform == 'cli'):
+ try:
+ _outputfile=_rl.GetOutputFile()
+ except AttributeError:
+ warnings.warn("Failed GetOutputFile")
+ have_readline = False
+
+# Test to see if libedit is being used instead of GNU readline.
+# Thanks to Boyd Waters for the original patch.
+uses_libedit = False
+
+if have_readline:
+ # Official Python docs state that 'libedit' is in the docstring for libedit readline:
+ uses_libedit = _rl.__doc__ and 'libedit' in _rl.__doc__
+ # Note that many non-System Pythons also do not use proper readline,
+ # but do not report libedit at all, nor are they linked dynamically against libedit.
+ # known culprits of this include: EPD, Fink
+ # There is not much we can do to detect this, until we find a specific failure
+ # case, rather than relying on the readline module to self-identify as broken.
+
+if uses_libedit and sys.platform == 'darwin':
+ _rl.parse_and_bind("bind ^I rl_complete")
+ warnings.warn('\n'.join(['', "*"*78,
+ "libedit detected - readline will not be well behaved, including but not limited to:",
+ " * crashes on tab completion",
+ " * incorrect history navigation",
+ " * corrupting long-lines",
+ " * failure to wrap or indent lines properly",
+ "It is highly recommended that you install gnureadline, which is installable with:",
+ " pip install gnureadline",
+ "*"*78]),
+ RuntimeWarning)
+
+# the clear_history() function was only introduced in Python 2.4 and is
+# actually optional in the readline API, so we must explicitly check for its
+# existence. Some known platforms actually don't have it. This thread:
+# http://mail.python.org/pipermail/python-dev/2003-August/037845.html
+# has the original discussion.
+
+if have_readline:
+ try:
+ _rl.clear_history
+ except AttributeError:
+ def clear_history(): pass
+ _rl.clear_history = clear_history
diff --git a/contrib/python/ipython/py2/IPython/utils/sentinel.py b/contrib/python/ipython/py2/IPython/utils/sentinel.py
index dc57a2591c..7af2558c1a 100644
--- a/contrib/python/ipython/py2/IPython/utils/sentinel.py
+++ b/contrib/python/ipython/py2/IPython/utils/sentinel.py
@@ -1,17 +1,17 @@
-"""Sentinel class for constants with useful reprs"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-class Sentinel(object):
-
- def __init__(self, name, module, docstring=None):
- self.name = name
- self.module = module
- if docstring:
- self.__doc__ = docstring
-
-
- def __repr__(self):
- return str(self.module)+'.'+self.name
-
+"""Sentinel class for constants with useful reprs"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+class Sentinel(object):
+
+ def __init__(self, name, module, docstring=None):
+ self.name = name
+ self.module = module
+ if docstring:
+ self.__doc__ = docstring
+
+
+ def __repr__(self):
+ return str(self.module)+'.'+self.name
+
diff --git a/contrib/python/ipython/py2/IPython/utils/shimmodule.py b/contrib/python/ipython/py2/IPython/utils/shimmodule.py
index 8b74f5011a..c2cf6c6de7 100644
--- a/contrib/python/ipython/py2/IPython/utils/shimmodule.py
+++ b/contrib/python/ipython/py2/IPython/utils/shimmodule.py
@@ -1,92 +1,92 @@
-"""A shim module for deprecated imports
-"""
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-import sys
-import types
-
-from .importstring import import_item
-
-class ShimWarning(Warning):
- """A warning to show when a module has moved, and a shim is in its place."""
-
-class ShimImporter(object):
- """Import hook for a shim.
-
- This ensures that submodule imports return the real target module,
- not a clone that will confuse `is` and `isinstance` checks.
- """
- def __init__(self, src, mirror):
- self.src = src
- self.mirror = mirror
-
- def _mirror_name(self, fullname):
- """get the name of the mirrored module"""
-
- return self.mirror + fullname[len(self.src):]
-
- def find_module(self, fullname, path=None):
- """Return self if we should be used to import the module."""
- if fullname.startswith(self.src + '.'):
- mirror_name = self._mirror_name(fullname)
- try:
- mod = import_item(mirror_name)
- except ImportError:
- return
- else:
- if not isinstance(mod, types.ModuleType):
- # not a module
- return None
- return self
-
- def load_module(self, fullname):
- """Import the mirrored module, and insert it into sys.modules"""
- mirror_name = self._mirror_name(fullname)
- mod = import_item(mirror_name)
- sys.modules[fullname] = mod
- return mod
-
-
-class ShimModule(types.ModuleType):
-
- def __init__(self, *args, **kwargs):
- self._mirror = kwargs.pop("mirror")
- src = kwargs.pop("src", None)
- if src:
- kwargs['name'] = src.rsplit('.', 1)[-1]
- super(ShimModule, self).__init__(*args, **kwargs)
- # add import hook for descendent modules
- if src:
- sys.meta_path.append(
- ShimImporter(src=src, mirror=self._mirror)
- )
-
- @property
- def __path__(self):
- return []
-
- @property
- def __spec__(self):
- """Don't produce __spec__ until requested"""
- return __import__(self._mirror).__spec__
-
- def __dir__(self):
- return dir(__import__(self._mirror))
-
- @property
- def __all__(self):
- """Ensure __all__ is always defined"""
- mod = __import__(self._mirror)
- try:
- return mod.__all__
- except AttributeError:
- return [name for name in dir(mod) if not name.startswith('_')]
-
- def __getattr__(self, key):
- # Use the equivalent of import_item(name), see below
- name = "%s.%s" % (self._mirror, key)
- try:
- return import_item(name)
- except ImportError:
- raise AttributeError(key)
+"""A shim module for deprecated imports
+"""
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import sys
+import types
+
+from .importstring import import_item
+
+class ShimWarning(Warning):
+ """A warning to show when a module has moved, and a shim is in its place."""
+
+class ShimImporter(object):
+ """Import hook for a shim.
+
+ This ensures that submodule imports return the real target module,
+ not a clone that will confuse `is` and `isinstance` checks.
+ """
+ def __init__(self, src, mirror):
+ self.src = src
+ self.mirror = mirror
+
+ def _mirror_name(self, fullname):
+ """get the name of the mirrored module"""
+
+ return self.mirror + fullname[len(self.src):]
+
+ def find_module(self, fullname, path=None):
+ """Return self if we should be used to import the module."""
+ if fullname.startswith(self.src + '.'):
+ mirror_name = self._mirror_name(fullname)
+ try:
+ mod = import_item(mirror_name)
+ except ImportError:
+ return
+ else:
+ if not isinstance(mod, types.ModuleType):
+ # not a module
+ return None
+ return self
+
+ def load_module(self, fullname):
+ """Import the mirrored module, and insert it into sys.modules"""
+ mirror_name = self._mirror_name(fullname)
+ mod = import_item(mirror_name)
+ sys.modules[fullname] = mod
+ return mod
+
+
+class ShimModule(types.ModuleType):
+
+ def __init__(self, *args, **kwargs):
+ self._mirror = kwargs.pop("mirror")
+ src = kwargs.pop("src", None)
+ if src:
+ kwargs['name'] = src.rsplit('.', 1)[-1]
+ super(ShimModule, self).__init__(*args, **kwargs)
+ # add import hook for descendent modules
+ if src:
+ sys.meta_path.append(
+ ShimImporter(src=src, mirror=self._mirror)
+ )
+
+ @property
+ def __path__(self):
+ return []
+
+ @property
+ def __spec__(self):
+ """Don't produce __spec__ until requested"""
+ return __import__(self._mirror).__spec__
+
+ def __dir__(self):
+ return dir(__import__(self._mirror))
+
+ @property
+ def __all__(self):
+ """Ensure __all__ is always defined"""
+ mod = __import__(self._mirror)
+ try:
+ return mod.__all__
+ except AttributeError:
+ return [name for name in dir(mod) if not name.startswith('_')]
+
+ def __getattr__(self, key):
+ # Use the equivalent of import_item(name), see below
+ name = "%s.%s" % (self._mirror, key)
+ try:
+ return import_item(name)
+ except ImportError:
+ raise AttributeError(key)
diff --git a/contrib/python/ipython/py2/IPython/utils/signatures.py b/contrib/python/ipython/py2/IPython/utils/signatures.py
index dedc51cfda..4d0eb74a7e 100644
--- a/contrib/python/ipython/py2/IPython/utils/signatures.py
+++ b/contrib/python/ipython/py2/IPython/utils/signatures.py
@@ -1,11 +1,11 @@
-"""Function signature objects for callables.
-
-Use the standard library version if available, as it is more up to date.
-Fallback on backport otherwise.
-"""
-
-
-try:
- from inspect import BoundArguments, Parameter, Signature, signature
-except ImportError:
- from ._signatures import BoundArguments, Parameter, Signature, signature
+"""Function signature objects for callables.
+
+Use the standard library version if available, as it is more up to date.
+Fallback on backport otherwise.
+"""
+
+
+try:
+ from inspect import BoundArguments, Parameter, Signature, signature
+except ImportError:
+ from ._signatures import BoundArguments, Parameter, Signature, signature
diff --git a/contrib/python/ipython/py2/IPython/utils/strdispatch.py b/contrib/python/ipython/py2/IPython/utils/strdispatch.py
index d6bf510535..a6183404e7 100644
--- a/contrib/python/ipython/py2/IPython/utils/strdispatch.py
+++ b/contrib/python/ipython/py2/IPython/utils/strdispatch.py
@@ -1,68 +1,68 @@
-"""String dispatch class to match regexps and dispatch commands.
-"""
-
-# Stdlib imports
-import re
-
-# Our own modules
-from IPython.core.hooks import CommandChainDispatcher
-
-# Code begins
-class StrDispatch(object):
- """Dispatch (lookup) a set of strings / regexps for match.
-
- Example:
-
- >>> dis = StrDispatch()
- >>> dis.add_s('hei',34, priority = 4)
- >>> dis.add_s('hei',123, priority = 2)
- >>> dis.add_re('h.i', 686)
- >>> print(list(dis.flat_matches('hei')))
- [123, 34, 686]
- """
-
- def __init__(self):
- self.strs = {}
- self.regexs = {}
-
- def add_s(self, s, obj, priority= 0 ):
- """ Adds a target 'string' for dispatching """
-
- chain = self.strs.get(s, CommandChainDispatcher())
- chain.add(obj,priority)
- self.strs[s] = chain
-
- def add_re(self, regex, obj, priority= 0 ):
- """ Adds a target regexp for dispatching """
-
- chain = self.regexs.get(regex, CommandChainDispatcher())
- chain.add(obj,priority)
- self.regexs[regex] = chain
-
- def dispatch(self, key):
- """ Get a seq of Commandchain objects that match key """
- if key in self.strs:
- yield self.strs[key]
-
- for r, obj in self.regexs.items():
- if re.match(r, key):
- yield obj
- else:
- #print "nomatch",key # dbg
- pass
-
- def __repr__(self):
- return "<Strdispatch %s, %s>" % (self.strs, self.regexs)
-
- def s_matches(self, key):
- if key not in self.strs:
- return
- for el in self.strs[key]:
- yield el[1]
-
- def flat_matches(self, key):
- """ Yield all 'value' targets, without priority """
- for val in self.dispatch(key):
- for el in val:
- yield el[1] # only value, no priority
- return
+"""String dispatch class to match regexps and dispatch commands.
+"""
+
+# Stdlib imports
+import re
+
+# Our own modules
+from IPython.core.hooks import CommandChainDispatcher
+
+# Code begins
+class StrDispatch(object):
+ """Dispatch (lookup) a set of strings / regexps for match.
+
+ Example:
+
+ >>> dis = StrDispatch()
+ >>> dis.add_s('hei',34, priority = 4)
+ >>> dis.add_s('hei',123, priority = 2)
+ >>> dis.add_re('h.i', 686)
+ >>> print(list(dis.flat_matches('hei')))
+ [123, 34, 686]
+ """
+
+ def __init__(self):
+ self.strs = {}
+ self.regexs = {}
+
+ def add_s(self, s, obj, priority= 0 ):
+ """ Adds a target 'string' for dispatching """
+
+ chain = self.strs.get(s, CommandChainDispatcher())
+ chain.add(obj,priority)
+ self.strs[s] = chain
+
+ def add_re(self, regex, obj, priority= 0 ):
+ """ Adds a target regexp for dispatching """
+
+ chain = self.regexs.get(regex, CommandChainDispatcher())
+ chain.add(obj,priority)
+ self.regexs[regex] = chain
+
+ def dispatch(self, key):
+ """ Get a seq of Commandchain objects that match key """
+ if key in self.strs:
+ yield self.strs[key]
+
+ for r, obj in self.regexs.items():
+ if re.match(r, key):
+ yield obj
+ else:
+ #print "nomatch",key # dbg
+ pass
+
+ def __repr__(self):
+ return "<Strdispatch %s, %s>" % (self.strs, self.regexs)
+
+ def s_matches(self, key):
+ if key not in self.strs:
+ return
+ for el in self.strs[key]:
+ yield el[1]
+
+ def flat_matches(self, key):
+ """ Yield all 'value' targets, without priority """
+ for val in self.dispatch(key):
+ for el in val:
+ yield el[1] # only value, no priority
+ return
diff --git a/contrib/python/ipython/py2/IPython/utils/sysinfo.py b/contrib/python/ipython/py2/IPython/utils/sysinfo.py
index db7f2914d4..51ca68d9cf 100644
--- a/contrib/python/ipython/py2/IPython/utils/sysinfo.py
+++ b/contrib/python/ipython/py2/IPython/utils/sysinfo.py
@@ -1,167 +1,167 @@
-# encoding: utf-8
-"""
-Utilities for getting information about IPython and the system it's running in.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import os
-import platform
-import pprint
-import sys
-import subprocess
-
-from IPython.core import release
-from IPython.utils import py3compat, _sysinfo, encoding
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-def pkg_commit_hash(pkg_path):
- """Get short form of commit hash given directory `pkg_path`
-
- We get the commit hash from (in order of preference):
-
- * IPython.utils._sysinfo.commit
- * git output, if we are in a git repository
-
- If these fail, we return a not-found placeholder tuple
-
- Parameters
- ----------
- pkg_path : str
- directory containing package
- only used for getting commit from active repo
-
- Returns
- -------
- hash_from : str
- Where we got the hash from - description
- hash_str : str
- short form of hash
- """
- # Try and get commit from written commit text file
- if _sysinfo.commit:
- return "installation", _sysinfo.commit
-
- # maybe we are in a repository
- proc = subprocess.Popen('git rev-parse --short HEAD',
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=pkg_path, shell=True)
- repo_commit, _ = proc.communicate()
- if repo_commit:
- return 'repository', repo_commit.strip().decode('ascii')
- return '(none found)', u'<not found>'
-
-
-def pkg_info(pkg_path):
- """Return dict describing the context of this package
-
- Parameters
- ----------
- pkg_path : str
- path containing __init__.py for package
-
- Returns
- -------
- context : dict
- with named parameters of interest
- """
- src, hsh = pkg_commit_hash(pkg_path)
- return dict(
- ipython_version=release.version,
- ipython_path=pkg_path,
- commit_source=src,
- commit_hash=hsh,
- sys_version=sys.version,
- sys_executable=sys.executable,
- sys_platform=sys.platform,
- platform=platform.platform(),
- os_name=os.name,
- default_encoding=encoding.DEFAULT_ENCODING,
- )
-
-def get_sys_info():
- """Return useful information about IPython and the system, as a dict."""
- p = os.path
- path = p.realpath(p.dirname(p.abspath(p.join(__file__, '..'))))
- return pkg_info(path)
-
-@py3compat.doctest_refactor_print
-def sys_info():
- """Return useful information about IPython and the system, as a string.
-
- Examples
- --------
- ::
-
- In [2]: print sys_info()
- {'commit_hash': '144fdae', # random
- 'commit_source': 'repository',
- 'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
- 'ipython_version': '0.11.dev',
- 'os_name': 'posix',
- 'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
- 'sys_executable': '/usr/bin/python',
- 'sys_platform': 'linux2',
- 'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
- """
- return pprint.pformat(get_sys_info())
-
-def _num_cpus_unix():
- """Return the number of active CPUs on a Unix system."""
- return os.sysconf("SC_NPROCESSORS_ONLN")
-
-
-def _num_cpus_darwin():
- """Return the number of active CPUs on a Darwin system."""
- p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
- return p.stdout.read()
-
-
-def _num_cpus_windows():
- """Return the number of active CPUs on a Windows system."""
- return os.environ.get("NUMBER_OF_PROCESSORS")
-
-
-def num_cpus():
- """Return the effective number of CPUs in the system as an integer.
-
- This cross-platform function makes an attempt at finding the total number of
- available CPUs in the system, as returned by various underlying system and
- python calls.
-
- If it can't find a sensible answer, it returns 1 (though an error *may* make
- it return a large positive number that's actually incorrect).
- """
-
- # Many thanks to the Parallel Python project (http://www.parallelpython.com)
- # for the names of the keys we needed to look up for this function. This
- # code was inspired by their equivalent function.
-
- ncpufuncs = {'Linux':_num_cpus_unix,
- 'Darwin':_num_cpus_darwin,
- 'Windows':_num_cpus_windows
- }
-
- ncpufunc = ncpufuncs.get(platform.system(),
- # default to unix version (Solaris, AIX, etc)
- _num_cpus_unix)
-
- try:
- ncpus = max(1,int(ncpufunc()))
- except:
- ncpus = 1
- return ncpus
-
+# encoding: utf-8
+"""
+Utilities for getting information about IPython and the system it's running in.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import platform
+import pprint
+import sys
+import subprocess
+
+from IPython.core import release
+from IPython.utils import py3compat, _sysinfo, encoding
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def pkg_commit_hash(pkg_path):
+ """Get short form of commit hash given directory `pkg_path`
+
+ We get the commit hash from (in order of preference):
+
+ * IPython.utils._sysinfo.commit
+ * git output, if we are in a git repository
+
+ If these fail, we return a not-found placeholder tuple
+
+ Parameters
+ ----------
+ pkg_path : str
+ directory containing package
+ only used for getting commit from active repo
+
+ Returns
+ -------
+ hash_from : str
+ Where we got the hash from - description
+ hash_str : str
+ short form of hash
+ """
+ # Try and get commit from written commit text file
+ if _sysinfo.commit:
+ return "installation", _sysinfo.commit
+
+ # maybe we are in a repository
+ proc = subprocess.Popen('git rev-parse --short HEAD',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=pkg_path, shell=True)
+ repo_commit, _ = proc.communicate()
+ if repo_commit:
+ return 'repository', repo_commit.strip().decode('ascii')
+ return '(none found)', u'<not found>'
+
+
+def pkg_info(pkg_path):
+ """Return dict describing the context of this package
+
+ Parameters
+ ----------
+ pkg_path : str
+ path containing __init__.py for package
+
+ Returns
+ -------
+ context : dict
+ with named parameters of interest
+ """
+ src, hsh = pkg_commit_hash(pkg_path)
+ return dict(
+ ipython_version=release.version,
+ ipython_path=pkg_path,
+ commit_source=src,
+ commit_hash=hsh,
+ sys_version=sys.version,
+ sys_executable=sys.executable,
+ sys_platform=sys.platform,
+ platform=platform.platform(),
+ os_name=os.name,
+ default_encoding=encoding.DEFAULT_ENCODING,
+ )
+
+def get_sys_info():
+ """Return useful information about IPython and the system, as a dict."""
+ p = os.path
+ path = p.realpath(p.dirname(p.abspath(p.join(__file__, '..'))))
+ return pkg_info(path)
+
+@py3compat.doctest_refactor_print
+def sys_info():
+ """Return useful information about IPython and the system, as a string.
+
+ Examples
+ --------
+ ::
+
+ In [2]: print sys_info()
+ {'commit_hash': '144fdae', # random
+ 'commit_source': 'repository',
+ 'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
+ 'ipython_version': '0.11.dev',
+ 'os_name': 'posix',
+ 'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
+ 'sys_executable': '/usr/bin/python',
+ 'sys_platform': 'linux2',
+ 'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
+ """
+ return pprint.pformat(get_sys_info())
+
+def _num_cpus_unix():
+ """Return the number of active CPUs on a Unix system."""
+ return os.sysconf("SC_NPROCESSORS_ONLN")
+
+
+def _num_cpus_darwin():
+ """Return the number of active CPUs on a Darwin system."""
+ p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
+ return p.stdout.read()
+
+
+def _num_cpus_windows():
+ """Return the number of active CPUs on a Windows system."""
+ return os.environ.get("NUMBER_OF_PROCESSORS")
+
+
+def num_cpus():
+ """Return the effective number of CPUs in the system as an integer.
+
+ This cross-platform function makes an attempt at finding the total number of
+ available CPUs in the system, as returned by various underlying system and
+ python calls.
+
+ If it can't find a sensible answer, it returns 1 (though an error *may* make
+ it return a large positive number that's actually incorrect).
+ """
+
+ # Many thanks to the Parallel Python project (http://www.parallelpython.com)
+ # for the names of the keys we needed to look up for this function. This
+ # code was inspired by their equivalent function.
+
+ ncpufuncs = {'Linux':_num_cpus_unix,
+ 'Darwin':_num_cpus_darwin,
+ 'Windows':_num_cpus_windows
+ }
+
+ ncpufunc = ncpufuncs.get(platform.system(),
+ # default to unix version (Solaris, AIX, etc)
+ _num_cpus_unix)
+
+ try:
+ ncpus = max(1,int(ncpufunc()))
+ except:
+ ncpus = 1
+ return ncpus
+
diff --git a/contrib/python/ipython/py2/IPython/utils/syspathcontext.py b/contrib/python/ipython/py2/IPython/utils/syspathcontext.py
index 89612038ff..fdcfbbee35 100644
--- a/contrib/python/ipython/py2/IPython/utils/syspathcontext.py
+++ b/contrib/python/ipython/py2/IPython/utils/syspathcontext.py
@@ -1,71 +1,71 @@
-# encoding: utf-8
-"""
-Context managers for adding things to sys.path temporarily.
-
-Authors:
-
-* Brian Granger
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import sys
-
-from IPython.utils.py3compat import cast_bytes_py2
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-class appended_to_syspath(object):
- """A context for appending a directory to sys.path for a second."""
-
- def __init__(self, dir):
- self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
-
- def __enter__(self):
- if self.dir not in sys.path:
- sys.path.append(self.dir)
- self.added = True
- else:
- self.added = False
-
- def __exit__(self, type, value, traceback):
- if self.added:
- try:
- sys.path.remove(self.dir)
- except ValueError:
- pass
- # Returning False causes any exceptions to be re-raised.
- return False
-
-class prepended_to_syspath(object):
- """A context for prepending a directory to sys.path for a second."""
-
- def __init__(self, dir):
- self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
-
- def __enter__(self):
- if self.dir not in sys.path:
- sys.path.insert(0,self.dir)
- self.added = True
- else:
- self.added = False
-
- def __exit__(self, type, value, traceback):
- if self.added:
- try:
- sys.path.remove(self.dir)
- except ValueError:
- pass
- # Returning False causes any exceptions to be re-raised.
- return False
+# encoding: utf-8
+"""
+Context managers for adding things to sys.path temporarily.
+
+Authors:
+
+* Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+from IPython.utils.py3compat import cast_bytes_py2
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+class appended_to_syspath(object):
+ """A context for appending a directory to sys.path for a second."""
+
+ def __init__(self, dir):
+ self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
+
+ def __enter__(self):
+ if self.dir not in sys.path:
+ sys.path.append(self.dir)
+ self.added = True
+ else:
+ self.added = False
+
+ def __exit__(self, type, value, traceback):
+ if self.added:
+ try:
+ sys.path.remove(self.dir)
+ except ValueError:
+ pass
+ # Returning False causes any exceptions to be re-raised.
+ return False
+
+class prepended_to_syspath(object):
+ """A context for prepending a directory to sys.path for a second."""
+
+ def __init__(self, dir):
+ self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
+
+ def __enter__(self):
+ if self.dir not in sys.path:
+ sys.path.insert(0,self.dir)
+ self.added = True
+ else:
+ self.added = False
+
+ def __exit__(self, type, value, traceback):
+ if self.added:
+ try:
+ sys.path.remove(self.dir)
+ except ValueError:
+ pass
+ # Returning False causes any exceptions to be re-raised.
+ return False
diff --git a/contrib/python/ipython/py2/IPython/utils/tempdir.py b/contrib/python/ipython/py2/IPython/utils/tempdir.py
index 951abd65c9..909205e192 100644
--- a/contrib/python/ipython/py2/IPython/utils/tempdir.py
+++ b/contrib/python/ipython/py2/IPython/utils/tempdir.py
@@ -1,145 +1,145 @@
-"""TemporaryDirectory class, copied from Python 3.2.
-
-This is copied from the stdlib and will be standard in Python 3.2 and onwards.
-"""
-from __future__ import print_function
-
-import os as _os
-import warnings as _warnings
-import sys as _sys
-
-# This code should only be used in Python versions < 3.2, since after that we
-# can rely on the stdlib itself.
-try:
- from tempfile import TemporaryDirectory
-
-except ImportError:
- from tempfile import mkdtemp, template
-
- class TemporaryDirectory(object):
- """Create and return a temporary directory. This has the same
- behavior as mkdtemp but can be used as a context manager. For
- example:
-
- with TemporaryDirectory() as tmpdir:
- ...
-
- Upon exiting the context, the directory and everthing contained
- in it are removed.
- """
-
- def __init__(self, suffix="", prefix=template, dir=None):
- self.name = mkdtemp(suffix, prefix, dir)
- self._closed = False
-
- def __enter__(self):
- return self.name
-
- def cleanup(self, _warn=False):
- if self.name and not self._closed:
- try:
- self._rmtree(self.name)
- except (TypeError, AttributeError) as ex:
- # Issue #10188: Emit a warning on stderr
- # if the directory could not be cleaned
- # up due to missing globals
- if "None" not in str(ex):
- raise
- print("ERROR: {!r} while cleaning up {!r}".format(ex, self,),
- file=_sys.stderr)
- return
- self._closed = True
- if _warn:
- self._warn("Implicitly cleaning up {!r}".format(self),
- Warning)
-
- def __exit__(self, exc, value, tb):
- self.cleanup()
-
- def __del__(self):
- # Issue a ResourceWarning if implicit cleanup needed
- self.cleanup(_warn=True)
-
-
- # XXX (ncoghlan): The following code attempts to make
- # this class tolerant of the module nulling out process
- # that happens during CPython interpreter shutdown
- # Alas, it doesn't actually manage it. See issue #10188
- _listdir = staticmethod(_os.listdir)
- _path_join = staticmethod(_os.path.join)
- _isdir = staticmethod(_os.path.isdir)
- _remove = staticmethod(_os.remove)
- _rmdir = staticmethod(_os.rmdir)
- _os_error = _os.error
- _warn = _warnings.warn
-
- def _rmtree(self, path):
- # Essentially a stripped down version of shutil.rmtree. We can't
- # use globals because they may be None'ed out at shutdown.
- for name in self._listdir(path):
- fullname = self._path_join(path, name)
- try:
- isdir = self._isdir(fullname)
- except self._os_error:
- isdir = False
- if isdir:
- self._rmtree(fullname)
- else:
- try:
- self._remove(fullname)
- except self._os_error:
- pass
- try:
- self._rmdir(path)
- except self._os_error:
- pass
-
-
-class NamedFileInTemporaryDirectory(object):
-
- def __init__(self, filename, mode='w+b', bufsize=-1, **kwds):
- """
- Open a file named `filename` in a temporary directory.
-
- This context manager is preferred over `NamedTemporaryFile` in
- stdlib `tempfile` when one needs to reopen the file.
-
- Arguments `mode` and `bufsize` are passed to `open`.
- Rest of the arguments are passed to `TemporaryDirectory`.
-
- """
- self._tmpdir = TemporaryDirectory(**kwds)
- path = _os.path.join(self._tmpdir.name, filename)
- self.file = open(path, mode, bufsize)
-
- def cleanup(self):
- self.file.close()
- self._tmpdir.cleanup()
-
- __del__ = cleanup
-
- def __enter__(self):
- return self.file
-
- def __exit__(self, type, value, traceback):
- self.cleanup()
-
-
-class TemporaryWorkingDirectory(TemporaryDirectory):
- """
- Creates a temporary directory and sets the cwd to that directory.
- Automatically reverts to previous cwd upon cleanup.
- Usage example:
-
- with TemporaryWorkingDirectory() as tmpdir:
- ...
- """
- def __enter__(self):
- self.old_wd = _os.getcwd()
- _os.chdir(self.name)
- return super(TemporaryWorkingDirectory, self).__enter__()
-
- def __exit__(self, exc, value, tb):
- _os.chdir(self.old_wd)
- return super(TemporaryWorkingDirectory, self).__exit__(exc, value, tb)
-
+"""TemporaryDirectory class, copied from Python 3.2.
+
+This is copied from the stdlib and will be standard in Python 3.2 and onwards.
+"""
+from __future__ import print_function
+
+import os as _os
+import warnings as _warnings
+import sys as _sys
+
+# This code should only be used in Python versions < 3.2, since after that we
+# can rely on the stdlib itself.
+try:
+ from tempfile import TemporaryDirectory
+
+except ImportError:
+ from tempfile import mkdtemp, template
+
+ class TemporaryDirectory(object):
+ """Create and return a temporary directory. This has the same
+ behavior as mkdtemp but can be used as a context manager. For
+ example:
+
+ with TemporaryDirectory() as tmpdir:
+ ...
+
+ Upon exiting the context, the directory and everthing contained
+ in it are removed.
+ """
+
+ def __init__(self, suffix="", prefix=template, dir=None):
+ self.name = mkdtemp(suffix, prefix, dir)
+ self._closed = False
+
+ def __enter__(self):
+ return self.name
+
+ def cleanup(self, _warn=False):
+ if self.name and not self._closed:
+ try:
+ self._rmtree(self.name)
+ except (TypeError, AttributeError) as ex:
+ # Issue #10188: Emit a warning on stderr
+ # if the directory could not be cleaned
+ # up due to missing globals
+ if "None" not in str(ex):
+ raise
+ print("ERROR: {!r} while cleaning up {!r}".format(ex, self,),
+ file=_sys.stderr)
+ return
+ self._closed = True
+ if _warn:
+ self._warn("Implicitly cleaning up {!r}".format(self),
+ Warning)
+
+ def __exit__(self, exc, value, tb):
+ self.cleanup()
+
+ def __del__(self):
+ # Issue a ResourceWarning if implicit cleanup needed
+ self.cleanup(_warn=True)
+
+
+ # XXX (ncoghlan): The following code attempts to make
+ # this class tolerant of the module nulling out process
+ # that happens during CPython interpreter shutdown
+ # Alas, it doesn't actually manage it. See issue #10188
+ _listdir = staticmethod(_os.listdir)
+ _path_join = staticmethod(_os.path.join)
+ _isdir = staticmethod(_os.path.isdir)
+ _remove = staticmethod(_os.remove)
+ _rmdir = staticmethod(_os.rmdir)
+ _os_error = _os.error
+ _warn = _warnings.warn
+
+ def _rmtree(self, path):
+ # Essentially a stripped down version of shutil.rmtree. We can't
+ # use globals because they may be None'ed out at shutdown.
+ for name in self._listdir(path):
+ fullname = self._path_join(path, name)
+ try:
+ isdir = self._isdir(fullname)
+ except self._os_error:
+ isdir = False
+ if isdir:
+ self._rmtree(fullname)
+ else:
+ try:
+ self._remove(fullname)
+ except self._os_error:
+ pass
+ try:
+ self._rmdir(path)
+ except self._os_error:
+ pass
+
+
+class NamedFileInTemporaryDirectory(object):
+
+ def __init__(self, filename, mode='w+b', bufsize=-1, **kwds):
+ """
+ Open a file named `filename` in a temporary directory.
+
+ This context manager is preferred over `NamedTemporaryFile` in
+ stdlib `tempfile` when one needs to reopen the file.
+
+ Arguments `mode` and `bufsize` are passed to `open`.
+ Rest of the arguments are passed to `TemporaryDirectory`.
+
+ """
+ self._tmpdir = TemporaryDirectory(**kwds)
+ path = _os.path.join(self._tmpdir.name, filename)
+ self.file = open(path, mode, bufsize)
+
+ def cleanup(self):
+ self.file.close()
+ self._tmpdir.cleanup()
+
+ __del__ = cleanup
+
+ def __enter__(self):
+ return self.file
+
+ def __exit__(self, type, value, traceback):
+ self.cleanup()
+
+
+class TemporaryWorkingDirectory(TemporaryDirectory):
+ """
+ Creates a temporary directory and sets the cwd to that directory.
+ Automatically reverts to previous cwd upon cleanup.
+ Usage example:
+
+ with TemporaryWorkingDirectory() as tmpdir:
+ ...
+ """
+ def __enter__(self):
+ self.old_wd = _os.getcwd()
+ _os.chdir(self.name)
+ return super(TemporaryWorkingDirectory, self).__enter__()
+
+ def __exit__(self, exc, value, tb):
+ _os.chdir(self.old_wd)
+ return super(TemporaryWorkingDirectory, self).__exit__(exc, value, tb)
+
diff --git a/contrib/python/ipython/py2/IPython/utils/terminal.py b/contrib/python/ipython/py2/IPython/utils/terminal.py
index e92c410c79..833debce41 100644
--- a/contrib/python/ipython/py2/IPython/utils/terminal.py
+++ b/contrib/python/ipython/py2/IPython/utils/terminal.py
@@ -1,22 +1,22 @@
-# encoding: utf-8
-"""
-Utilities for working with terminals.
-
-Authors:
-
-* Brian E. Granger
-* Fernando Perez
-* Alexander Belchenko (e-mail: bialix AT ukr.net)
-"""
-
+# encoding: utf-8
+"""
+Utilities for working with terminals.
+
+Authors:
+
+* Brian E. Granger
+* Fernando Perez
+* Alexander Belchenko (e-mail: bialix AT ukr.net)
+"""
+
from __future__ import absolute_import
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
-
-import os
-import sys
-import warnings
+
+import os
+import sys
+import warnings
try:
from shutil import get_terminal_size as _get_terminal_size
except ImportError:
@@ -25,101 +25,101 @@ except ImportError:
from backports.shutil_get_terminal_size import get_terminal_size as _get_terminal_size
except ImportError:
from ._get_terminal_size import get_terminal_size as _get_terminal_size
-
-from . import py3compat
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-# This variable is part of the expected API of the module:
-ignore_termtitle = True
-
-
-
-if os.name == 'posix':
- def _term_clear():
- os.system('clear')
-elif sys.platform == 'win32':
- def _term_clear():
- os.system('cls')
-else:
- def _term_clear():
- pass
-
-
-
-def toggle_set_term_title(val):
- """Control whether set_term_title is active or not.
-
- set_term_title() allows writing to the console titlebar. In embedded
- widgets this can cause problems, so this call can be used to toggle it on
- or off as needed.
-
- The default state of the module is for the function to be disabled.
-
- Parameters
- ----------
- val : bool
- If True, set_term_title() actually writes to the terminal (using the
- appropriate platform-specific module). If False, it is a no-op.
- """
- global ignore_termtitle
- ignore_termtitle = not(val)
-
-
-def _set_term_title(*args,**kw):
- """Dummy no-op."""
- pass
-
-
-def _set_term_title_xterm(title):
- """ Change virtual terminal title in xterm-workalikes """
- sys.stdout.write('\033]0;%s\007' % title)
-
-if os.name == 'posix':
- TERM = os.environ.get('TERM','')
- if TERM.startswith('xterm'):
- _set_term_title = _set_term_title_xterm
-elif sys.platform == 'win32':
- try:
- import ctypes
-
- SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
- SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
-
- def _set_term_title(title):
- """Set terminal title using ctypes to access the Win32 APIs."""
- SetConsoleTitleW(title)
- except ImportError:
- def _set_term_title(title):
- """Set terminal title using the 'title' command."""
- global ignore_termtitle
-
- try:
- # Cannot be on network share when issuing system commands
- curr = py3compat.getcwd()
- os.chdir("C:")
- ret = os.system("title " + title)
- finally:
- os.chdir(curr)
- if ret:
- # non-zero return code signals error, don't try again
- ignore_termtitle = True
-
-
-def set_term_title(title):
- """Set terminal title using the necessary platform-dependent calls."""
- if ignore_termtitle:
- return
- _set_term_title(title)
-
-
-def freeze_term_title():
- warnings.warn("This function is deprecated, use toggle_set_term_title()")
- global ignore_termtitle
- ignore_termtitle = True
-
-
+
+from . import py3compat
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# This variable is part of the expected API of the module:
+ignore_termtitle = True
+
+
+
+if os.name == 'posix':
+ def _term_clear():
+ os.system('clear')
+elif sys.platform == 'win32':
+ def _term_clear():
+ os.system('cls')
+else:
+ def _term_clear():
+ pass
+
+
+
+def toggle_set_term_title(val):
+ """Control whether set_term_title is active or not.
+
+ set_term_title() allows writing to the console titlebar. In embedded
+ widgets this can cause problems, so this call can be used to toggle it on
+ or off as needed.
+
+ The default state of the module is for the function to be disabled.
+
+ Parameters
+ ----------
+ val : bool
+ If True, set_term_title() actually writes to the terminal (using the
+ appropriate platform-specific module). If False, it is a no-op.
+ """
+ global ignore_termtitle
+ ignore_termtitle = not(val)
+
+
+def _set_term_title(*args,**kw):
+ """Dummy no-op."""
+ pass
+
+
+def _set_term_title_xterm(title):
+ """ Change virtual terminal title in xterm-workalikes """
+ sys.stdout.write('\033]0;%s\007' % title)
+
+if os.name == 'posix':
+ TERM = os.environ.get('TERM','')
+ if TERM.startswith('xterm'):
+ _set_term_title = _set_term_title_xterm
+elif sys.platform == 'win32':
+ try:
+ import ctypes
+
+ SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
+ SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
+
+ def _set_term_title(title):
+ """Set terminal title using ctypes to access the Win32 APIs."""
+ SetConsoleTitleW(title)
+ except ImportError:
+ def _set_term_title(title):
+ """Set terminal title using the 'title' command."""
+ global ignore_termtitle
+
+ try:
+ # Cannot be on network share when issuing system commands
+ curr = py3compat.getcwd()
+ os.chdir("C:")
+ ret = os.system("title " + title)
+ finally:
+ os.chdir(curr)
+ if ret:
+ # non-zero return code signals error, don't try again
+ ignore_termtitle = True
+
+
+def set_term_title(title):
+ """Set terminal title using the necessary platform-dependent calls."""
+ if ignore_termtitle:
+ return
+ _set_term_title(title)
+
+
+def freeze_term_title():
+ warnings.warn("This function is deprecated, use toggle_set_term_title()")
+ global ignore_termtitle
+ ignore_termtitle = True
+
+
def get_terminal_size(defaultx=80, defaulty=25):
return _get_terminal_size((defaultx, defaulty))
diff --git a/contrib/python/ipython/py2/IPython/utils/text.py b/contrib/python/ipython/py2/IPython/utils/text.py
index 5ed1a845e3..50ff04e1fc 100644
--- a/contrib/python/ipython/py2/IPython/utils/text.py
+++ b/contrib/python/ipython/py2/IPython/utils/text.py
@@ -1,783 +1,783 @@
-# encoding: utf-8
-"""
-Utilities for working with strings and text.
-
-Inheritance diagram:
-
-.. inheritance-diagram:: IPython.utils.text
- :parts: 3
-"""
-from __future__ import absolute_import
-
-import os
-import re
-import sys
-import textwrap
-from string import Formatter
+# encoding: utf-8
+"""
+Utilities for working with strings and text.
+
+Inheritance diagram:
+
+.. inheritance-diagram:: IPython.utils.text
+ :parts: 3
+"""
+from __future__ import absolute_import
+
+import os
+import re
+import sys
+import textwrap
+from string import Formatter
try:
from pathlib import Path
except ImportError:
# Python 2 backport
from pathlib2 import Path
-
-from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
-from IPython.utils import py3compat
-
-# datetime.strftime date format for ipython
-if sys.platform == 'win32':
- date_format = "%B %d, %Y"
-else:
- date_format = "%B %-d, %Y"
-
-class LSString(str):
- """String derivative with a special access attributes.
-
- These are normal strings, but with the special attributes:
-
- .l (or .list) : value as list (split on newlines).
- .n (or .nlstr): original value (the string itself).
- .s (or .spstr): value as whitespace-separated string.
- .p (or .paths): list of path objects (requires path.py package)
-
- Any values which require transformations are computed only once and
- cached.
-
- Such strings are very useful to efficiently interact with the shell, which
- typically only understands whitespace-separated options for commands."""
-
- def get_list(self):
- try:
- return self.__list
- except AttributeError:
- self.__list = self.split('\n')
- return self.__list
-
- l = list = property(get_list)
-
- def get_spstr(self):
- try:
- return self.__spstr
- except AttributeError:
- self.__spstr = self.replace('\n',' ')
- return self.__spstr
-
- s = spstr = property(get_spstr)
-
- def get_nlstr(self):
- return self
-
- n = nlstr = property(get_nlstr)
-
- def get_paths(self):
- try:
- return self.__paths
- except AttributeError:
+
+from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
+from IPython.utils import py3compat
+
+# datetime.strftime date format for ipython
+if sys.platform == 'win32':
+ date_format = "%B %d, %Y"
+else:
+ date_format = "%B %-d, %Y"
+
+class LSString(str):
+ """String derivative with a special access attributes.
+
+ These are normal strings, but with the special attributes:
+
+ .l (or .list) : value as list (split on newlines).
+ .n (or .nlstr): original value (the string itself).
+ .s (or .spstr): value as whitespace-separated string.
+ .p (or .paths): list of path objects (requires path.py package)
+
+ Any values which require transformations are computed only once and
+ cached.
+
+ Such strings are very useful to efficiently interact with the shell, which
+ typically only understands whitespace-separated options for commands."""
+
+ def get_list(self):
+ try:
+ return self.__list
+ except AttributeError:
+ self.__list = self.split('\n')
+ return self.__list
+
+ l = list = property(get_list)
+
+ def get_spstr(self):
+ try:
+ return self.__spstr
+ except AttributeError:
+ self.__spstr = self.replace('\n',' ')
+ return self.__spstr
+
+ s = spstr = property(get_spstr)
+
+ def get_nlstr(self):
+ return self
+
+ n = nlstr = property(get_nlstr)
+
+ def get_paths(self):
+ try:
+ return self.__paths
+ except AttributeError:
self.__paths = [Path(p) for p in self.split('\n') if os.path.exists(p)]
- return self.__paths
-
- p = paths = property(get_paths)
-
-# FIXME: We need to reimplement type specific displayhook and then add this
-# back as a custom printer. This should also be moved outside utils into the
-# core.
-
-# def print_lsstring(arg):
-# """ Prettier (non-repr-like) and more informative printer for LSString """
-# print "LSString (.p, .n, .l, .s available). Value:"
-# print arg
-#
-#
-# print_lsstring = result_display.when_type(LSString)(print_lsstring)
-
-
-class SList(list):
- """List derivative with a special access attributes.
-
- These are normal lists, but with the special attributes:
-
- * .l (or .list) : value as list (the list itself).
- * .n (or .nlstr): value as a string, joined on newlines.
- * .s (or .spstr): value as a string, joined on spaces.
- * .p (or .paths): list of path objects (requires path.py package)
-
- Any values which require transformations are computed only once and
- cached."""
-
- def get_list(self):
- return self
-
- l = list = property(get_list)
-
- def get_spstr(self):
- try:
- return self.__spstr
- except AttributeError:
- self.__spstr = ' '.join(self)
- return self.__spstr
-
- s = spstr = property(get_spstr)
-
- def get_nlstr(self):
- try:
- return self.__nlstr
- except AttributeError:
- self.__nlstr = '\n'.join(self)
- return self.__nlstr
-
- n = nlstr = property(get_nlstr)
-
- def get_paths(self):
- try:
- return self.__paths
- except AttributeError:
+ return self.__paths
+
+ p = paths = property(get_paths)
+
+# FIXME: We need to reimplement type specific displayhook and then add this
+# back as a custom printer. This should also be moved outside utils into the
+# core.
+
+# def print_lsstring(arg):
+# """ Prettier (non-repr-like) and more informative printer for LSString """
+# print "LSString (.p, .n, .l, .s available). Value:"
+# print arg
+#
+#
+# print_lsstring = result_display.when_type(LSString)(print_lsstring)
+
+
+class SList(list):
+ """List derivative with a special access attributes.
+
+ These are normal lists, but with the special attributes:
+
+ * .l (or .list) : value as list (the list itself).
+ * .n (or .nlstr): value as a string, joined on newlines.
+ * .s (or .spstr): value as a string, joined on spaces.
+ * .p (or .paths): list of path objects (requires path.py package)
+
+ Any values which require transformations are computed only once and
+ cached."""
+
+ def get_list(self):
+ return self
+
+ l = list = property(get_list)
+
+ def get_spstr(self):
+ try:
+ return self.__spstr
+ except AttributeError:
+ self.__spstr = ' '.join(self)
+ return self.__spstr
+
+ s = spstr = property(get_spstr)
+
+ def get_nlstr(self):
+ try:
+ return self.__nlstr
+ except AttributeError:
+ self.__nlstr = '\n'.join(self)
+ return self.__nlstr
+
+ n = nlstr = property(get_nlstr)
+
+ def get_paths(self):
+ try:
+ return self.__paths
+ except AttributeError:
self.__paths = [Path(p) for p in self if os.path.exists(p)]
- return self.__paths
-
- p = paths = property(get_paths)
-
- def grep(self, pattern, prune = False, field = None):
- """ Return all strings matching 'pattern' (a regex or callable)
-
- This is case-insensitive. If prune is true, return all items
- NOT matching the pattern.
-
- If field is specified, the match must occur in the specified
- whitespace-separated field.
-
- Examples::
-
- a.grep( lambda x: x.startswith('C') )
- a.grep('Cha.*log', prune=1)
- a.grep('chm', field=-1)
- """
-
- def match_target(s):
- if field is None:
- return s
- parts = s.split()
- try:
- tgt = parts[field]
- return tgt
- except IndexError:
- return ""
-
- if isinstance(pattern, py3compat.string_types):
- pred = lambda x : re.search(pattern, x, re.IGNORECASE)
- else:
- pred = pattern
- if not prune:
- return SList([el for el in self if pred(match_target(el))])
- else:
- return SList([el for el in self if not pred(match_target(el))])
-
- def fields(self, *fields):
- """ Collect whitespace-separated fields from string list
-
- Allows quick awk-like usage of string lists.
-
- Example data (in var a, created by 'a = !ls -l')::
-
- -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
- drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
-
- * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
- * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
- (note the joining by space).
- * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
-
- IndexErrors are ignored.
-
- Without args, fields() just split()'s the strings.
- """
- if len(fields) == 0:
- return [el.split() for el in self]
-
- res = SList()
- for el in [f.split() for f in self]:
- lineparts = []
-
- for fd in fields:
- try:
- lineparts.append(el[fd])
- except IndexError:
- pass
- if lineparts:
- res.append(" ".join(lineparts))
-
- return res
-
- def sort(self,field= None, nums = False):
- """ sort by specified fields (see fields())
-
- Example::
-
- a.sort(1, nums = True)
-
- Sorts a by second field, in numerical order (so that 21 > 3)
-
- """
-
- #decorate, sort, undecorate
- if field is not None:
- dsu = [[SList([line]).fields(field), line] for line in self]
- else:
- dsu = [[line, line] for line in self]
- if nums:
- for i in range(len(dsu)):
- numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
- try:
- n = int(numstr)
- except ValueError:
- n = 0
- dsu[i][0] = n
-
-
- dsu.sort()
- return SList([t[1] for t in dsu])
-
-
-# FIXME: We need to reimplement type specific displayhook and then add this
-# back as a custom printer. This should also be moved outside utils into the
-# core.
-
-# def print_slist(arg):
-# """ Prettier (non-repr-like) and more informative printer for SList """
-# print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
-# if hasattr(arg, 'hideonce') and arg.hideonce:
-# arg.hideonce = False
-# return
-#
-# nlprint(arg) # This was a nested list printer, now removed.
-#
-# print_slist = result_display.when_type(SList)(print_slist)
-
-
-def indent(instr,nspaces=4, ntabs=0, flatten=False):
- """Indent a string a given number of spaces or tabstops.
-
- indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
-
- Parameters
- ----------
-
- instr : basestring
- The string to be indented.
- nspaces : int (default: 4)
- The number of spaces to be indented.
- ntabs : int (default: 0)
- The number of tabs to be indented.
- flatten : bool (default: False)
- Whether to scrub existing indentation. If True, all lines will be
- aligned to the same indentation. If False, existing indentation will
- be strictly increased.
-
- Returns
- -------
-
- str|unicode : string indented by ntabs and nspaces.
-
- """
- if instr is None:
- return
- ind = '\t'*ntabs+' '*nspaces
- if flatten:
- pat = re.compile(r'^\s*', re.MULTILINE)
- else:
- pat = re.compile(r'^', re.MULTILINE)
- outstr = re.sub(pat, ind, instr)
- if outstr.endswith(os.linesep+ind):
- return outstr[:-len(ind)]
- else:
- return outstr
-
-
-def list_strings(arg):
- """Always return a list of strings, given a string or list of strings
- as input.
-
- Examples
- --------
- ::
-
- In [7]: list_strings('A single string')
- Out[7]: ['A single string']
-
- In [8]: list_strings(['A single string in a list'])
- Out[8]: ['A single string in a list']
-
- In [9]: list_strings(['A','list','of','strings'])
- Out[9]: ['A', 'list', 'of', 'strings']
- """
-
- if isinstance(arg, py3compat.string_types): return [arg]
- else: return arg
-
-
-def marquee(txt='',width=78,mark='*'):
- """Return the input string centered in a 'marquee'.
-
- Examples
- --------
- ::
-
- In [16]: marquee('A test',40)
- Out[16]: '**************** A test ****************'
-
- In [17]: marquee('A test',40,'-')
- Out[17]: '---------------- A test ----------------'
-
- In [18]: marquee('A test',40,' ')
- Out[18]: ' A test '
-
- """
- if not txt:
- return (mark*width)[:width]
- nmark = (width-len(txt)-2)//len(mark)//2
- if nmark < 0: nmark =0
- marks = mark*nmark
- return '%s %s %s' % (marks,txt,marks)
-
-
-ini_spaces_re = re.compile(r'^(\s+)')
-
-def num_ini_spaces(strng):
- """Return the number of initial spaces in a string"""
-
- ini_spaces = ini_spaces_re.match(strng)
- if ini_spaces:
- return ini_spaces.end()
- else:
- return 0
-
-
-def format_screen(strng):
- """Format a string for screen printing.
-
- This removes some latex-type format codes."""
- # Paragraph continue
- par_re = re.compile(r'\\$',re.MULTILINE)
- strng = par_re.sub('',strng)
- return strng
-
-
-def dedent(text):
- """Equivalent of textwrap.dedent that ignores unindented first line.
-
- This means it will still dedent strings like:
- '''foo
- is a bar
- '''
-
- For use in wrap_paragraphs.
- """
-
- if text.startswith('\n'):
- # text starts with blank line, don't ignore the first line
- return textwrap.dedent(text)
-
- # split first line
- splits = text.split('\n',1)
- if len(splits) == 1:
- # only one line
- return textwrap.dedent(text)
-
- first, rest = splits
- # dedent everything but the first line
- rest = textwrap.dedent(rest)
- return '\n'.join([first, rest])
-
-
-def wrap_paragraphs(text, ncols=80):
- """Wrap multiple paragraphs to fit a specified width.
-
- This is equivalent to textwrap.wrap, but with support for multiple
- paragraphs, as separated by empty lines.
-
- Returns
- -------
-
- list of complete paragraphs, wrapped to fill `ncols` columns.
- """
- paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
- text = dedent(text).strip()
- paragraphs = paragraph_re.split(text)[::2] # every other entry is space
- out_ps = []
- indent_re = re.compile(r'\n\s+', re.MULTILINE)
- for p in paragraphs:
- # presume indentation that survives dedent is meaningful formatting,
- # so don't fill unless text is flush.
- if indent_re.search(p) is None:
- # wrap paragraph
- p = textwrap.fill(p, ncols)
- out_ps.append(p)
- return out_ps
-
-
-def long_substr(data):
- """Return the longest common substring in a list of strings.
-
- Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
- """
- substr = ''
- if len(data) > 1 and len(data[0]) > 0:
- for i in range(len(data[0])):
- for j in range(len(data[0])-i+1):
- if j > len(substr) and all(data[0][i:i+j] in x for x in data):
- substr = data[0][i:i+j]
- elif len(data) == 1:
- substr = data[0]
- return substr
-
-
-def strip_email_quotes(text):
- """Strip leading email quotation characters ('>').
-
- Removes any combination of leading '>' interspersed with whitespace that
- appears *identically* in all lines of the input text.
-
- Parameters
- ----------
- text : str
-
- Examples
- --------
-
- Simple uses::
-
- In [2]: strip_email_quotes('> > text')
- Out[2]: 'text'
-
- In [3]: strip_email_quotes('> > text\\n> > more')
- Out[3]: 'text\\nmore'
-
- Note how only the common prefix that appears in all lines is stripped::
-
- In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
- Out[4]: '> text\\n> more\\nmore...'
-
- So if any line has no quote marks ('>') , then none are stripped from any
- of them ::
-
- In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
- Out[5]: '> > text\\n> > more\\nlast different'
- """
- lines = text.splitlines()
- matches = set()
- for line in lines:
- prefix = re.match(r'^(\s*>[ >]*)', line)
- if prefix:
- matches.add(prefix.group(1))
- else:
- break
- else:
- prefix = long_substr(list(matches))
- if prefix:
- strip = len(prefix)
- text = '\n'.join([ ln[strip:] for ln in lines])
- return text
-
-def strip_ansi(source):
- """
- Remove ansi escape codes from text.
-
- Parameters
- ----------
- source : str
- Source to remove the ansi from
- """
- return re.sub(r'\033\[(\d|;)+?m', '', source)
-
-
-class EvalFormatter(Formatter):
- """A String Formatter that allows evaluation of simple expressions.
-
- Note that this version interprets a : as specifying a format string (as per
- standard string formatting), so if slicing is required, you must explicitly
- create a slice.
-
- This is to be used in templating cases, such as the parallel batch
- script templates, where simple arithmetic on arguments is useful.
-
- Examples
- --------
- ::
-
- In [1]: f = EvalFormatter()
- In [2]: f.format('{n//4}', n=8)
- Out[2]: '2'
-
- In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
- Out[3]: 'll'
- """
- def get_field(self, name, args, kwargs):
- v = eval(name, kwargs)
- return v, name
-
-#XXX: As of Python 3.4, the format string parsing no longer splits on a colon
-# inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
-# above, it should be possible to remove FullEvalFormatter.
-
-@skip_doctest_py3
-class FullEvalFormatter(Formatter):
- """A String Formatter that allows evaluation of simple expressions.
-
- Any time a format key is not found in the kwargs,
- it will be tried as an expression in the kwargs namespace.
-
- Note that this version allows slicing using [1:2], so you cannot specify
- a format string. Use :class:`EvalFormatter` to permit format strings.
-
- Examples
- --------
- ::
-
- In [1]: f = FullEvalFormatter()
- In [2]: f.format('{n//4}', n=8)
- Out[2]: u'2'
-
- In [3]: f.format('{list(range(5))[2:4]}')
- Out[3]: u'[2, 3]'
-
- In [4]: f.format('{3*2}')
- Out[4]: u'6'
- """
- # copied from Formatter._vformat with minor changes to allow eval
- # and replace the format_spec code with slicing
- def vformat(self, format_string, args, kwargs):
- result = []
- for literal_text, field_name, format_spec, conversion in \
- self.parse(format_string):
-
- # output the literal text
- if literal_text:
- result.append(literal_text)
-
- # if there's a field, output it
- if field_name is not None:
- # this is some markup, find the object and do
- # the formatting
-
- if format_spec:
- # override format spec, to allow slicing:
- field_name = ':'.join([field_name, format_spec])
-
- # eval the contents of the field for the object
- # to be formatted
- obj = eval(field_name, kwargs)
-
- # do any conversion on the resulting object
- obj = self.convert_field(obj, conversion)
-
- # format the object and append to the result
- result.append(self.format_field(obj, ''))
-
- return u''.join(py3compat.cast_unicode(s) for s in result)
-
-
-@skip_doctest_py3
-class DollarFormatter(FullEvalFormatter):
- """Formatter allowing Itpl style $foo replacement, for names and attribute
- access only. Standard {foo} replacement also works, and allows full
- evaluation of its arguments.
-
- Examples
- --------
- ::
-
- In [1]: f = DollarFormatter()
- In [2]: f.format('{n//4}', n=8)
- Out[2]: u'2'
-
- In [3]: f.format('23 * 76 is $result', result=23*76)
- Out[3]: u'23 * 76 is 1748'
-
- In [4]: f.format('$a or {b}', a=1, b=2)
- Out[4]: u'1 or 2'
- """
- _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
- def parse(self, fmt_string):
- for literal_txt, field_name, format_spec, conversion \
- in Formatter.parse(self, fmt_string):
-
- # Find $foo patterns in the literal text.
- continue_from = 0
- txt = ""
- for m in self._dollar_pattern.finditer(literal_txt):
- new_txt, new_field = m.group(1,2)
- # $$foo --> $foo
- if new_field.startswith("$"):
- txt += new_txt + new_field
- else:
- yield (txt + new_txt, new_field, "", None)
- txt = ""
- continue_from = m.end()
-
- # Re-yield the {foo} style pattern
- yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
-
-#-----------------------------------------------------------------------------
-# Utils to columnize a list of string
-#-----------------------------------------------------------------------------
-
-def _col_chunks(l, max_rows, row_first=False):
- """Yield successive max_rows-sized column chunks from l."""
- if row_first:
- ncols = (len(l) // max_rows) + (len(l) % max_rows > 0)
- for i in py3compat.xrange(ncols):
- yield [l[j] for j in py3compat.xrange(i, len(l), ncols)]
- else:
- for i in py3compat.xrange(0, len(l), max_rows):
- yield l[i:(i + max_rows)]
-
-
-def _find_optimal(rlist, row_first=False, separator_size=2, displaywidth=80):
- """Calculate optimal info to columnize a list of string"""
- for max_rows in range(1, len(rlist) + 1):
- col_widths = list(map(max, _col_chunks(rlist, max_rows, row_first)))
- sumlength = sum(col_widths)
- ncols = len(col_widths)
- if sumlength + separator_size * (ncols - 1) <= displaywidth:
- break
- return {'num_columns': ncols,
- 'optimal_separator_width': (displaywidth - sumlength) / (ncols - 1) if (ncols - 1) else 0,
- 'max_rows': max_rows,
- 'column_widths': col_widths
- }
-
-
-def _get_or_default(mylist, i, default=None):
- """return list item number, or default if don't exist"""
- if i >= len(mylist):
- return default
- else :
- return mylist[i]
-
-
-def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) :
- """Returns a nested list, and info to columnize items
-
- Parameters
- ----------
-
- items
- list of strings to columize
- row_first : (default False)
- Whether to compute columns for a row-first matrix instead of
- column-first (default).
- empty : (default None)
- default value to fill list if needed
- separator_size : int (default=2)
- How much caracters will be used as a separation between each columns.
- displaywidth : int (default=80)
- The width of the area onto wich the columns should enter
-
- Returns
- -------
-
- strings_matrix
-
- nested list of string, the outer most list contains as many list as
- rows, the innermost lists have each as many element as colums. If the
- total number of elements in `items` does not equal the product of
- rows*columns, the last element of some lists are filled with `None`.
-
- dict_info
- some info to make columnize easier:
-
- num_columns
- number of columns
- max_rows
- maximum number of rows (final number may be less)
- column_widths
- list of with of each columns
- optimal_separator_width
- best separator width between columns
-
- Examples
- --------
- ::
-
- In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
- ...: compute_item_matrix(l, displaywidth=12)
- Out[1]:
- ([['aaa', 'f', 'k'],
- ['b', 'g', 'l'],
- ['cc', 'h', None],
- ['d', 'i', None],
- ['eeeee', 'j', None]],
- {'num_columns': 3,
- 'column_widths': [5, 1, 1],
- 'optimal_separator_width': 2,
- 'max_rows': 5})
- """
- info = _find_optimal(list(map(len, items)), row_first, *args, **kwargs)
- nrow, ncol = info['max_rows'], info['num_columns']
- if row_first:
- return ([[_get_or_default(items, r * ncol + c, default=empty) for c in range(ncol)] for r in range(nrow)], info)
- else:
- return ([[_get_or_default(items, c * nrow + r, default=empty) for c in range(ncol)] for r in range(nrow)], info)
-
-
-def columnize(items, row_first=False, separator=' ', displaywidth=80, spread=False):
- """ Transform a list of strings into a single string with columns.
-
- Parameters
- ----------
- items : sequence of strings
- The strings to process.
-
- row_first : (default False)
- Whether to compute columns for a row-first matrix instead of
- column-first (default).
-
- separator : str, optional [default is two spaces]
- The string that separates columns.
-
- displaywidth : int, optional [default is 80]
- Width of the display in number of characters.
-
- Returns
- -------
- The formatted string.
- """
- if not items:
- return '\n'
- matrix, info = compute_item_matrix(items, row_first=row_first, separator_size=len(separator), displaywidth=displaywidth)
- if spread:
- separator = separator.ljust(int(info['optimal_separator_width']))
- fmatrix = [filter(None, x) for x in matrix]
- sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['column_widths'])])
- return '\n'.join(map(sjoin, fmatrix))+'\n'
-
-
-def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
- """
- Return a string with a natural enumeration of items
-
- >>> get_text_list(['a', 'b', 'c', 'd'])
- 'a, b, c and d'
- >>> get_text_list(['a', 'b', 'c'], ' or ')
- 'a, b or c'
- >>> get_text_list(['a', 'b', 'c'], ', ')
- 'a, b, c'
- >>> get_text_list(['a', 'b'], ' or ')
- 'a or b'
- >>> get_text_list(['a'])
- 'a'
- >>> get_text_list([])
- ''
- >>> get_text_list(['a', 'b'], wrap_item_with="`")
- '`a` and `b`'
- >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
- 'a + b + c = d'
- """
- if len(list_) == 0:
- return ''
- if wrap_item_with:
- list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
- item in list_]
- if len(list_) == 1:
- return list_[0]
- return '%s%s%s' % (
- sep.join(i for i in list_[:-1]),
- last_sep, list_[-1])
+ return self.__paths
+
+ p = paths = property(get_paths)
+
+ def grep(self, pattern, prune = False, field = None):
+ """ Return all strings matching 'pattern' (a regex or callable)
+
+ This is case-insensitive. If prune is true, return all items
+ NOT matching the pattern.
+
+ If field is specified, the match must occur in the specified
+ whitespace-separated field.
+
+ Examples::
+
+ a.grep( lambda x: x.startswith('C') )
+ a.grep('Cha.*log', prune=1)
+ a.grep('chm', field=-1)
+ """
+
+ def match_target(s):
+ if field is None:
+ return s
+ parts = s.split()
+ try:
+ tgt = parts[field]
+ return tgt
+ except IndexError:
+ return ""
+
+ if isinstance(pattern, py3compat.string_types):
+ pred = lambda x : re.search(pattern, x, re.IGNORECASE)
+ else:
+ pred = pattern
+ if not prune:
+ return SList([el for el in self if pred(match_target(el))])
+ else:
+ return SList([el for el in self if not pred(match_target(el))])
+
+ def fields(self, *fields):
+ """ Collect whitespace-separated fields from string list
+
+ Allows quick awk-like usage of string lists.
+
+ Example data (in var a, created by 'a = !ls -l')::
+
+ -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog
+ drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython
+
+ * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
+ * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
+ (note the joining by space).
+ * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
+
+ IndexErrors are ignored.
+
+ Without args, fields() just split()'s the strings.
+ """
+ if len(fields) == 0:
+ return [el.split() for el in self]
+
+ res = SList()
+ for el in [f.split() for f in self]:
+ lineparts = []
+
+ for fd in fields:
+ try:
+ lineparts.append(el[fd])
+ except IndexError:
+ pass
+ if lineparts:
+ res.append(" ".join(lineparts))
+
+ return res
+
+ def sort(self,field= None, nums = False):
+ """ sort by specified fields (see fields())
+
+ Example::
+
+ a.sort(1, nums = True)
+
+ Sorts a by second field, in numerical order (so that 21 > 3)
+
+ """
+
+ #decorate, sort, undecorate
+ if field is not None:
+ dsu = [[SList([line]).fields(field), line] for line in self]
+ else:
+ dsu = [[line, line] for line in self]
+ if nums:
+ for i in range(len(dsu)):
+ numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
+ try:
+ n = int(numstr)
+ except ValueError:
+ n = 0
+ dsu[i][0] = n
+
+
+ dsu.sort()
+ return SList([t[1] for t in dsu])
+
+
+# FIXME: We need to reimplement type specific displayhook and then add this
+# back as a custom printer. This should also be moved outside utils into the
+# core.
+
+# def print_slist(arg):
+# """ Prettier (non-repr-like) and more informative printer for SList """
+# print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
+# if hasattr(arg, 'hideonce') and arg.hideonce:
+# arg.hideonce = False
+# return
+#
+# nlprint(arg) # This was a nested list printer, now removed.
+#
+# print_slist = result_display.when_type(SList)(print_slist)
+
+
+def indent(instr,nspaces=4, ntabs=0, flatten=False):
+ """Indent a string a given number of spaces or tabstops.
+
+ indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
+
+ Parameters
+ ----------
+
+ instr : basestring
+ The string to be indented.
+ nspaces : int (default: 4)
+ The number of spaces to be indented.
+ ntabs : int (default: 0)
+ The number of tabs to be indented.
+ flatten : bool (default: False)
+ Whether to scrub existing indentation. If True, all lines will be
+ aligned to the same indentation. If False, existing indentation will
+ be strictly increased.
+
+ Returns
+ -------
+
+ str|unicode : string indented by ntabs and nspaces.
+
+ """
+ if instr is None:
+ return
+ ind = '\t'*ntabs+' '*nspaces
+ if flatten:
+ pat = re.compile(r'^\s*', re.MULTILINE)
+ else:
+ pat = re.compile(r'^', re.MULTILINE)
+ outstr = re.sub(pat, ind, instr)
+ if outstr.endswith(os.linesep+ind):
+ return outstr[:-len(ind)]
+ else:
+ return outstr
+
+
+def list_strings(arg):
+ """Always return a list of strings, given a string or list of strings
+ as input.
+
+ Examples
+ --------
+ ::
+
+ In [7]: list_strings('A single string')
+ Out[7]: ['A single string']
+
+ In [8]: list_strings(['A single string in a list'])
+ Out[8]: ['A single string in a list']
+
+ In [9]: list_strings(['A','list','of','strings'])
+ Out[9]: ['A', 'list', 'of', 'strings']
+ """
+
+ if isinstance(arg, py3compat.string_types): return [arg]
+ else: return arg
+
+
+def marquee(txt='',width=78,mark='*'):
+ """Return the input string centered in a 'marquee'.
+
+ Examples
+ --------
+ ::
+
+ In [16]: marquee('A test',40)
+ Out[16]: '**************** A test ****************'
+
+ In [17]: marquee('A test',40,'-')
+ Out[17]: '---------------- A test ----------------'
+
+ In [18]: marquee('A test',40,' ')
+ Out[18]: ' A test '
+
+ """
+ if not txt:
+ return (mark*width)[:width]
+ nmark = (width-len(txt)-2)//len(mark)//2
+ if nmark < 0: nmark =0
+ marks = mark*nmark
+ return '%s %s %s' % (marks,txt,marks)
+
+
+ini_spaces_re = re.compile(r'^(\s+)')
+
+def num_ini_spaces(strng):
+ """Return the number of initial spaces in a string"""
+
+ ini_spaces = ini_spaces_re.match(strng)
+ if ini_spaces:
+ return ini_spaces.end()
+ else:
+ return 0
+
+
+def format_screen(strng):
+ """Format a string for screen printing.
+
+ This removes some latex-type format codes."""
+ # Paragraph continue
+ par_re = re.compile(r'\\$',re.MULTILINE)
+ strng = par_re.sub('',strng)
+ return strng
+
+
+def dedent(text):
+ """Equivalent of textwrap.dedent that ignores unindented first line.
+
+ This means it will still dedent strings like:
+ '''foo
+ is a bar
+ '''
+
+ For use in wrap_paragraphs.
+ """
+
+ if text.startswith('\n'):
+ # text starts with blank line, don't ignore the first line
+ return textwrap.dedent(text)
+
+ # split first line
+ splits = text.split('\n',1)
+ if len(splits) == 1:
+ # only one line
+ return textwrap.dedent(text)
+
+ first, rest = splits
+ # dedent everything but the first line
+ rest = textwrap.dedent(rest)
+ return '\n'.join([first, rest])
+
+
+def wrap_paragraphs(text, ncols=80):
+ """Wrap multiple paragraphs to fit a specified width.
+
+ This is equivalent to textwrap.wrap, but with support for multiple
+ paragraphs, as separated by empty lines.
+
+ Returns
+ -------
+
+ list of complete paragraphs, wrapped to fill `ncols` columns.
+ """
+ paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
+ text = dedent(text).strip()
+ paragraphs = paragraph_re.split(text)[::2] # every other entry is space
+ out_ps = []
+ indent_re = re.compile(r'\n\s+', re.MULTILINE)
+ for p in paragraphs:
+ # presume indentation that survives dedent is meaningful formatting,
+ # so don't fill unless text is flush.
+ if indent_re.search(p) is None:
+ # wrap paragraph
+ p = textwrap.fill(p, ncols)
+ out_ps.append(p)
+ return out_ps
+
+
+def long_substr(data):
+ """Return the longest common substring in a list of strings.
+
+ Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
+ """
+ substr = ''
+ if len(data) > 1 and len(data[0]) > 0:
+ for i in range(len(data[0])):
+ for j in range(len(data[0])-i+1):
+ if j > len(substr) and all(data[0][i:i+j] in x for x in data):
+ substr = data[0][i:i+j]
+ elif len(data) == 1:
+ substr = data[0]
+ return substr
+
+
+def strip_email_quotes(text):
+ """Strip leading email quotation characters ('>').
+
+ Removes any combination of leading '>' interspersed with whitespace that
+ appears *identically* in all lines of the input text.
+
+ Parameters
+ ----------
+ text : str
+
+ Examples
+ --------
+
+ Simple uses::
+
+ In [2]: strip_email_quotes('> > text')
+ Out[2]: 'text'
+
+ In [3]: strip_email_quotes('> > text\\n> > more')
+ Out[3]: 'text\\nmore'
+
+ Note how only the common prefix that appears in all lines is stripped::
+
+ In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
+ Out[4]: '> text\\n> more\\nmore...'
+
+ So if any line has no quote marks ('>') , then none are stripped from any
+ of them ::
+
+ In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
+ Out[5]: '> > text\\n> > more\\nlast different'
+ """
+ lines = text.splitlines()
+ matches = set()
+ for line in lines:
+ prefix = re.match(r'^(\s*>[ >]*)', line)
+ if prefix:
+ matches.add(prefix.group(1))
+ else:
+ break
+ else:
+ prefix = long_substr(list(matches))
+ if prefix:
+ strip = len(prefix)
+ text = '\n'.join([ ln[strip:] for ln in lines])
+ return text
+
+def strip_ansi(source):
+ """
+ Remove ansi escape codes from text.
+
+ Parameters
+ ----------
+ source : str
+ Source to remove the ansi from
+ """
+ return re.sub(r'\033\[(\d|;)+?m', '', source)
+
+
+class EvalFormatter(Formatter):
+ """A String Formatter that allows evaluation of simple expressions.
+
+ Note that this version interprets a : as specifying a format string (as per
+ standard string formatting), so if slicing is required, you must explicitly
+ create a slice.
+
+ This is to be used in templating cases, such as the parallel batch
+ script templates, where simple arithmetic on arguments is useful.
+
+ Examples
+ --------
+ ::
+
+ In [1]: f = EvalFormatter()
+ In [2]: f.format('{n//4}', n=8)
+ Out[2]: '2'
+
+ In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
+ Out[3]: 'll'
+ """
+ def get_field(self, name, args, kwargs):
+ v = eval(name, kwargs)
+ return v, name
+
+#XXX: As of Python 3.4, the format string parsing no longer splits on a colon
+# inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
+# above, it should be possible to remove FullEvalFormatter.
+
+@skip_doctest_py3
+class FullEvalFormatter(Formatter):
+ """A String Formatter that allows evaluation of simple expressions.
+
+ Any time a format key is not found in the kwargs,
+ it will be tried as an expression in the kwargs namespace.
+
+ Note that this version allows slicing using [1:2], so you cannot specify
+ a format string. Use :class:`EvalFormatter` to permit format strings.
+
+ Examples
+ --------
+ ::
+
+ In [1]: f = FullEvalFormatter()
+ In [2]: f.format('{n//4}', n=8)
+ Out[2]: u'2'
+
+ In [3]: f.format('{list(range(5))[2:4]}')
+ Out[3]: u'[2, 3]'
+
+ In [4]: f.format('{3*2}')
+ Out[4]: u'6'
+ """
+ # copied from Formatter._vformat with minor changes to allow eval
+ # and replace the format_spec code with slicing
+ def vformat(self, format_string, args, kwargs):
+ result = []
+ for literal_text, field_name, format_spec, conversion in \
+ self.parse(format_string):
+
+ # output the literal text
+ if literal_text:
+ result.append(literal_text)
+
+ # if there's a field, output it
+ if field_name is not None:
+ # this is some markup, find the object and do
+ # the formatting
+
+ if format_spec:
+ # override format spec, to allow slicing:
+ field_name = ':'.join([field_name, format_spec])
+
+ # eval the contents of the field for the object
+ # to be formatted
+ obj = eval(field_name, kwargs)
+
+ # do any conversion on the resulting object
+ obj = self.convert_field(obj, conversion)
+
+ # format the object and append to the result
+ result.append(self.format_field(obj, ''))
+
+ return u''.join(py3compat.cast_unicode(s) for s in result)
+
+
+@skip_doctest_py3
+class DollarFormatter(FullEvalFormatter):
+ """Formatter allowing Itpl style $foo replacement, for names and attribute
+ access only. Standard {foo} replacement also works, and allows full
+ evaluation of its arguments.
+
+ Examples
+ --------
+ ::
+
+ In [1]: f = DollarFormatter()
+ In [2]: f.format('{n//4}', n=8)
+ Out[2]: u'2'
+
+ In [3]: f.format('23 * 76 is $result', result=23*76)
+ Out[3]: u'23 * 76 is 1748'
+
+ In [4]: f.format('$a or {b}', a=1, b=2)
+ Out[4]: u'1 or 2'
+ """
+ _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
+ def parse(self, fmt_string):
+ for literal_txt, field_name, format_spec, conversion \
+ in Formatter.parse(self, fmt_string):
+
+ # Find $foo patterns in the literal text.
+ continue_from = 0
+ txt = ""
+ for m in self._dollar_pattern.finditer(literal_txt):
+ new_txt, new_field = m.group(1,2)
+ # $$foo --> $foo
+ if new_field.startswith("$"):
+ txt += new_txt + new_field
+ else:
+ yield (txt + new_txt, new_field, "", None)
+ txt = ""
+ continue_from = m.end()
+
+ # Re-yield the {foo} style pattern
+ yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
+
+#-----------------------------------------------------------------------------
+# Utils to columnize a list of string
+#-----------------------------------------------------------------------------
+
+def _col_chunks(l, max_rows, row_first=False):
+ """Yield successive max_rows-sized column chunks from l."""
+ if row_first:
+ ncols = (len(l) // max_rows) + (len(l) % max_rows > 0)
+ for i in py3compat.xrange(ncols):
+ yield [l[j] for j in py3compat.xrange(i, len(l), ncols)]
+ else:
+ for i in py3compat.xrange(0, len(l), max_rows):
+ yield l[i:(i + max_rows)]
+
+
+def _find_optimal(rlist, row_first=False, separator_size=2, displaywidth=80):
+ """Calculate optimal info to columnize a list of string"""
+ for max_rows in range(1, len(rlist) + 1):
+ col_widths = list(map(max, _col_chunks(rlist, max_rows, row_first)))
+ sumlength = sum(col_widths)
+ ncols = len(col_widths)
+ if sumlength + separator_size * (ncols - 1) <= displaywidth:
+ break
+ return {'num_columns': ncols,
+ 'optimal_separator_width': (displaywidth - sumlength) / (ncols - 1) if (ncols - 1) else 0,
+ 'max_rows': max_rows,
+ 'column_widths': col_widths
+ }
+
+
+def _get_or_default(mylist, i, default=None):
+ """return list item number, or default if don't exist"""
+ if i >= len(mylist):
+ return default
+ else :
+ return mylist[i]
+
+
+def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) :
+ """Returns a nested list, and info to columnize items
+
+ Parameters
+ ----------
+
+ items
+ list of strings to columize
+ row_first : (default False)
+ Whether to compute columns for a row-first matrix instead of
+ column-first (default).
+ empty : (default None)
+ default value to fill list if needed
+ separator_size : int (default=2)
+ How much caracters will be used as a separation between each columns.
+ displaywidth : int (default=80)
+ The width of the area onto wich the columns should enter
+
+ Returns
+ -------
+
+ strings_matrix
+
+ nested list of string, the outer most list contains as many list as
+ rows, the innermost lists have each as many element as colums. If the
+ total number of elements in `items` does not equal the product of
+ rows*columns, the last element of some lists are filled with `None`.
+
+ dict_info
+ some info to make columnize easier:
+
+ num_columns
+ number of columns
+ max_rows
+ maximum number of rows (final number may be less)
+ column_widths
+ list of with of each columns
+ optimal_separator_width
+ best separator width between columns
+
+ Examples
+ --------
+ ::
+
+ In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
+ ...: compute_item_matrix(l, displaywidth=12)
+ Out[1]:
+ ([['aaa', 'f', 'k'],
+ ['b', 'g', 'l'],
+ ['cc', 'h', None],
+ ['d', 'i', None],
+ ['eeeee', 'j', None]],
+ {'num_columns': 3,
+ 'column_widths': [5, 1, 1],
+ 'optimal_separator_width': 2,
+ 'max_rows': 5})
+ """
+ info = _find_optimal(list(map(len, items)), row_first, *args, **kwargs)
+ nrow, ncol = info['max_rows'], info['num_columns']
+ if row_first:
+ return ([[_get_or_default(items, r * ncol + c, default=empty) for c in range(ncol)] for r in range(nrow)], info)
+ else:
+ return ([[_get_or_default(items, c * nrow + r, default=empty) for c in range(ncol)] for r in range(nrow)], info)
+
+
+def columnize(items, row_first=False, separator=' ', displaywidth=80, spread=False):
+ """ Transform a list of strings into a single string with columns.
+
+ Parameters
+ ----------
+ items : sequence of strings
+ The strings to process.
+
+ row_first : (default False)
+ Whether to compute columns for a row-first matrix instead of
+ column-first (default).
+
+ separator : str, optional [default is two spaces]
+ The string that separates columns.
+
+ displaywidth : int, optional [default is 80]
+ Width of the display in number of characters.
+
+ Returns
+ -------
+ The formatted string.
+ """
+ if not items:
+ return '\n'
+ matrix, info = compute_item_matrix(items, row_first=row_first, separator_size=len(separator), displaywidth=displaywidth)
+ if spread:
+ separator = separator.ljust(int(info['optimal_separator_width']))
+ fmatrix = [filter(None, x) for x in matrix]
+ sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['column_widths'])])
+ return '\n'.join(map(sjoin, fmatrix))+'\n'
+
+
+def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
+ """
+ Return a string with a natural enumeration of items
+
+ >>> get_text_list(['a', 'b', 'c', 'd'])
+ 'a, b, c and d'
+ >>> get_text_list(['a', 'b', 'c'], ' or ')
+ 'a, b or c'
+ >>> get_text_list(['a', 'b', 'c'], ', ')
+ 'a, b, c'
+ >>> get_text_list(['a', 'b'], ' or ')
+ 'a or b'
+ >>> get_text_list(['a'])
+ 'a'
+ >>> get_text_list([])
+ ''
+ >>> get_text_list(['a', 'b'], wrap_item_with="`")
+ '`a` and `b`'
+ >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
+ 'a + b + c = d'
+ """
+ if len(list_) == 0:
+ return ''
+ if wrap_item_with:
+ list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
+ item in list_]
+ if len(list_) == 1:
+ return list_[0]
+ return '%s%s%s' % (
+ sep.join(i for i in list_[:-1]),
+ last_sep, list_[-1])
diff --git a/contrib/python/ipython/py2/IPython/utils/timing.py b/contrib/python/ipython/py2/IPython/utils/timing.py
index 99b7bbc59a..ff88bf664d 100644
--- a/contrib/python/ipython/py2/IPython/utils/timing.py
+++ b/contrib/python/ipython/py2/IPython/utils/timing.py
@@ -1,118 +1,118 @@
-# encoding: utf-8
-"""
-Utilities for timing code execution.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2008-2011 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import time
-
-from .py3compat import xrange
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-# If possible (Unix), use the resource module instead of time.clock()
-try:
- import resource
- def clocku():
- """clocku() -> floating point number
-
- Return the *USER* CPU time in seconds since the start of the process.
- This is done via a call to resource.getrusage, so it avoids the
- wraparound problems in time.clock()."""
-
- return resource.getrusage(resource.RUSAGE_SELF)[0]
-
- def clocks():
- """clocks() -> floating point number
-
- Return the *SYSTEM* CPU time in seconds since the start of the process.
- This is done via a call to resource.getrusage, so it avoids the
- wraparound problems in time.clock()."""
-
- return resource.getrusage(resource.RUSAGE_SELF)[1]
-
- def clock():
- """clock() -> floating point number
-
- Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
- the process. This is done via a call to resource.getrusage, so it
- avoids the wraparound problems in time.clock()."""
-
- u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
- return u+s
-
- def clock2():
- """clock2() -> (t_user,t_system)
-
- Similar to clock(), but return a tuple of user/system times."""
- return resource.getrusage(resource.RUSAGE_SELF)[:2]
-except ImportError:
- # There is no distinction of user/system time under windows, so we just use
- # time.clock() for everything...
- clocku = clocks = clock = time.clock
- def clock2():
- """Under windows, system CPU time can't be measured.
-
- This just returns clock() and zero."""
- return time.clock(),0.0
-
-
-def timings_out(reps,func,*args,**kw):
- """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
-
- Execute a function reps times, return a tuple with the elapsed total
- CPU time in seconds, the time per call and the function's output.
-
- Under Unix, the return value is the sum of user+system time consumed by
- the process, computed via the resource module. This prevents problems
- related to the wraparound effect which the time.clock() function has.
-
- Under Windows the return value is in wall clock seconds. See the
- documentation for the time module for more details."""
-
- reps = int(reps)
- assert reps >=1, 'reps must be >= 1'
- if reps==1:
- start = clock()
- out = func(*args,**kw)
- tot_time = clock()-start
- else:
- rng = xrange(reps-1) # the last time is executed separately to store output
- start = clock()
- for dummy in rng: func(*args,**kw)
- out = func(*args,**kw) # one last time
- tot_time = clock()-start
- av_time = tot_time / reps
- return tot_time,av_time,out
-
-
-def timings(reps,func,*args,**kw):
- """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
-
- Execute a function reps times, return a tuple with the elapsed total CPU
- time in seconds and the time per call. These are just the first two values
- in timings_out()."""
-
- return timings_out(reps,func,*args,**kw)[0:2]
-
-
-def timing(func,*args,**kw):
- """timing(func,*args,**kw) -> t_total
-
- Execute a function once, return the elapsed total CPU time in
- seconds. This is just the first value in timings_out()."""
-
- return timings_out(1,func,*args,**kw)[0]
-
+# encoding: utf-8
+"""
+Utilities for timing code execution.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+from .py3compat import xrange
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# If possible (Unix), use the resource module instead of time.clock()
+try:
+ import resource
+ def clocku():
+ """clocku() -> floating point number
+
+ Return the *USER* CPU time in seconds since the start of the process.
+ This is done via a call to resource.getrusage, so it avoids the
+ wraparound problems in time.clock()."""
+
+ return resource.getrusage(resource.RUSAGE_SELF)[0]
+
+ def clocks():
+ """clocks() -> floating point number
+
+ Return the *SYSTEM* CPU time in seconds since the start of the process.
+ This is done via a call to resource.getrusage, so it avoids the
+ wraparound problems in time.clock()."""
+
+ return resource.getrusage(resource.RUSAGE_SELF)[1]
+
+ def clock():
+ """clock() -> floating point number
+
+ Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
+ the process. This is done via a call to resource.getrusage, so it
+ avoids the wraparound problems in time.clock()."""
+
+ u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
+ return u+s
+
+ def clock2():
+ """clock2() -> (t_user,t_system)
+
+ Similar to clock(), but return a tuple of user/system times."""
+ return resource.getrusage(resource.RUSAGE_SELF)[:2]
+except ImportError:
+ # There is no distinction of user/system time under windows, so we just use
+ # time.clock() for everything...
+ clocku = clocks = clock = time.clock
+ def clock2():
+ """Under windows, system CPU time can't be measured.
+
+ This just returns clock() and zero."""
+ return time.clock(),0.0
+
+
+def timings_out(reps,func,*args,**kw):
+ """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
+
+ Execute a function reps times, return a tuple with the elapsed total
+ CPU time in seconds, the time per call and the function's output.
+
+ Under Unix, the return value is the sum of user+system time consumed by
+ the process, computed via the resource module. This prevents problems
+ related to the wraparound effect which the time.clock() function has.
+
+ Under Windows the return value is in wall clock seconds. See the
+ documentation for the time module for more details."""
+
+ reps = int(reps)
+ assert reps >=1, 'reps must be >= 1'
+ if reps==1:
+ start = clock()
+ out = func(*args,**kw)
+ tot_time = clock()-start
+ else:
+ rng = xrange(reps-1) # the last time is executed separately to store output
+ start = clock()
+ for dummy in rng: func(*args,**kw)
+ out = func(*args,**kw) # one last time
+ tot_time = clock()-start
+ av_time = tot_time / reps
+ return tot_time,av_time,out
+
+
+def timings(reps,func,*args,**kw):
+ """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
+
+ Execute a function reps times, return a tuple with the elapsed total CPU
+ time in seconds and the time per call. These are just the first two values
+ in timings_out()."""
+
+ return timings_out(reps,func,*args,**kw)[0:2]
+
+
+def timing(func,*args,**kw):
+ """timing(func,*args,**kw) -> t_total
+
+ Execute a function once, return the elapsed total CPU time in
+ seconds. This is just the first value in timings_out()."""
+
+ return timings_out(1,func,*args,**kw)[0]
+
diff --git a/contrib/python/ipython/py2/IPython/utils/tokenize2.py b/contrib/python/ipython/py2/IPython/utils/tokenize2.py
index cbb5292e5a..7e60a8a629 100644
--- a/contrib/python/ipython/py2/IPython/utils/tokenize2.py
+++ b/contrib/python/ipython/py2/IPython/utils/tokenize2.py
@@ -1,9 +1,9 @@
-"""Load our patched versions of tokenize.
-"""
-
-import sys
-
-if sys.version_info[0] >= 3:
- from ._tokenize_py3 import *
-else:
- from ._tokenize_py2 import *
+"""Load our patched versions of tokenize.
+"""
+
+import sys
+
+if sys.version_info[0] >= 3:
+ from ._tokenize_py3 import *
+else:
+ from ._tokenize_py2 import *
diff --git a/contrib/python/ipython/py2/IPython/utils/tokenutil.py b/contrib/python/ipython/py2/IPython/utils/tokenutil.py
index f52d3b7658..940da98d3d 100644
--- a/contrib/python/ipython/py2/IPython/utils/tokenutil.py
+++ b/contrib/python/ipython/py2/IPython/utils/tokenutil.py
@@ -1,128 +1,128 @@
-"""Token-related utilities"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-from __future__ import absolute_import, print_function
-
-from collections import namedtuple
-from io import StringIO
-from keyword import iskeyword
-
-from . import tokenize2
-from .py3compat import cast_unicode_py2
-
-Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
-
-def generate_tokens(readline):
- """wrap generate_tokens to catch EOF errors"""
- try:
- for token in tokenize2.generate_tokens(readline):
- yield token
- except tokenize2.TokenError:
- # catch EOF error
- return
-
-def line_at_cursor(cell, cursor_pos=0):
- """Return the line in a cell at a given cursor position
-
- Used for calling line-based APIs that don't support multi-line input, yet.
-
- Parameters
- ----------
-
+"""Token-related utilities"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from __future__ import absolute_import, print_function
+
+from collections import namedtuple
+from io import StringIO
+from keyword import iskeyword
+
+from . import tokenize2
+from .py3compat import cast_unicode_py2
+
+Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
+
+def generate_tokens(readline):
+ """wrap generate_tokens to catch EOF errors"""
+ try:
+ for token in tokenize2.generate_tokens(readline):
+ yield token
+ except tokenize2.TokenError:
+ # catch EOF error
+ return
+
+def line_at_cursor(cell, cursor_pos=0):
+ """Return the line in a cell at a given cursor position
+
+ Used for calling line-based APIs that don't support multi-line input, yet.
+
+ Parameters
+ ----------
+
cell: str
- multiline block of text
- cursor_pos: integer
- the cursor position
-
- Returns
- -------
-
- (line, offset): (text, integer)
- The line with the current cursor, and the character offset of the start of the line.
- """
- offset = 0
- lines = cell.splitlines(True)
- for line in lines:
- next_offset = offset + len(line)
- if next_offset >= cursor_pos:
- break
- offset = next_offset
- else:
- line = ""
- return (line, offset)
-
-def token_at_cursor(cell, cursor_pos=0):
- """Get the token at a given cursor
-
- Used for introspection.
-
- Function calls are prioritized, so the token for the callable will be returned
- if the cursor is anywhere inside the call.
-
- Parameters
- ----------
-
- cell : unicode
- A block of Python code
- cursor_pos : int
- The location of the cursor in the block where the token should be found
- """
- cell = cast_unicode_py2(cell)
- names = []
- tokens = []
- call_names = []
-
- offsets = {1: 0} # lines start at 1
- for tup in generate_tokens(StringIO(cell).readline):
-
- tok = Token(*tup)
-
- # token, text, start, end, line = tup
- start_line, start_col = tok.start
- end_line, end_col = tok.end
- if end_line + 1 not in offsets:
- # keep track of offsets for each line
- lines = tok.line.splitlines(True)
- for lineno, line in zip(range(start_line + 1, end_line + 2), lines):
- if lineno not in offsets:
- offsets[lineno] = offsets[lineno-1] + len(line)
-
- offset = offsets[start_line]
- # allow '|foo' to find 'foo' at the beginning of a line
- boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
- if offset + start_col >= boundary:
- # current token starts after the cursor,
- # don't consume it
- break
-
- if tok.token == tokenize2.NAME and not iskeyword(tok.text):
- if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
- names[-1] = "%s.%s" % (names[-1], tok.text)
- else:
- names.append(tok.text)
- elif tok.token == tokenize2.OP:
- if tok.text == '=' and names:
- # don't inspect the lhs of an assignment
- names.pop(-1)
- if tok.text == '(' and names:
- # if we are inside a function call, inspect the function
- call_names.append(names[-1])
- elif tok.text == ')' and call_names:
- call_names.pop(-1)
-
- tokens.append(tok)
-
- if offsets[end_line] + end_col > cursor_pos:
- # we found the cursor, stop reading
- break
-
- if call_names:
- return call_names[-1]
- elif names:
- return names[-1]
- else:
- return ''
-
-
+ multiline block of text
+ cursor_pos: integer
+ the cursor position
+
+ Returns
+ -------
+
+ (line, offset): (text, integer)
+ The line with the current cursor, and the character offset of the start of the line.
+ """
+ offset = 0
+ lines = cell.splitlines(True)
+ for line in lines:
+ next_offset = offset + len(line)
+ if next_offset >= cursor_pos:
+ break
+ offset = next_offset
+ else:
+ line = ""
+ return (line, offset)
+
+def token_at_cursor(cell, cursor_pos=0):
+ """Get the token at a given cursor
+
+ Used for introspection.
+
+ Function calls are prioritized, so the token for the callable will be returned
+ if the cursor is anywhere inside the call.
+
+ Parameters
+ ----------
+
+ cell : unicode
+ A block of Python code
+ cursor_pos : int
+ The location of the cursor in the block where the token should be found
+ """
+ cell = cast_unicode_py2(cell)
+ names = []
+ tokens = []
+ call_names = []
+
+ offsets = {1: 0} # lines start at 1
+ for tup in generate_tokens(StringIO(cell).readline):
+
+ tok = Token(*tup)
+
+ # token, text, start, end, line = tup
+ start_line, start_col = tok.start
+ end_line, end_col = tok.end
+ if end_line + 1 not in offsets:
+ # keep track of offsets for each line
+ lines = tok.line.splitlines(True)
+ for lineno, line in zip(range(start_line + 1, end_line + 2), lines):
+ if lineno not in offsets:
+ offsets[lineno] = offsets[lineno-1] + len(line)
+
+ offset = offsets[start_line]
+ # allow '|foo' to find 'foo' at the beginning of a line
+ boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
+ if offset + start_col >= boundary:
+ # current token starts after the cursor,
+ # don't consume it
+ break
+
+ if tok.token == tokenize2.NAME and not iskeyword(tok.text):
+ if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
+ names[-1] = "%s.%s" % (names[-1], tok.text)
+ else:
+ names.append(tok.text)
+ elif tok.token == tokenize2.OP:
+ if tok.text == '=' and names:
+ # don't inspect the lhs of an assignment
+ names.pop(-1)
+ if tok.text == '(' and names:
+ # if we are inside a function call, inspect the function
+ call_names.append(names[-1])
+ elif tok.text == ')' and call_names:
+ call_names.pop(-1)
+
+ tokens.append(tok)
+
+ if offsets[end_line] + end_col > cursor_pos:
+ # we found the cursor, stop reading
+ break
+
+ if call_names:
+ return call_names[-1]
+ elif names:
+ return names[-1]
+ else:
+ return ''
+
+
diff --git a/contrib/python/ipython/py2/IPython/utils/traitlets.py b/contrib/python/ipython/py2/IPython/utils/traitlets.py
index b4ff7a2689..0ff664fb5c 100644
--- a/contrib/python/ipython/py2/IPython/utils/traitlets.py
+++ b/contrib/python/ipython/py2/IPython/utils/traitlets.py
@@ -1,7 +1,7 @@
-from __future__ import absolute_import
-
-from warnings import warn
-
-warn("IPython.utils.traitlets has moved to a top-level traitlets package.")
-
-from traitlets import *
+from __future__ import absolute_import
+
+from warnings import warn
+
+warn("IPython.utils.traitlets has moved to a top-level traitlets package.")
+
+from traitlets import *
diff --git a/contrib/python/ipython/py2/IPython/utils/tz.py b/contrib/python/ipython/py2/IPython/utils/tz.py
index b315d532d1..14172b2f4a 100644
--- a/contrib/python/ipython/py2/IPython/utils/tz.py
+++ b/contrib/python/ipython/py2/IPython/utils/tz.py
@@ -1,46 +1,46 @@
-# encoding: utf-8
-"""
-Timezone utilities
-
-Just UTC-awareness right now
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2013 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-from datetime import tzinfo, timedelta, datetime
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-# constant for zero offset
-ZERO = timedelta(0)
-
-class tzUTC(tzinfo):
- """tzinfo object for UTC (zero offset)"""
-
- def utcoffset(self, d):
- return ZERO
-
- def dst(self, d):
- return ZERO
-
-UTC = tzUTC()
-
-def utc_aware(unaware):
- """decorator for adding UTC tzinfo to datetime's utcfoo methods"""
- def utc_method(*args, **kwargs):
- dt = unaware(*args, **kwargs)
- return dt.replace(tzinfo=UTC)
- return utc_method
-
-utcfromtimestamp = utc_aware(datetime.utcfromtimestamp)
-utcnow = utc_aware(datetime.utcnow)
+# encoding: utf-8
+"""
+Timezone utilities
+
+Just UTC-awareness right now
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2013 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from datetime import tzinfo, timedelta, datetime
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+# constant for zero offset
+ZERO = timedelta(0)
+
+class tzUTC(tzinfo):
+ """tzinfo object for UTC (zero offset)"""
+
+ def utcoffset(self, d):
+ return ZERO
+
+ def dst(self, d):
+ return ZERO
+
+UTC = tzUTC()
+
+def utc_aware(unaware):
+ """decorator for adding UTC tzinfo to datetime's utcfoo methods"""
+ def utc_method(*args, **kwargs):
+ dt = unaware(*args, **kwargs)
+ return dt.replace(tzinfo=UTC)
+ return utc_method
+
+utcfromtimestamp = utc_aware(datetime.utcfromtimestamp)
+utcnow = utc_aware(datetime.utcnow)
diff --git a/contrib/python/ipython/py2/IPython/utils/ulinecache.py b/contrib/python/ipython/py2/IPython/utils/ulinecache.py
index f53b0dde69..886454c267 100644
--- a/contrib/python/ipython/py2/IPython/utils/ulinecache.py
+++ b/contrib/python/ipython/py2/IPython/utils/ulinecache.py
@@ -1,45 +1,45 @@
-"""Wrapper around linecache which decodes files to unicode according to PEP 263.
-
-This is only needed for Python 2 - linecache in Python 3 does the same thing
-itself.
-"""
-import functools
-import linecache
-import sys
-
-from IPython.utils import py3compat
-from IPython.utils import openpy
-
-if py3compat.PY3:
- getline = linecache.getline
-
- # getlines has to be looked up at runtime, because doctests monkeypatch it.
- @functools.wraps(linecache.getlines)
- def getlines(filename, module_globals=None):
- return linecache.getlines(filename, module_globals=module_globals)
-
-else:
- def getlines(filename, module_globals=None):
- """Get the lines (as unicode) for a file from the cache.
- Update the cache if it doesn't contain an entry for this file already."""
- filename = py3compat.cast_bytes(filename, sys.getfilesystemencoding())
- lines = linecache.getlines(filename, module_globals=module_globals)
-
- # The bits we cache ourselves can be unicode.
- if (not lines) or isinstance(lines[0], py3compat.unicode_type):
- return lines
-
- readline = openpy._list_readline(lines)
- try:
- encoding, _ = openpy.detect_encoding(readline)
- except SyntaxError:
- encoding = 'ascii'
- return [l.decode(encoding, 'replace') for l in lines]
-
- # This is a straight copy of linecache.getline
- def getline(filename, lineno, module_globals=None):
- lines = getlines(filename, module_globals)
- if 1 <= lineno <= len(lines):
- return lines[lineno-1]
- else:
- return ''
+"""Wrapper around linecache which decodes files to unicode according to PEP 263.
+
+This is only needed for Python 2 - linecache in Python 3 does the same thing
+itself.
+"""
+import functools
+import linecache
+import sys
+
+from IPython.utils import py3compat
+from IPython.utils import openpy
+
+if py3compat.PY3:
+ getline = linecache.getline
+
+ # getlines has to be looked up at runtime, because doctests monkeypatch it.
+ @functools.wraps(linecache.getlines)
+ def getlines(filename, module_globals=None):
+ return linecache.getlines(filename, module_globals=module_globals)
+
+else:
+ def getlines(filename, module_globals=None):
+ """Get the lines (as unicode) for a file from the cache.
+ Update the cache if it doesn't contain an entry for this file already."""
+ filename = py3compat.cast_bytes(filename, sys.getfilesystemencoding())
+ lines = linecache.getlines(filename, module_globals=module_globals)
+
+ # The bits we cache ourselves can be unicode.
+ if (not lines) or isinstance(lines[0], py3compat.unicode_type):
+ return lines
+
+ readline = openpy._list_readline(lines)
+ try:
+ encoding, _ = openpy.detect_encoding(readline)
+ except SyntaxError:
+ encoding = 'ascii'
+ return [l.decode(encoding, 'replace') for l in lines]
+
+ # This is a straight copy of linecache.getline
+ def getline(filename, lineno, module_globals=None):
+ lines = getlines(filename, module_globals)
+ if 1 <= lineno <= len(lines):
+ return lines[lineno-1]
+ else:
+ return ''
diff --git a/contrib/python/ipython/py2/IPython/utils/version.py b/contrib/python/ipython/py2/IPython/utils/version.py
index 1de0047e6b..3d1018f7bd 100644
--- a/contrib/python/ipython/py2/IPython/utils/version.py
+++ b/contrib/python/ipython/py2/IPython/utils/version.py
@@ -1,36 +1,36 @@
-# encoding: utf-8
-"""
-Utilities for version comparison
-
-It is a bit ridiculous that we need these.
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (C) 2013 The IPython Development Team
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-from distutils.version import LooseVersion
-
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
-def check_version(v, check):
- """check version string v >= check
-
- If dev/prerelease tags result in TypeError for string-number comparison,
- it is assumed that the dependency is satisfied.
- Users on dev branches are responsible for keeping their own packages up to date.
- """
- try:
- return LooseVersion(v) >= LooseVersion(check)
- except TypeError:
- return True
-
+# encoding: utf-8
+"""
+Utilities for version comparison
+
+It is a bit ridiculous that we need these.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2013 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from distutils.version import LooseVersion
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def check_version(v, check):
+ """check version string v >= check
+
+ If dev/prerelease tags result in TypeError for string-number comparison,
+ it is assumed that the dependency is satisfied.
+ Users on dev branches are responsible for keeping their own packages up to date.
+ """
+ try:
+ return LooseVersion(v) >= LooseVersion(check)
+ except TypeError:
+ return True
+
diff --git a/contrib/python/ipython/py2/IPython/utils/warn.py b/contrib/python/ipython/py2/IPython/utils/warn.py
index dd4852227b..831e4265ac 100644
--- a/contrib/python/ipython/py2/IPython/utils/warn.py
+++ b/contrib/python/ipython/py2/IPython/utils/warn.py
@@ -1,65 +1,65 @@
-# encoding: utf-8
-"""
-Utilities for warnings. Shoudn't we just use the built in warnings module.
-"""
-
-# Copyright (c) IPython Development Team.
-# Distributed under the terms of the Modified BSD License.
-
-from __future__ import print_function
-
-import sys
+# encoding: utf-8
+"""
+Utilities for warnings. Shoudn't we just use the built in warnings module.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from __future__ import print_function
+
+import sys
import warnings
-
+
warnings.warn("The module IPython.utils.warn is deprecated since IPython 4.0, use the standard warnings module instead", DeprecationWarning)
-
-def warn(msg,level=2,exit_val=1):
+
+def warn(msg,level=2,exit_val=1):
"""Deprecated
-
+
Standard warning printer. Gives formatting consistency.
-
+
Output is sent to sys.stderr.
- Options:
-
- -level(2): allows finer control:
- 0 -> Do nothing, dummy function.
- 1 -> Print message.
- 2 -> Print 'WARNING:' + message. (Default level).
- 3 -> Print 'ERROR:' + message.
- 4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
-
- -exit_val (1): exit value returned by sys.exit() for a level 4
- warning. Ignored for all other levels."""
+ Options:
+
+ -level(2): allows finer control:
+ 0 -> Do nothing, dummy function.
+ 1 -> Print message.
+ 2 -> Print 'WARNING:' + message. (Default level).
+ 3 -> Print 'ERROR:' + message.
+ 4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
+
+ -exit_val (1): exit value returned by sys.exit() for a level 4
+ warning. Ignored for all other levels."""
warnings.warn("The module IPython.utils.warn is deprecated since IPython 4.0, use the standard warnings module instead", DeprecationWarning)
- if level>0:
- header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
+ if level>0:
+ header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
print(header[level], msg, sep='', file=sys.stderr)
- if level == 4:
+ if level == 4:
print('Exiting.\n', file=sys.stderr)
- sys.exit(exit_val)
-
-
-def info(msg):
+ sys.exit(exit_val)
+
+
+def info(msg):
"""Deprecated
Equivalent to warn(msg,level=1)."""
-
- warn(msg,level=1)
-
-
-def error(msg):
+
+ warn(msg,level=1)
+
+
+def error(msg):
"""Deprecated
Equivalent to warn(msg,level=3)."""
-
- warn(msg,level=3)
-
-
-def fatal(msg,exit_val=1):
+
+ warn(msg,level=3)
+
+
+def fatal(msg,exit_val=1):
"""Deprecated
Equivalent to warn(msg,exit_val=exit_val,level=4)."""
-
- warn(msg,exit_val=exit_val,level=4)
+
+ warn(msg,exit_val=exit_val,level=4)
diff --git a/contrib/python/ipython/py2/IPython/utils/wildcard.py b/contrib/python/ipython/py2/IPython/utils/wildcard.py
index d22491bd96..f8e895752c 100644
--- a/contrib/python/ipython/py2/IPython/utils/wildcard.py
+++ b/contrib/python/ipython/py2/IPython/utils/wildcard.py
@@ -1,112 +1,112 @@
-# -*- coding: utf-8 -*-
-"""Support for wildcard pattern matching in object inspection.
-
-Authors
--------
-- Jörgen Stenarson <jorgen.stenarson@bostream.nu>
-- Thomas Kluyver
-"""
-
-#*****************************************************************************
-# Copyright (C) 2005 Jörgen Stenarson <jorgen.stenarson@bostream.nu>
-#
-# Distributed under the terms of the BSD License. The full license is in
-# the file COPYING, distributed as part of this software.
-#*****************************************************************************
-
-import re
-import types
-
-from IPython.utils.dir2 import dir2
-from .py3compat import iteritems
-
-def create_typestr2type_dicts(dont_include_in_type2typestr=["lambda"]):
- """Return dictionaries mapping lower case typename (e.g. 'tuple') to type
- objects from the types package, and vice versa."""
- typenamelist = [tname for tname in dir(types) if tname.endswith("Type")]
- typestr2type, type2typestr = {}, {}
-
- for tname in typenamelist:
- name = tname[:-4].lower() # Cut 'Type' off the end of the name
- obj = getattr(types, tname)
- typestr2type[name] = obj
- if name not in dont_include_in_type2typestr:
- type2typestr[obj] = name
- return typestr2type, type2typestr
-
-typestr2type, type2typestr = create_typestr2type_dicts()
-
-def is_type(obj, typestr_or_type):
- """is_type(obj, typestr_or_type) verifies if obj is of a certain type. It
- can take strings or actual python types for the second argument, i.e.
- 'tuple'<->TupleType. 'all' matches all types.
-
- TODO: Should be extended for choosing more than one type."""
- if typestr_or_type == "all":
- return True
- if type(typestr_or_type) == type:
- test_type = typestr_or_type
- else:
- test_type = typestr2type.get(typestr_or_type, False)
- if test_type:
- return isinstance(obj, test_type)
- return False
-
-def show_hidden(str, show_all=False):
- """Return true for strings starting with single _ if show_all is true."""
- return show_all or str.startswith("__") or not str.startswith("_")
-
-def dict_dir(obj):
- """Produce a dictionary of an object's attributes. Builds on dir2 by
- checking that a getattr() call actually succeeds."""
- ns = {}
- for key in dir2(obj):
- # This seemingly unnecessary try/except is actually needed
- # because there is code out there with metaclasses that
- # create 'write only' attributes, where a getattr() call
- # will fail even if the attribute appears listed in the
- # object's dictionary. Properties can actually do the same
- # thing. In particular, Traits use this pattern
- try:
- ns[key] = getattr(obj, key)
- except AttributeError:
- pass
- return ns
-
-def filter_ns(ns, name_pattern="*", type_pattern="all", ignore_case=True,
- show_all=True):
- """Filter a namespace dictionary by name pattern and item type."""
- pattern = name_pattern.replace("*",".*").replace("?",".")
- if ignore_case:
- reg = re.compile(pattern+"$", re.I)
- else:
- reg = re.compile(pattern+"$")
-
- # Check each one matches regex; shouldn't be hidden; of correct type.
- return dict((key,obj) for key, obj in iteritems(ns) if reg.match(key) \
- and show_hidden(key, show_all) \
- and is_type(obj, type_pattern) )
-
-def list_namespace(namespace, type_pattern, filter, ignore_case=False, show_all=False):
- """Return dictionary of all objects in a namespace dictionary that match
- type_pattern and filter."""
- pattern_list=filter.split(".")
- if len(pattern_list) == 1:
- return filter_ns(namespace, name_pattern=pattern_list[0],
- type_pattern=type_pattern,
- ignore_case=ignore_case, show_all=show_all)
- else:
- # This is where we can change if all objects should be searched or
- # only modules. Just change the type_pattern to module to search only
- # modules
- filtered = filter_ns(namespace, name_pattern=pattern_list[0],
- type_pattern="all",
- ignore_case=ignore_case, show_all=show_all)
- results = {}
- for name, obj in iteritems(filtered):
- ns = list_namespace(dict_dir(obj), type_pattern,
- ".".join(pattern_list[1:]),
- ignore_case=ignore_case, show_all=show_all)
- for inner_name, inner_obj in iteritems(ns):
- results["%s.%s"%(name,inner_name)] = inner_obj
- return results
+# -*- coding: utf-8 -*-
+"""Support for wildcard pattern matching in object inspection.
+
+Authors
+-------
+- Jörgen Stenarson <jorgen.stenarson@bostream.nu>
+- Thomas Kluyver
+"""
+
+#*****************************************************************************
+# Copyright (C) 2005 Jörgen Stenarson <jorgen.stenarson@bostream.nu>
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+import re
+import types
+
+from IPython.utils.dir2 import dir2
+from .py3compat import iteritems
+
+def create_typestr2type_dicts(dont_include_in_type2typestr=["lambda"]):
+ """Return dictionaries mapping lower case typename (e.g. 'tuple') to type
+ objects from the types package, and vice versa."""
+ typenamelist = [tname for tname in dir(types) if tname.endswith("Type")]
+ typestr2type, type2typestr = {}, {}
+
+ for tname in typenamelist:
+ name = tname[:-4].lower() # Cut 'Type' off the end of the name
+ obj = getattr(types, tname)
+ typestr2type[name] = obj
+ if name not in dont_include_in_type2typestr:
+ type2typestr[obj] = name
+ return typestr2type, type2typestr
+
+typestr2type, type2typestr = create_typestr2type_dicts()
+
+def is_type(obj, typestr_or_type):
+ """is_type(obj, typestr_or_type) verifies if obj is of a certain type. It
+ can take strings or actual python types for the second argument, i.e.
+ 'tuple'<->TupleType. 'all' matches all types.
+
+ TODO: Should be extended for choosing more than one type."""
+ if typestr_or_type == "all":
+ return True
+ if type(typestr_or_type) == type:
+ test_type = typestr_or_type
+ else:
+ test_type = typestr2type.get(typestr_or_type, False)
+ if test_type:
+ return isinstance(obj, test_type)
+ return False
+
+def show_hidden(str, show_all=False):
+ """Return true for strings starting with single _ if show_all is true."""
+ return show_all or str.startswith("__") or not str.startswith("_")
+
+def dict_dir(obj):
+ """Produce a dictionary of an object's attributes. Builds on dir2 by
+ checking that a getattr() call actually succeeds."""
+ ns = {}
+ for key in dir2(obj):
+ # This seemingly unnecessary try/except is actually needed
+ # because there is code out there with metaclasses that
+ # create 'write only' attributes, where a getattr() call
+ # will fail even if the attribute appears listed in the
+ # object's dictionary. Properties can actually do the same
+ # thing. In particular, Traits use this pattern
+ try:
+ ns[key] = getattr(obj, key)
+ except AttributeError:
+ pass
+ return ns
+
+def filter_ns(ns, name_pattern="*", type_pattern="all", ignore_case=True,
+ show_all=True):
+ """Filter a namespace dictionary by name pattern and item type."""
+ pattern = name_pattern.replace("*",".*").replace("?",".")
+ if ignore_case:
+ reg = re.compile(pattern+"$", re.I)
+ else:
+ reg = re.compile(pattern+"$")
+
+ # Check each one matches regex; shouldn't be hidden; of correct type.
+ return dict((key,obj) for key, obj in iteritems(ns) if reg.match(key) \
+ and show_hidden(key, show_all) \
+ and is_type(obj, type_pattern) )
+
+def list_namespace(namespace, type_pattern, filter, ignore_case=False, show_all=False):
+ """Return dictionary of all objects in a namespace dictionary that match
+ type_pattern and filter."""
+ pattern_list=filter.split(".")
+ if len(pattern_list) == 1:
+ return filter_ns(namespace, name_pattern=pattern_list[0],
+ type_pattern=type_pattern,
+ ignore_case=ignore_case, show_all=show_all)
+ else:
+ # This is where we can change if all objects should be searched or
+ # only modules. Just change the type_pattern to module to search only
+ # modules
+ filtered = filter_ns(namespace, name_pattern=pattern_list[0],
+ type_pattern="all",
+ ignore_case=ignore_case, show_all=show_all)
+ results = {}
+ for name, obj in iteritems(filtered):
+ ns = list_namespace(dict_dir(obj), type_pattern,
+ ".".join(pattern_list[1:]),
+ ignore_case=ignore_case, show_all=show_all)
+ for inner_name, inner_obj in iteritems(ns):
+ results["%s.%s"%(name,inner_name)] = inner_obj
+ return results