Update contrib/python/ipython/py3 to 8.3.0

ref:e84342d4d30476f9148137f37fd0c6405fd36f55

13 files changed, 440 insertions, 312 deletions

diff --git a/contrib/python/ipython/py3/IPython/core/magics/auto.py b/contrib/python/ipython/py3/IPython/core/magics/auto.py
index a18542f43d1..56aa4f72eb3 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/auto.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/auto.py
@@ -104,16 +104,32 @@ class AutoMagics(Magics):
         # all-random (note for auto-testing)
         """
 
+        valid_modes = {
+            0: "Off",
+            1: "Smart",
+            2: "Full",
+        }
+
+        def errorMessage() -> str:
+            error = "Valid modes: "
+            for k, v in valid_modes.items():
+                error += str(k) + "->" + v + ", "
+            error = error[:-2]  # remove tailing `, ` after last element
+            return error
+
         if parameter_s:
+            if not parameter_s in map(str, valid_modes.keys()):
+                error(errorMessage())
+                return
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
-        if not arg in (0, 1, 2, 'toggle'):
-            error('Valid modes: (0->Off, 1->Smart, 2->Full')
+        if not arg in (*list(valid_modes.keys()), "toggle"):
+            error(errorMessage())
             return
 
-        if arg in (0, 1, 2):
+        if arg in (valid_modes.keys()):
             self.shell.autocall = arg
         else: # toggle
             if self.shell.autocall:
@@ -125,4 +141,4 @@ class AutoMagics(Magics):
                 except AttributeError:
                     self.shell.autocall = self._magic_state.autocall_save = 1
 
-        print("Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall])
+        print("Automatic calling is:", list(valid_modes.values())[self.shell.autocall])
diff --git a/contrib/python/ipython/py3/IPython/core/magics/basic.py b/contrib/python/ipython/py3/IPython/core/magics/basic.py
index 72cfc804143..c1f69451100 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/basic.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/basic.py
@@ -4,6 +4,7 @@
 import argparse
 from logging import error
 import io
+import os
 from pprint import pformat
 import sys
 from warnings import warn
@@ -45,7 +46,7 @@ class MagicsDisplay(object):
     
     def _jsonable(self):
         """turn magics dict into jsonable dict of the same structure
-        
+
         replaces object instances with their class names as strings
         """
         magic_dict = {}
@@ -74,6 +75,7 @@ class BasicMagics(Magics):
     These are various magics that don't fit into specific categories but that
     are all part of the base 'IPython experience'."""
 
+    @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-l', '--line', action='store_true',
@@ -122,7 +124,7 @@ class BasicMagics(Magics):
 
           In [6]: %whereami
           Out[6]: u'/home/testuser'
-          
+
           In [7]: %alias_magic h history "-p -l 30" --line
           Created `%h` as an alias for `%history -l 30`.
         """
@@ -366,7 +368,7 @@ Currently the magic system has the following functions:""",
 
         If called without arguments, acts as a toggle.
 
-        When in verbose mode the value --show (and --hide) 
+        When in verbose mode the value --show (and --hide)
         will respectively show (or hide) frames with ``__tracebackhide__ =
         True`` value set.
         """
@@ -560,10 +562,6 @@ Currently the magic system has the following functions:""",
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
-        '-e', '--export', action='store_true', default=False,
-        help=argparse.SUPPRESS
-    )
-    @magic_arguments.argument(
         'filename', type=str,
         help='Notebook name or filename'
     )
@@ -573,11 +571,9 @@ Currently the magic system has the following functions:""",
 
         This function can export the current IPython history to a notebook file.
         For example, to export the history to "foo.ipynb" do "%notebook foo.ipynb".
-
-        The -e or --export flag is deprecated in IPython 5.2, and will be
-        removed in the future.
         """
         args = magic_arguments.parse_argstring(self.notebook, s)
+        outfname = os.path.expanduser(args.filename)
 
         from nbformat import write, v4
 
@@ -591,7 +587,7 @@ Currently the magic system has the following functions:""",
                 source=source
             ))
         nb = v4.new_notebook(cells=cells)
-        with io.open(args.filename, 'w', encoding='utf-8') as f:
+        with io.open(outfname, "w", encoding="utf-8") as f:
             write(nb, f, version=4)
 
 @magics_class
@@ -622,12 +618,11 @@ class AsyncMagics(BasicMagics):
 
         If the passed parameter does not match any of the above and is a python
         identifier, get said object from user namespace and set it as the
-        runner, and activate autoawait. 
+        runner, and activate autoawait.
 
         If the object is a fully qualified object name, attempt to import it and
         set it as the runner, and activate autoawait.
-        
-        
+
         The exact behavior of autoawait is experimental and subject to change
         across version of IPython and Python.
         """
diff --git a/contrib/python/ipython/py3/IPython/core/magics/code.py b/contrib/python/ipython/py3/IPython/core/magics/code.py
index d446d35ac6b..65ba52b8bbf 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/code.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/code.py
@@ -22,6 +22,7 @@ import ast
 from itertools import chain
 from urllib.request import Request, urlopen
 from urllib.parse import urlencode
+from pathlib import Path
 
 # Our own packages
 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
@@ -184,7 +185,7 @@ class CodeMagics(Magics):
         """Save a set of lines or a macro to a given filename.
 
         Usage:\\
-          %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
+          %save [options] filename [history]
 
         Options:
 
@@ -198,9 +199,12 @@ class CodeMagics(Magics):
 
           -a: append to the file instead of overwriting it.
 
-        This function uses the same syntax as %history for input ranges,
+        The history argument uses the same syntax as %history for input ranges,
         then saves the lines to the filename you specify.
 
+        If no ranges are specified, saves history of the current session up to
+        this point.
+
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files.
 
@@ -218,6 +222,7 @@ class CodeMagics(Magics):
         fname, codefrom = args[0], " ".join(args[1:])
         if not fname.endswith(('.py','.ipy')):
             fname += ext
+        fname = os.path.expanduser(fname)
         file_exists = os.path.isfile(fname)
         if file_exists and not force and not append:
             try:
@@ -253,6 +258,9 @@ class CodeMagics(Magics):
         The argument can be an input history range, a filename, or the name of a
         string or macro.
 
+        If no arguments are given, uploads the history of this session up to
+        this point.
+
         Options:
 
           -d: Pass a custom description. The default will say
@@ -314,6 +322,9 @@ class CodeMagics(Magics):
           where source can be a filename, URL, input history range, macro, or
           element in the user namespace
 
+        If no arguments are given, loads the history of this session up to this
+        point.
+
         Options:
 
           -r <lines>: Specify lines or ranges of lines to load from the source.
@@ -332,6 +343,7 @@ class CodeMagics(Magics):
         confirmation before loading source with more than 200 000 characters, unless
         -y flag is passed or if the frontend does not support raw_input::
 
+        %load
         %load myscript.py
         %load 7-27
         %load myMacro
@@ -343,13 +355,7 @@ class CodeMagics(Magics):
         %load -n my_module.wonder_function
         """
         opts,args = self.parse_options(arg_s,'yns:r:')
-
-        if not args:
-            raise UsageError('Missing filename, URL, input history range, '
-                             'macro, or element in the user namespace.')
-
         search_ns = 'n' in opts
-
         contents = self.shell.find_user_code(args, search_ns=search_ns)
 
         if 's' in opts:
@@ -460,10 +466,10 @@ class CodeMagics(Magics):
                         return (None, None, None)
                     use_temp = False
 
-                except DataIsObject:
+                except DataIsObject as e:
                     # macros have a special edit function
                     if isinstance(data, Macro):
-                        raise MacroToEdit(data)
+                        raise MacroToEdit(data) from e
 
                     # For objects, try to edit the file where they are defined
                     filename = find_file(data)
@@ -487,8 +493,8 @@ class CodeMagics(Magics):
                         
                         m = ipython_input_pat.match(os.path.basename(filename))
                         if m:
-                            raise InteractivelyDefined(int(m.groups()[0]))
-                        
+                            raise InteractivelyDefined(int(m.groups()[0])) from e
+
                         datafile = 1
                     if filename is None:
                         filename = make_filename(args)
@@ -532,8 +538,7 @@ class CodeMagics(Magics):
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
-        with open(filename) as mfile:
-            mvalue = mfile.read()
+        mvalue = Path(filename).read_text(encoding="utf-8")
         self.shell.user_ns[mname] = Macro(mvalue)
 
     @skip_doctest
@@ -708,20 +713,22 @@ class CodeMagics(Magics):
         # do actual editing here
         print('Editing...', end=' ')
         sys.stdout.flush()
+        filepath = Path(filename)
         try:
-            # Quote filenames that may have spaces in them
-            if ' ' in filename:
-                filename = "'%s'" % filename
-            self.shell.hooks.editor(filename,lineno)
+            # Quote filenames that may have spaces in them when opening
+            # the editor
+            quoted = filename = str(filepath.absolute())
+            if " " in quoted:
+                quoted = "'%s'" % quoted
+            self.shell.hooks.editor(quoted, lineno)
         except TryNext:
             warn('Could not open editor')
             return
 
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
-        if args.strip() == 'pasted_block':
-            with open(filename, 'r') as f:
-                self.shell.user_ns['pasted_block'] = f.read()
+        if args.strip() == "pasted_block":
+            self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8")
 
         if 'x' in opts:  # -x prevents actual execution
             print()
@@ -729,10 +736,9 @@ class CodeMagics(Magics):
             print('done. Executing edited code...')
             with preserve_keys(self.shell.user_ns, '__file__'):
                 if not is_temp:
-                    self.shell.user_ns['__file__'] = filename
-                if 'r' in opts:    # Untranslated IPython code
-                    with open(filename, 'r') as f:
-                        source = f.read()
+                    self.shell.user_ns["__file__"] = filename
+                if "r" in opts:  # Untranslated IPython code
+                    source = filepath.read_text(encoding="utf-8")
                     self.shell.run_cell(source, store_history=False)
                 else:
                     self.shell.safe_execfile(filename, self.shell.user_ns,
@@ -740,10 +746,9 @@ class CodeMagics(Magics):
 
         if is_temp:
             try:
-                with open(filename) as f:
-                    return f.read()
+                return filepath.read_text(encoding="utf-8")
             except IOError as msg:
-                if msg.filename == filename:
+                if Path(msg.filename) == filepath:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
diff --git a/contrib/python/ipython/py3/IPython/core/magics/config.py b/contrib/python/ipython/py3/IPython/core/magics/config.py
index 97b13df02e6..c1387b601b8 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/config.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/config.py
@@ -54,44 +54,73 @@ class ConfigMagics(Magics):
 
             In [1]: %config
             Available objects for config:
-                TerminalInteractiveShell
-                HistoryManager
-                PrefilterManager
                 AliasManager
-                IPCompleter
                 DisplayFormatter
+                HistoryManager
+                IPCompleter
+                LoggingMagics
+                MagicsManager
+                OSMagics
+                PrefilterManager
+                ScriptMagics
+                TerminalInteractiveShell
 
         To view what is configurable on a given class, just pass the class
         name::
 
             In [2]: %config IPCompleter
-            IPCompleter options
-            -----------------
-            IPCompleter.omit__names=<Enum>
-                Current: 2
-                Choices: (0, 1, 2)
-                Instruct the completer to omit private method names
-                Specifically, when completing on ``object.<tab>``.
-                When 2 [default]: all names that start with '_' will be excluded.
-                When 1: all 'magic' names (``__foo__``) will be excluded.
-                When 0: nothing will be excluded.
-            IPCompleter.merge_completions=<CBool>
+            IPCompleter(Completer) options
+            ----------------------------
+            IPCompleter.backslash_combining_completions=<Bool>
+                Enable unicode completions, e.g. \\alpha<tab> . Includes completion of latex
+                commands, unicode names, and expanding unicode characters back to latex
+                commands.
                 Current: True
-                Whether to merge completion results into a single list
-                If False, only the completion results from the first non-empty
-                completer will be returned.
-            IPCompleter.limit_to__all__=<CBool>
+            IPCompleter.debug=<Bool>
+                Enable debug for the Completer. Mostly print extra information for
+                experimental jedi integration.
+                Current: False
+            IPCompleter.greedy=<Bool>
+                Activate greedy completion
+                        PENDING DEPRECATION. this is now mostly taken care of with Jedi.
+                        This will enable completion on elements of lists, results of function calls, etc.,
+                        but can be unsafe because the code is actually evaluated on TAB.
                 Current: False
+            IPCompleter.jedi_compute_type_timeout=<Int>
+                Experimental: restrict time (in milliseconds) during which Jedi can compute types.
+                        Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
+                        performance by preventing jedi to build its cache.
+                Current: 400
+            IPCompleter.limit_to__all__=<Bool>
+                DEPRECATED as of version 5.0.
                 Instruct the completer to use __all__ for the completion
                 Specifically, when completing on ``object.<tab>``.
                 When True: only those names in obj.__all__ will be included.
                 When False [default]: the __all__ attribute is ignored
-            IPCompleter.greedy=<CBool>
                 Current: False
-                Activate greedy completion
-                This will enable completion on elements of lists, results of
-                function calls, etc., but can be unsafe because the code is
-                actually evaluated on TAB.
+            IPCompleter.merge_completions=<Bool>
+                Whether to merge completion results into a single list
+                        If False, only the completion results from the first non-empty
+                        completer will be returned.
+                Current: True
+            IPCompleter.omit__names=<Enum>
+                Instruct the completer to omit private method names
+                        Specifically, when completing on ``object.<tab>``.
+                        When 2 [default]: all names that start with '_' will be excluded.
+                        When 1: all 'magic' names (``__foo__``) will be excluded.
+                        When 0: nothing will be excluded.
+                Choices: any of [0, 1, 2]
+                Current: 2
+            IPCompleter.profile_completions=<Bool>
+                If True, emit profiling data for completion subsystem using cProfile.
+                Current: False
+            IPCompleter.profiler_output_dir=<Unicode>
+                Template for path at which to output profile data for completions.
+                Current: '.completion_profiles'
+            IPCompleter.use_jedi=<Bool>
+                Experimental: Use Jedi to generate autocompletions. Default to True if jedi
+                is installed.
+                Current: True
 
         but the real use is in setting values::
 
@@ -118,7 +147,7 @@ class ConfigMagics(Magics):
             # print available configurable names
             print("Available objects for config:")
             for name in classnames:
-                print("    ", name)
+                print("   ", name)
             return
         elif line in classnames:
             # `%config TerminalInteractiveShell` will print trait info for
@@ -149,7 +178,7 @@ class ConfigMagics(Magics):
         # leave quotes on args when splitting, because we want
         # unquoted args to eval in user_ns
         cfg = Config()
-        exec("cfg."+line, locals(), self.shell.user_ns)
+        exec("cfg."+line, self.shell.user_ns, locals())
 
         for configurable in configurables:
             try:
diff --git a/contrib/python/ipython/py3/IPython/core/magics/display.py b/contrib/python/ipython/py3/IPython/core/magics/display.py
index 07853944715..6c0eff6884f 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/display.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/display.py
@@ -12,7 +12,7 @@
 #-----------------------------------------------------------------------------
 
 # Our own packages
-from IPython.core.display import display, Javascript, Latex, SVG, HTML, Markdown
+from IPython.display import display, Javascript, Latex, SVG, HTML, Markdown
 from IPython.core.magic import  (
     Magics, magics_class, cell_magic
 )
@@ -36,22 +36,33 @@ class DisplayMagics(Magics):
         """Run the cell block of Javascript code
 
         Alias of `%%javascript`
+
+        Starting with IPython 8.0 %%javascript is pending deprecation to be replaced
+        by a more flexible system
+
+        Please See https://github.com/ipython/ipython/issues/13376
         """
         self.javascript(line, cell)
 
     @cell_magic
     def javascript(self, line, cell):
-        """Run the cell block of Javascript code"""
+        """Run the cell block of Javascript code
+
+        Starting with IPython 8.0 %%javascript is pending deprecation to be replaced
+        by a more flexible system
+
+        Please See https://github.com/ipython/ipython/issues/13376
+        """
         display(Javascript(cell))
 
 
     @cell_magic
     def latex(self, line, cell):
-        """Render the cell as a block of latex
+        """Render the cell as a block of LaTeX
 
-        The subset of latex which is support depends on the implementation in
+        The subset of LaTeX which is supported depends on the implementation in
         the client.  In the Jupyter Notebook, this magic only renders the subset
-        of latex defined by MathJax
+        of LaTeX defined by MathJax
         [here](https://docs.mathjax.org/en/v2.5-latest/tex.html)."""
         display(Latex(cell))
 
diff --git a/contrib/python/ipython/py3/IPython/core/magics/execution.py b/contrib/python/ipython/py3/IPython/core/magics/execution.py
index 6b651939f8f..da7f780b9cb 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/execution.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/execution.py
@@ -8,55 +8,44 @@
 import ast
 import bdb
 import builtins as builtin_mod
+import cProfile as profile
 import gc
 import itertools
+import math
 import os
+import pstats
+import re
 import shlex
 import sys
 import time
 import timeit
-import math
-import re
+from ast import Module
+from io import StringIO
+from logging import error
+from pathlib import Path
 from pdb import Restart
+from warnings import warn
 
-# cProfile was added in Python2.5
-try:
-    import cProfile as profile
-    import pstats
-except ImportError:
-    # profile isn't bundled by default in Debian for license reasons
-    try:
-        import profile, pstats
-    except ImportError:
-        profile = pstats = None
-
-from IPython.core import oinspect
-from IPython.core import magic_arguments
-from IPython.core import page
+from IPython.core import magic_arguments, oinspect, page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
-from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
-                                line_cell_magic, on_off, needs_local_scope,
-                                no_var_expand)
+from IPython.core.magic import (
+    Magics,
+    cell_magic,
+    line_cell_magic,
+    line_magic,
+    magics_class,
+    needs_local_scope,
+    no_var_expand,
+    on_off,
+)
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils.contexts import preserve_keys
 from IPython.utils.capture import capture_output
+from IPython.utils.contexts import preserve_keys
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
-from warnings import warn
-from logging import error
-from io import StringIO
-
-if sys.version_info > (3,8):
-    from ast import Module
-else :
-    # mock the new API, ignore second argument
-    # see https://github.com/ipython/ipython/issues/11590
-    from ast import Module as OriginalModule
-    Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
-
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
@@ -103,17 +92,15 @@ class TimeitResult(object):
                 pm = u'\xb1'
             except:
                 pass
-        return (
-            u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
-                .format(
-                    pm = pm,
-                    runs = self.repeat,
-                    loops = self.loops,
-                    loop_plural = "" if self.loops == 1 else "s",
-                    run_plural = "" if self.repeat == 1 else "s",
-                    mean = _format_time(self.average, self._precision),
-                    std = _format_time(self.stdev, self._precision))
-                )
+        return "{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
+            pm=pm,
+            runs=self.repeat,
+            loops=self.loops,
+            loop_plural="" if self.loops == 1 else "s",
+            run_plural="" if self.repeat == 1 else "s",
+            mean=_format_time(self.average, self._precision),
+            std=_format_time(self.stdev, self._precision),
+        )
 
     def _repr_pretty_(self, p , cycle):
         unic = self.__str__()
@@ -181,17 +168,9 @@ class ExecutionMagics(Magics):
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
-        if profile is None:
-            self.prun = self.profile_missing_notice
         # Default execution function used to actually run user code.
         self.default_runner = None
 
-    def profile_missing_notice(self, *args, **kwargs):
-        error("""\
-The profile module could not be found. It has been removed from the standard
-python packages because of its non-free license. To use profiling, install the
-python-profiler package from non-free.""")
-
     @skip_doctest
     @no_var_expand
     @line_cell_magic
@@ -375,18 +354,22 @@ python-profiler package from non-free.""")
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
-            print('\n*** Profile stats marshalled to file',\
-                  repr(dump_file)+'.',sys_exit)
+            print(
+                f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}"
+            )
         if text_file:
-            with open(text_file, 'w') as pfile:
-                pfile.write(output)
-            print('\n*** Profile printout saved to text file',\
-                  repr(text_file)+'.',sys_exit)
+            pfile = Path(text_file)
+            pfile.touch(exist_ok=True)
+            pfile.write_text(output, encoding="utf-8")
+
+            print(
+                f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}"
+            )
 
         if 'r' in opts:
             return stats
-        else:
-            return None
+
+        return None
 
     @line_magic
     def pdb(self, parameter_s=''):
@@ -423,7 +406,6 @@ python-profiler package from non-free.""")
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
-    @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
@@ -529,7 +511,7 @@ python-profiler package from non-free.""")
         """Run the named file inside IPython as a program.
 
         Usage::
-        
+
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | filename ) [args]
@@ -570,7 +552,7 @@ python-profiler package from non-free.""")
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
-        On Windows systems, the use of single quotes `'` when specifying 
+        On Windows systems, the use of single quotes `'` when specifying
         a file is not supported. Use double quotes `"`.
 
         Options:
@@ -712,9 +694,9 @@ python-profiler package from non-free.""")
             fpath = None # initialize to make sure fpath is in scope later
             fpath = arg_lst[0]
             filename = file_finder(fpath)
-        except IndexError:
+        except IndexError as e:
             msg = 'you must provide at least a filename.'
-            raise Exception(msg)
+            raise Exception(msg) from e
         except IOError as e:
             try:
                 msg = str(e)
@@ -722,7 +704,7 @@ python-profiler package from non-free.""")
                 msg = e.message
             if os.name == 'nt' and re.match(r"^'.*'$",fpath):
                 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
-            raise Exception(msg)
+            raise Exception(msg) from e
         except TypeError:
             if fpath in sys.meta_path:
                 filename = ""
@@ -751,7 +733,7 @@ python-profiler package from non-free.""")
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
-            name = os.path.splitext(os.path.basename(filename))[0]
+            name = Path(filename).stem
         else:
             name = '__main__'
 
@@ -1085,7 +1067,6 @@ python-profiler package from non-free.""")
 
           In [6]: %timeit -n1 time.sleep(2)
 
-
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
@@ -1094,8 +1075,9 @@ python-profiler package from non-free.""")
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
-        opts, stmt = self.parse_options(line,'n:r:tcp:qo',
-                                        posix=False, strict=False)
+        opts, stmt = self.parse_options(
+            line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True
+        )
         if stmt == "" and cell is None:
             return
         
@@ -1218,7 +1200,7 @@ python-profiler package from non-free.""")
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
-        
+
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
@@ -1255,7 +1237,6 @@ python-profiler package from non-free.""")
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
-
         .. note::
             The time needed by Python to compile the given expression will be
             reported if it is more than 0.1s.
@@ -1345,19 +1326,22 @@ python-profiler package from non-free.""")
 
         wall_end = wtime()
         # Compute actual times and report
-        wall_time = wall_end-wall_st
-        cpu_user = end[0]-st[0]
-        cpu_sys = end[1]-st[1]
-        cpu_tot = cpu_user+cpu_sys
-        # On windows cpu_sys is always zero, so no new information to the next print 
-        if sys.platform != 'win32':
-            print("CPU times: user %s, sys: %s, total: %s" % \
-                (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
-        print("Wall time: %s" % _format_time(wall_time))
+        wall_time = wall_end - wall_st
+        cpu_user = end[0] - st[0]
+        cpu_sys = end[1] - st[1]
+        cpu_tot = cpu_user + cpu_sys
+        # On windows cpu_sys is always zero, so only total is displayed
+        if sys.platform != "win32":
+            print(
+                f"CPU times: user {_format_time(cpu_user)}, sys: {_format_time(cpu_sys)}, total: {_format_time(cpu_tot)}"
+            )
+        else:
+            print(f"CPU times: total: {_format_time(cpu_tot)}")
+        print(f"Wall time: {_format_time(wall_time)}")
         if tc > tc_min:
-            print("Compiler : %s" % _format_time(tc))
+            print(f"Compiler : {_format_time(tc)}")
         if tp > tp_min:
-            print("Parser   : %s" % _format_time(tp))
+            print(f"Parser   : {_format_time(tp)}")
         return out
 
     @skip_doctest
diff --git a/contrib/python/ipython/py3/IPython/core/magics/extension.py b/contrib/python/ipython/py3/IPython/core/magics/extension.py
index ba93b3be754..2bc76b2d552 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/extension.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/extension.py
@@ -41,7 +41,7 @@ class ExtensionMagics(Magics):
     @line_magic
     def unload_ext(self, module_str):
         """Unload an IPython extension by its module name.
-        
+
         Not all extensions can be unloaded, only those which define an
         ``unload_ipython_extension`` function.
         """
diff --git a/contrib/python/ipython/py3/IPython/core/magics/history.py b/contrib/python/ipython/py3/IPython/core/magics/history.py
index 5af09e5ce10..faa4335faa8 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/history.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/history.py
@@ -16,6 +16,7 @@
 import os
 import sys
 from io import open as io_open
+import fnmatch
 
 # Our own packages
 from IPython.core.error import StdinNotImplementedError
@@ -104,7 +105,7 @@ class HistoryMagics(Magics):
 
         By default, all input history from the current session is displayed.
         Ranges of history can be indicated using the syntax:
-        
+
         ``4``
             Line 4, current session
         ``4-6``
@@ -116,7 +117,7 @@ class HistoryMagics(Magics):
         ``~8/1-~6/5``
             From the first line of 8 sessions ago, to the fifth line of 6
             sessions ago.
-        
+
         Multiple ranges can be entered, separated by spaces
 
         The same syntax is used by %macro, %save, %edit, %rerun
@@ -150,6 +151,7 @@ class HistoryMagics(Magics):
             # We don't want to close stdout at the end!
             close_at_end = False
         else:
+            outfname = os.path.expanduser(outfname)
             if os.path.exists(outfname):
                 try:
                     ans = io.ask_yes_no("File %r exists. Overwrite?" % outfname)
@@ -170,7 +172,8 @@ class HistoryMagics(Magics):
         pattern = None
         limit = None if args.limit is _unspecified else args.limit
 
-        if args.pattern is not None:
+        range_pattern = False
+        if args.pattern is not None and not args.range:
             if args.pattern:
                 pattern = "*" + " ".join(args.pattern) + "*"
             else:
@@ -182,11 +185,12 @@ class HistoryMagics(Magics):
             n = 10 if limit is None else limit
             hist = history_manager.get_tail(n, raw=raw, output=get_output)
         else:
-            if args.range:      # Get history by ranges
-                hist = history_manager.get_range_by_str(" ".join(args.range),
-                                                        raw, get_output)
-            else:               # Just get history for the current session
-                hist = history_manager.get_range(raw=raw, output=get_output)
+            if args.pattern:
+                range_pattern = "*" + " ".join(args.pattern) + "*"
+                print_nums = True
+            hist = history_manager.get_range_by_str(
+                " ".join(args.range), raw, get_output
+            )
 
         # We could be displaying the entire history, so let's not try to pull
         # it into a list in memory. Anything that needs more space will just
@@ -200,6 +204,9 @@ class HistoryMagics(Magics):
             # into an editor.
             if get_output:
                 inline, output = inline
+            if range_pattern:
+                if not fnmatch.fnmatch(inline, range_pattern):
+                    continue
             inline = inline.expandtabs(4).rstrip()
 
             multiline = "\n" in inline
@@ -293,7 +300,19 @@ class HistoryMagics(Magics):
         """
         opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
         if "l" in opts:         # Last n lines
-            n = int(opts['l'])
+            try:
+                n = int(opts["l"])
+            except ValueError:
+                print("Number of lines must be an integer")
+                return
+
+            if n == 0:
+                print("Requested 0 last lines - nothing to run")
+                return
+            elif n < 0:
+                print("Number of lines to rerun cannot be negative")
+                return
+
             hist = self.shell.history_manager.get_tail(n)
         elif "g" in opts:       # Search
             p = "*"+opts['g']+"*"
diff --git a/contrib/python/ipython/py3/IPython/core/magics/namespace.py b/contrib/python/ipython/py3/IPython/core/magics/namespace.py
index 5cc2d81ca2a..c86d3de9b65 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/namespace.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/namespace.py
@@ -173,7 +173,7 @@ class NamespaceMagics(Magics):
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
-          
+
           -l: List all available object types for object matching. This function
           can be used without arguments.
 
@@ -203,9 +203,9 @@ class NamespaceMagics(Magics):
         Show objects beginning with a single _::
 
           %psearch -a _*         list objects beginning with a single underscore
-          
+
         List available objects::
-        
+
           %psearch -l            list all available object types
         """
         # default namespaces to be searched
@@ -252,7 +252,6 @@ class NamespaceMagics(Magics):
 
         Examples
         --------
-
         Define two variables and list them with who_ls::
 
           In [1]: alpha = 123
@@ -367,7 +366,6 @@ class NamespaceMagics(Magics):
 
         Examples
         --------
-
         Define two variables and list them with whos::
 
           In [1]: alpha = 123
@@ -484,24 +482,26 @@ class NamespaceMagics(Magics):
 
         Parameters
         ----------
-        -f : force reset without asking for confirmation.
-
-        -s : 'Soft' reset: Only clears your namespace, leaving history intact.
+        -f
+            force reset without asking for confirmation.
+        -s
+            'Soft' reset: Only clears your namespace, leaving history intact.
             References to objects may be kept. By default (without this option),
             we do a 'hard' reset, giving you a new session and removing all
             references to objects from the current session.
-
-        --aggressive: Try to aggressively remove modules from sys.modules ; this
+        --aggressive
+            Try to aggressively remove modules from sys.modules ; this
             may allow you to reimport Python modules that have been updated and
             pick up changes, but can have unattended consequences.
 
-        in : reset input history
-
-        out : reset output history
-
-        dhist : reset directory history
-
-        array : reset only variables that are NumPy arrays
+        in
+            reset input history
+        out
+            reset output history
+        dhist
+            reset directory history
+        array
+            reset only variables that are NumPy arrays
 
         See Also
         --------
@@ -624,7 +624,6 @@ class NamespaceMagics(Magics):
 
         Examples
         --------
-
         We first fully reset the namespace so your output looks identical to
         this example for pedagogical reasons; in practice you do not need a
         full reset::
@@ -687,8 +686,8 @@ class NamespaceMagics(Magics):
         else:
             try:
                 m = re.compile(regex)
-            except TypeError:
-                raise TypeError('regex must be a string or compiled pattern')
+            except TypeError as e:
+                raise TypeError('regex must be a string or compiled pattern') from e
             for i in self.who_ls():
                 if m.search(i):
                     del(user_ns[i])
diff --git a/contrib/python/ipython/py3/IPython/core/magics/osm.py b/contrib/python/ipython/py3/IPython/core/magics/osm.py
index 90da7e22803..41957a28509 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/osm.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/osm.py
@@ -63,10 +63,9 @@ class OSMagics(Magics):
         super().__init__(shell=shell, **kwargs)
 
 
-    @skip_doctest
     def _isexec_POSIX(self, file):
         """
-            Test for executable on a POSIX system
+        Test for executable on a POSIX system
         """
         if os.access(file.path, os.X_OK):
             # will fail on maxOS if access is not X_OK
@@ -75,17 +74,15 @@ class OSMagics(Magics):
 
 
     
-    @skip_doctest
     def _isexec_WIN(self, file):
         """
-            Test for executable file on non POSIX system
+        Test for executable file on non POSIX system
         """
         return file.is_file() and self.execre.match(file.name) is not None
 
-    @skip_doctest
     def isexec(self, file):
         """
-            Test for executable file on non POSIX system
+        Test for executable file on non POSIX system
         """
         if self.is_posix:
             return self._isexec_POSIX(file)
@@ -130,7 +127,7 @@ class OSMagics(Magics):
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
-        http://www.python.org/peps/pep-0215.html.  This is the library used by
+        https://peps.python.org/pep-0215/.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
@@ -293,8 +290,8 @@ class OSMagics(Magics):
         """
         try:
             return os.getcwd()
-        except FileNotFoundError:
-            raise UsageError("CWD no longer exists - please use %cd to change directory.")
+        except FileNotFoundError as e:
+            raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
 
     @skip_doctest
     @line_magic
@@ -302,33 +299,34 @@ class OSMagics(Magics):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
-        you visit during your IPython session, in the variable _dh. The
-        command %dhist shows this history nicely formatted. You can also
-        do 'cd -<tab>' to see directory history conveniently.
-
+        you visit during your IPython session, in the variable ``_dh``. The
+        command :magic:`%dhist` shows this history nicely formatted. You can
+        also do ``cd -<tab>`` to see directory history conveniently.
         Usage:
 
-          cd 'dir': changes to directory 'dir'.
-
-          cd -: changes to the last visited directory.
-
-          cd -<n>: changes to the n-th directory in the directory history.
-
-          cd --foo: change to directory that matches 'foo' in history
+          - ``cd 'dir'``: changes to directory 'dir'.
+          - ``cd -``: changes to the last visited directory.
+          - ``cd -<n>``: changes to the n-th directory in the directory history.
+          - ``cd --foo``: change to directory that matches 'foo' in history
+          - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
+          - Hitting a tab key after ``cd -b`` allows you to tab-complete
+            bookmark names.
 
-          cd -b <bookmark_name>: jump to a bookmark set by %bookmark
-             (note: cd <bookmark_name> is enough if there is no
-              directory <bookmark_name>, but a bookmark with the name exists.)
-              'cd -b <tab>' allows you to tab-complete bookmark names.
+          .. note::
+            ``cd <bookmark_name>`` is enough if there is no directory
+            ``<bookmark_name>``, but a bookmark with the name exists.
 
         Options:
 
-        -q: quiet.  Do not print the working directory after the cd command is
-        executed.  By default IPython's cd command does print this directory,
-        since the default prompts do not display path information.
+        -q               Be quiet. Do not print the working directory after the
+                         cd command is executed. By default IPython's cd
+                         command does print this directory, since the default
+                         prompts do not display path information.
 
-        Note that !cd doesn't work for this purpose because the shell where
-        !command runs is immediately discarded after executing 'command'.
+        .. note::
+           Note that ``!cd`` doesn't work for this purpose because the shell
+           where ``!command`` runs is immediately discarded after executing
+           'command'.
 
         Examples
         --------
@@ -386,8 +384,8 @@ class OSMagics(Magics):
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
-            except IndexError:
-                raise UsageError('%cd -: No previous directory to change to.')
+            except IndexError as e:
+                raise UsageError('%cd -: No previous directory to change to.') from e
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
@@ -436,11 +434,11 @@ class OSMagics(Magics):
 
         Usage:\\
 
-          %env: lists all environment variables/values
-          %env var: get value for var
-          %env var val: set value for var
-          %env var=val: set value for var
-          %env var=$val: set value for var, using python expansion if possible
+          :``%env``: lists all environment variables/values
+          :``%env var``: get value for var
+          :``%env var val``: set value for var
+          :``%env var=val``: set value for var
+          :``%env var=$val``: set value for var, using python expansion if possible
         """
         if parameter_s.strip():
             split = '=' if '=' in parameter_s else ' '
@@ -506,7 +504,7 @@ class OSMagics(Magics):
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
-        return self.shell.magic('dirs')
+        return self.shell.run_line_magic('dirs', '')
 
     @line_magic
     def popd(self, parameter_s=''):
@@ -630,8 +628,8 @@ class OSMagics(Magics):
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
-              ...:      !wc -l $f
-              ...:
+               ...:      !wc -l $f
+               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
@@ -764,15 +762,15 @@ class OSMagics(Magics):
         if 'd' in opts:
             try:
                 todel = args[0]
-            except IndexError:
+            except IndexError as e:
                 raise UsageError(
-                    "%bookmark -d: must provide a bookmark to delete")
+                    "%bookmark -d: must provide a bookmark to delete") from e
             else:
                 try:
                     del bkms[todel]
-                except KeyError:
+                except KeyError as e:
                     raise UsageError(
-                        "%%bookmark -d: Can't delete bookmark '%s'" % todel)
+                        "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
 
         elif 'r' in opts:
             bkms = {}
@@ -803,18 +801,17 @@ class OSMagics(Magics):
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
-        an history range (see %history) or a macro as argument ::
+        an history range (see %history) or a macro as argument.
+
+        If no parameter is given, prints out history of current session up to
+        this point. ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
-        if not parameter_s:
-            raise UsageError('Missing filename, URL, input history range, '
-                             'or macro.')
-
-        try :
+        try:
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
@@ -835,7 +832,7 @@ class OSMagics(Magics):
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
-        
+
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
diff --git a/contrib/python/ipython/py3/IPython/core/magics/packaging.py b/contrib/python/ipython/py3/IPython/core/magics/packaging.py
index 04bde051ae0..2f7652c169b 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/packaging.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/packaging.py
@@ -8,10 +8,10 @@
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import os
 import re
 import shlex
 import sys
+from pathlib import Path
 
 from IPython.core.magic import Magics, magics_class, line_magic
 
@@ -19,27 +19,28 @@ from IPython.core.magic import Magics, magics_class, line_magic
 def _is_conda_environment():
     """Return True if the current Python executable is in a conda env"""
     # TODO: does this need to change on windows?
-    conda_history = os.path.join(sys.prefix, 'conda-meta', 'history')
-    return os.path.exists(conda_history)
+    return Path(sys.prefix, "conda-meta", "history").exists()
 
 
 def _get_conda_executable():
     """Find the path to the conda executable"""
     # Check if there is a conda executable in the same directory as the Python executable.
     # This is the case within conda's root environment.
-    conda = os.path.join(os.path.dirname(sys.executable), 'conda')
-    if os.path.isfile(conda):
-        return conda
+    conda = Path(sys.executable).parent / "conda"
+    if conda.is_file():
+        return str(conda)
 
     # Otherwise, attempt to extract the executable from conda history.
     # This applies in any conda environment.
-    R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]")
-    with open(os.path.join(sys.prefix, 'conda-meta', 'history')) as f:
-        for line in f:
-            match = R.match(line)
-            if match:
-                return match.groupdict()['command']
-    
+    history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8")
+    match = re.search(
+        r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]",
+        history,
+        flags=re.MULTILINE,
+    )
+    if match:
+        return match.groupdict()["command"]
+
     # Fallback: assume conda is available on the system path.
     return "conda"
 
@@ -78,18 +79,19 @@ class PackagingMagics(Magics):
     @line_magic
     def conda(self, line):
         """Run the conda package manager within the current kernel.
-        
+
         Usage:
           %conda install [pkgs]
         """
         if not _is_conda_environment():
             raise ValueError("The python kernel does not appear to be a conda environment.  "
                              "Please use ``%pip install`` instead.")
-        
+
         conda = _get_conda_executable()
         args = shlex.split(line)
-        command = args[0]
-        args = args[1:]
+        command = args[0] if len(args) > 0 else ""
+        args = args[1:] if len(args) > 1 else [""]
+
         extra_args = []
 
         # When the subprocess does not allow us to respond "yes" during the installation,
diff --git a/contrib/python/ipython/py3/IPython/core/magics/pylab.py b/contrib/python/ipython/py3/IPython/core/magics/pylab.py
index 9ec441a3e2c..0f3fff62faf 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/pylab.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/pylab.py
@@ -154,6 +154,9 @@ class PylabMagics(Magics):
 
         gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all)
         self._show_matplotlib_backend(args.gui, backend)
+        print(
+            "%pylab is deprecated, use %matplotlib inline and import the required libraries."
+        )
         print("Populating the interactive namespace from numpy and matplotlib")
         if clobbered:
             warn("pylab import has clobbered these variables: %s"  % clobbered +
diff --git a/contrib/python/ipython/py3/IPython/core/magics/script.py b/contrib/python/ipython/py3/IPython/core/magics/script.py
index 8b7f6f94e09..9fd2fc6c0dd 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/script.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/script.py
@@ -3,22 +3,22 @@
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import asyncio
+import atexit
 import errno
 import os
-import sys
 import signal
+import sys
 import time
-from subprocess import Popen, PIPE, CalledProcessError
-import atexit
+from subprocess import CalledProcessError
+from threading import Thread
+
+from traitlets import Any, Dict, List, default
 
 from IPython.core import magic_arguments
-from IPython.core.magic import  (
-    Magics, magics_class, line_magic, cell_magic
-)
-from IPython.lib.backgroundjobs import BackgroundJobManager
-from IPython.utils import py3compat
+from IPython.core.async_helpers import _AsyncIOProxy
+from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
 from IPython.utils.process import arg_split
-from traitlets import List, Dict, default
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
@@ -56,15 +56,16 @@ def script_args(f):
         ),
         magic_arguments.argument(
             '--no-raise-error', action="store_false", dest='raise_error',
-            help="""Whether you should raise an error message in addition to 
+            help="""Whether you should raise an error message in addition to
             a stream on stderr if you get a nonzero exit code.
-            """
-        )
+            """,
+        ),
     ]
     for arg in args:
         f = arg(f)
     return f
 
+
 @magics_class
 class ScriptMagics(Magics):
     """Magics for talking to scripts
@@ -73,6 +74,17 @@ class ScriptMagics(Magics):
     with a program in a subprocess, and registers a few top-level
     magics that call %%script with common interpreters.
     """
+
+    event_loop = Any(
+        help="""
+        The event loop on which to run subprocesses
+
+        Not the main event loop,
+        because we want to be able to make blocking calls
+        and have certain requirements we don't want to impose on the main loop.
+        """
+    )
+
     script_magics = List(
         help="""Extra script cell magics to define
         
@@ -114,7 +126,6 @@ class ScriptMagics(Magics):
     def __init__(self, shell=None):
         super(ScriptMagics, self).__init__(shell=shell)
         self._generate_script_magics()
-        self.job_manager = BackgroundJobManager()
         self.bg_processes = []
         atexit.register(self.kill_bg_processes)
 
@@ -136,7 +147,7 @@ class ScriptMagics(Magics):
         def named_script_magic(line, cell):
             # if line, add it as cl-flags
             if line:
-                 line = "%s %s" % (script, line)
+                line = "%s %s" % (script, line)
             else:
                 line = script
             return self.shebang(line, cell)
@@ -157,16 +168,16 @@ class ScriptMagics(Magics):
     @cell_magic("script")
     def shebang(self, line, cell):
         """Run a cell via a shell command
-        
+
         The `%%script` line is like the #! line of script,
         specifying a program (bash, perl, ruby, etc.) with which to run.
-        
+
         The rest of the cell is run by that program.
-        
+
         Examples
         --------
         ::
-        
+
             In [1]: %%script bash
                ...: for i in 1 2 3; do
                ...:   echo $i
@@ -175,18 +186,70 @@ class ScriptMagics(Magics):
             2
             3
         """
-        argv = arg_split(line, posix = not sys.platform.startswith('win'))
+
+        # Create the event loop in which to run script magics
+        # this operates on a background thread
+        if self.event_loop is None:
+            if sys.platform == "win32":
+                # don't override the current policy,
+                # just create an event loop
+                event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop()
+            else:
+                event_loop = asyncio.new_event_loop()
+            self.event_loop = event_loop
+
+            # start the loop in a background thread
+            asyncio_thread = Thread(target=event_loop.run_forever, daemon=True)
+            asyncio_thread.start()
+        else:
+            event_loop = self.event_loop
+
+        def in_thread(coro):
+            """Call a coroutine on the asyncio thread"""
+            return asyncio.run_coroutine_threadsafe(coro, event_loop).result()
+
+        async def _handle_stream(stream, stream_arg, file_object):
+            while True:
+                line = (await stream.readline()).decode("utf8")
+                if not line:
+                    break
+                if stream_arg:
+                    self.shell.user_ns[stream_arg] = line
+                else:
+                    file_object.write(line)
+                    file_object.flush()
+
+        async def _stream_communicate(process, cell):
+            process.stdin.write(cell)
+            process.stdin.close()
+            stdout_task = asyncio.create_task(
+                _handle_stream(process.stdout, args.out, sys.stdout)
+            )
+            stderr_task = asyncio.create_task(
+                _handle_stream(process.stderr, args.err, sys.stderr)
+            )
+            await asyncio.wait([stdout_task, stderr_task])
+            await process.wait()
+
+        argv = arg_split(line, posix=not sys.platform.startswith("win"))
         args, cmd = self.shebang.parser.parse_known_args(argv)
-        
+
         try:
-            p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
+            p = in_thread(
+                asyncio.create_subprocess_exec(
+                    *cmd,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    stdin=asyncio.subprocess.PIPE,
+                )
+            )
         except OSError as e:
             if e.errno == errno.ENOENT:
                 print("Couldn't find program: %r" % cmd[0])
                 return
             else:
                 raise
-        
+
         if not cell.endswith('\n'):
             cell += '\n'
         cell = cell.encode('utf8', 'replace')
@@ -195,30 +258,35 @@ class ScriptMagics(Magics):
             self._gc_bg_processes()
             to_close = []
             if args.out:
-                self.shell.user_ns[args.out] = p.stdout
+                self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop)
             else:
                 to_close.append(p.stdout)
             if args.err:
-                self.shell.user_ns[args.err] = p.stderr
+                self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop)
             else:
                 to_close.append(p.stderr)
-            self.job_manager.new(self._run_script, p, cell, to_close, daemon=True)
+            event_loop.call_soon_threadsafe(
+                lambda: asyncio.Task(self._run_script(p, cell, to_close))
+            )
             if args.proc:
-                self.shell.user_ns[args.proc] = p
+                proc_proxy = _AsyncIOProxy(p, event_loop)
+                proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop)
+                proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop)
+                self.shell.user_ns[args.proc] = proc_proxy
             return
-        
+
         try:
-            out, err = p.communicate(cell)
+            in_thread(_stream_communicate(p, cell))
         except KeyboardInterrupt:
             try:
                 p.send_signal(signal.SIGINT)
-                time.sleep(0.1)
-                if p.poll() is not None:
+                in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
+                if p.returncode is not None:
                     print("Process is interrupted.")
                     return
                 p.terminate()
-                time.sleep(0.1)
-                if p.poll() is not None:
+                in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
+                if p.returncode is not None:
                     print("Process is terminated.")
                     return
                 p.kill()
@@ -226,31 +294,31 @@ class ScriptMagics(Magics):
             except OSError:
                 pass
             except Exception as e:
-                print("Error while terminating subprocess (pid=%i): %s" \
-                    % (p.pid, e))
+                print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e))
             return
-        out = py3compat.decode(out)
-        err = py3compat.decode(err)
-        if args.out:
-            self.shell.user_ns[args.out] = out
-        else:
-            sys.stdout.write(out)
-            sys.stdout.flush()
-        if args.err:
-            self.shell.user_ns[args.err] = err
-        else:
-            sys.stderr.write(err)
-            sys.stderr.flush()
-        if args.raise_error and p.returncode!=0:
-            raise CalledProcessError(p.returncode, cell, output=out, stderr=err)
-    
-    def _run_script(self, p, cell, to_close):
+
+        if args.raise_error and p.returncode != 0:
+            # If we get here and p.returncode is still None, we must have
+            # killed it but not yet seen its return code. We don't wait for it,
+            # in case it's stuck in uninterruptible sleep. -9 = SIGKILL
+            rc = p.returncode or -9
+            raise CalledProcessError(rc, cell)
+
+    shebang.__skip_doctest__ = os.name != "posix"
+
+    async def _run_script(self, p, cell, to_close):
         """callback for running the script in the background"""
+
         p.stdin.write(cell)
+        await p.stdin.drain()
         p.stdin.close()
+        await p.stdin.wait_closed()
+        await p.wait()
+        # asyncio read pipes have no close
+        # but we should drain the data anyway
         for s in to_close:
-            s.close()
-        p.wait()
+            await s.read()
+        self._gc_bg_processes()
 
     @line_magic("killbgscripts")
     def killbgscripts(self, _nouse_=''):
@@ -263,7 +331,7 @@ class ScriptMagics(Magics):
         if not self.bg_processes:
             return
         for p in self.bg_processes:
-            if p.poll() is None:
+            if p.returncode is None:
                 try:
                     p.send_signal(signal.SIGINT)
                 except:
@@ -273,7 +341,7 @@ class ScriptMagics(Magics):
         if not self.bg_processes:
             return
         for p in self.bg_processes:
-            if p.poll() is None:
+            if p.returncode is None:
                 try:
                     p.terminate()
                 except:
@@ -283,7 +351,7 @@ class ScriptMagics(Magics):
         if not self.bg_processes:
             return
         for p in self.bg_processes:
-            if p.poll() is None:
+            if p.returncode is None:
                 try:
                     p.kill()
                 except:
@@ -291,4 +359,4 @@ class ScriptMagics(Magics):
         self._gc_bg_processes()
 
     def _gc_bg_processes(self):
-        self.bg_processes = [p for p in self.bg_processes if p.poll() is None]
+        self.bg_processes = [p for p in self.bg_processes if p.returncode is None]