Update contrib/python/ipython/py3 to 8.3.0

ref:e84342d4d30476f9148137f37fd0c6405fd36f55

1 files changed, 209 insertions, 88 deletions

diff --git a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py
index 4e35aadd616..06724bea870 100644
--- a/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py
+++ b/contrib/python/ipython/py3/IPython/terminal/interactiveshell.py
@@ -3,23 +3,34 @@
 import asyncio
 import os
 import sys
-import warnings
 from warnings import warn
 
+from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
-from IPython.utils import io
 from IPython.utils.py3compat import input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import (
-    Bool, Unicode, Dict, Integer, observe, Instance, Type, default, Enum, Union,
-    Any, validate
+    Bool,
+    Unicode,
+    Dict,
+    Integer,
+    observe,
+    Instance,
+    Type,
+    default,
+    Enum,
+    Union,
+    Any,
+    validate,
+    Float,
 )
 
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.formatted_text import PygmentsTokens
-from prompt_toolkit.history import InMemoryHistory
+from prompt_toolkit.history import History
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.output import ColorDepth
 from prompt_toolkit.patch_stdout import patch_stdout
@@ -39,7 +50,6 @@ from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
 
-DISPLAY_BANNER_DEPRECATED = object()
 PTK3 = ptk_version.startswith('3.')
 
 
@@ -48,17 +58,17 @@ class _NoStyle(Style): pass
 
 
 _style_overrides_light_bg = {
-            Token.Prompt: '#0000ff',
-            Token.PromptNum: '#0000ee bold',
-            Token.OutPrompt: '#cc0000',
-            Token.OutPromptNum: '#bb0000 bold',
+            Token.Prompt: '#ansibrightblue',
+            Token.PromptNum: '#ansiblue bold',
+            Token.OutPrompt: '#ansibrightred',
+            Token.OutPromptNum: '#ansired bold',
 }
 
 _style_overrides_linux = {
-            Token.Prompt: '#00cc00',
-            Token.PromptNum: '#00bb00 bold',
-            Token.OutPrompt: '#cc0000',
-            Token.OutPromptNum: '#bb0000 bold',
+            Token.Prompt: '#ansibrightgreen',
+            Token.PromptNum: '#ansigreen bold',
+            Token.OutPrompt: '#ansibrightred',
+            Token.OutPromptNum: '#ansired bold',
 }
 
 def get_default_editor():
@@ -91,13 +101,72 @@ else:
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
 def black_reformat_handler(text_before_cursor):
+    """
+    We do not need to protect against error,
+    this is taken care at a higher level where any reformat error is ignored.
+    Indeed we may call reformatting on incomplete code.
+    """
     import black
+
     formatted_text = black.format_str(text_before_cursor, mode=black.FileMode())
-    if not text_before_cursor.endswith('\n') and formatted_text.endswith('\n'):
-       formatted_text = formatted_text[:-1]
+    if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"):
+        formatted_text = formatted_text[:-1]
     return formatted_text
 
 
+def yapf_reformat_handler(text_before_cursor):
+    from yapf.yapflib import file_resources
+    from yapf.yapflib import yapf_api
+
+    style_config = file_resources.GetDefaultStyleForDir(os.getcwd())
+    formatted_text, was_formatted = yapf_api.FormatCode(
+        text_before_cursor, style_config=style_config
+    )
+    if was_formatted:
+        if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"):
+            formatted_text = formatted_text[:-1]
+        return formatted_text
+    else:
+        return text_before_cursor
+
+
+class PtkHistoryAdapter(History):
+    """
+    Prompt toolkit has it's own way of handling history, Where it assumes it can
+    Push/pull from history.
+
+    """
+
+    def __init__(self, shell):
+        super().__init__()
+        self.shell = shell
+        self._refresh()
+
+    def append_string(self, string):
+        # we rely on sql for that.
+        self._loaded = False
+        self._refresh()
+
+    def _refresh(self):
+        if not self._loaded:
+            self._loaded_strings = list(self.load_history_strings())
+
+    def load_history_strings(self):
+        last_cell = ""
+        res = []
+        for __, ___, cell in self.shell.history_manager.get_tail(
+            self.shell.history_load_length, include_latest=True
+        ):
+            # Ignore blank lines and consecutive duplicates
+            cell = cell.rstrip()
+            if cell and (cell != last_cell):
+                res.append(cell)
+                last_cell = cell
+        yield from res[::-1]
+
+    def store_string(self, string: str) -> None:
+        pass
+
 class TerminalInteractiveShell(InteractiveShell):
     mime_renderers = Dict().tag(config=True)
 
@@ -112,6 +181,10 @@ class TerminalInteractiveShell(InteractiveShell):
     pt_app = None
     debugger_history = None
 
+    debugger_history_file = Unicode(
+        "~/.pdbhistory", help="File in which to store and read history"
+    ).tag(config=True)
+
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion and prompt colors.
 
@@ -137,11 +210,45 @@ class TerminalInteractiveShell(InteractiveShell):
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
-    autoformatter = Unicode(None,
-        help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
+    emacs_bindings_in_vi_insert_mode = Bool(
+        True,
+        help="Add shortcuts from 'emacs' insert mode to 'vi' insert mode.",
+    ).tag(config=True)
+
+    modal_cursor = Bool(
+        True,
+        help="""
+       Cursor shape changes depending on vi mode: beam in vi insert mode,
+       block in nav mode, underscore in replace mode.""",
+    ).tag(config=True)
+
+    ttimeoutlen = Float(
+        0.01,
+        help="""The time in milliseconds that is waited for a key code
+       to complete.""",
+    ).tag(config=True)
+
+    timeoutlen = Float(
+        0.5,
+        help="""The time in milliseconds that is waited for a mapped key
+       sequence to complete.""",
+    ).tag(config=True)
+
+    autoformatter = Unicode(
+        None,
+        help="Autoformatter to reformat Terminal code. Can be `'black'`, `'yapf'` or `None`",
         allow_none=True
     ).tag(config=True)
 
+    auto_match = Bool(
+        False,
+        help="""
+        Automatically add/delete closing bracket or quote when opening bracket or quote is entered/deleted.
+        Brackets: (), [], {}
+        Quotes: '', \"\"
+        """,
+    ).tag(config=True)
+
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
     ).tag(config=True)
@@ -171,16 +278,21 @@ class TerminalInteractiveShell(InteractiveShell):
         if self.pt_app:
             self.pt_app.editing_mode = getattr(EditingMode, change.new.upper())
 
-    @observe('autoformatter')
-    def _autoformatter_changed(self, change):
-        formatter = change.new
+    def _set_formatter(self, formatter):
         if formatter is None:
             self.reformat_handler = lambda x:x
         elif formatter == 'black':
             self.reformat_handler = black_reformat_handler
+        elif formatter == "yapf":
+            self.reformat_handler = yapf_reformat_handler
         else:
             raise ValueError
 
+    @observe("autoformatter")
+    def _autoformatter_changed(self, change):
+        formatter = change.new
+        self._set_formatter(formatter)
+
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
@@ -195,10 +307,12 @@ class TerminalInteractiveShell(InteractiveShell):
     ).tag(config=True)
 
     true_color = Bool(False,
-        help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
-              "If your terminal supports true color, the following command "
-              "should print 'TRUECOLOR' in orange: "
-              "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")
+        help="""Use 24bit colors instead of 256 colors in prompt highlighting.
+        If your terminal supports true color, the following command should
+        print ``TRUECOLOR`` in orange::
+
+            printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"
+        """,
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
@@ -256,6 +370,29 @@ class TerminalInteractiveShell(InteractiveShell):
         help="Allows to enable/disable the prompt toolkit history search"
     ).tag(config=True)
 
+    autosuggestions_provider = Unicode(
+        "AutoSuggestFromHistory",
+        help="Specifies from which source automatic suggestions are provided. "
+        "Can be set to `'AutoSuggestFromHistory`' or `None` to disable"
+        "automatic suggestions. Default is `'AutoSuggestFromHistory`'.",
+        allow_none=True,
+    ).tag(config=True)
+
+    def _set_autosuggestions(self, provider):
+        if provider is None:
+            self.auto_suggest = None
+        elif provider == "AutoSuggestFromHistory":
+            self.auto_suggest = AutoSuggestFromHistory()
+        else:
+            raise ValueError("No valid provider.")
+        if self.pt_app:
+            self.pt_app.auto_suggest = self.auto_suggest
+
+    @observe("autosuggestions_provider")
+    def _autosuggestions_provider_changed(self, change):
+        provider = change.new
+        self._set_autosuggestions(provider)
+
     prompt_includes_vi_mode = Bool(True,
         help="Display the current vi mode (when using vi editing mode)."
     ).tag(config=True)
@@ -276,9 +413,7 @@ class TerminalInteractiveShell(InteractiveShell):
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
-        self.display_formatter.active_types = ['text/plain']
-        # disable `_ipython_display_`
-        self.display_formatter.ipython_display_formatter.enabled = False
+        self.display_formatter.active_types = ["text/plain"]
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
@@ -297,16 +432,9 @@ class TerminalInteractiveShell(InteractiveShell):
         # Set up keyboard shortcuts
         key_bindings = create_ipython_shortcuts(self)
 
+
         # Pre-populate history from IPython's history database
-        history = InMemoryHistory()
-        last_cell = u""
-        for __, ___, cell in self.history_manager.get_tail(self.history_load_length,
-                                                        include_latest=True):
-            # Ignore blank lines and consecutive duplicates
-            cell = cell.rstrip()
-            if cell and (cell != last_cell):
-                history.append_string(cell)
-                last_cell = cell
+        history = PtkHistoryAdapter(self)
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         self.style = DynamicStyle(lambda: self._style)
@@ -315,18 +443,20 @@ class TerminalInteractiveShell(InteractiveShell):
 
         self.pt_loop = asyncio.new_event_loop()
         self.pt_app = PromptSession(
-                            editing_mode=editing_mode,
-                            key_bindings=key_bindings,
-                            history=history,
-                            completer=IPythonPTCompleter(shell=self),
-                            enable_history_search = self.enable_history_search,
-                            style=self.style,
-                            include_default_pygments_style=False,
-                            mouse_support=self.mouse_support,
-                            enable_open_in_editor=self.extra_open_editor_shortcuts,
-                            color_depth=self.color_depth,
-                            tempfile_suffix=".py",
-                            **self._extra_prompt_options())
+            auto_suggest=self.auto_suggest,
+            editing_mode=editing_mode,
+            key_bindings=key_bindings,
+            history=history,
+            completer=IPythonPTCompleter(shell=self),
+            enable_history_search=self.enable_history_search,
+            style=self.style,
+            include_default_pygments_style=False,
+            mouse_support=self.mouse_support,
+            enable_open_in_editor=self.extra_open_editor_shortcuts,
+            color_depth=self.color_depth,
+            tempfile_suffix=".py",
+            **self._extra_prompt_options()
+        )
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
@@ -349,16 +479,16 @@ class TerminalInteractiveShell(InteractiveShell):
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
-                    Token.Number: '#007700',
+                    Token.Number: '#ansigreen',
                     Token.Operator: 'noinherit',
-                    Token.String: '#BB6622',
-                    Token.Name.Function: '#2080D0',
-                    Token.Name.Class: 'bold #2080D0',
-                    Token.Name.Namespace: 'bold #2080D0',
+                    Token.String: '#ansiyellow',
+                    Token.Name.Function: '#ansiblue',
+                    Token.Name.Class: 'bold #ansiblue',
+                    Token.Name.Namespace: 'bold #ansiblue',
                     Token.Name.Variable.Magic: '#ansiblue',
-                    Token.Prompt: '#009900',
+                    Token.Prompt: '#ansigreen',
                     Token.PromptNum: '#ansibrightgreen bold',
-                    Token.OutPrompt: '#990000',
+                    Token.OutPrompt: '#ansired',
                     Token.OutPromptNum: '#ansibrightred bold',
                 })
 
@@ -382,9 +512,9 @@ class TerminalInteractiveShell(InteractiveShell):
             else:
                 style_cls = name_or_cls
             style_overrides = {
-                Token.Prompt: '#009900',
+                Token.Prompt: '#ansigreen',
                 Token.PromptNum: '#ansibrightgreen bold',
-                Token.OutPrompt: '#990000',
+                Token.OutPrompt: '#ansired',
                 Token.OutPromptNum: '#ansibrightred bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
@@ -461,14 +591,14 @@ class TerminalInteractiveShell(InteractiveShell):
         # while/true inside which will freeze the prompt.
 
         policy = asyncio.get_event_loop_policy()
-        try:
-            old_loop = policy.get_event_loop()
-        except RuntimeError:
-            # This happens when the the event loop is closed,
-            # e.g. by calling `asyncio.run()`.
-            old_loop = None
-
-        policy.set_event_loop(self.pt_loop)
+        old_loop = get_asyncio_loop()
+
+        # FIXME: prompt_toolkit is using the deprecated `asyncio.get_event_loop`
+        # to get the current event loop.
+        # This will probably be replaced by an attribute or input argument,
+        # at which point we can stop calling the soon-to-be-deprecated `set_event_loop` here.
+        if old_loop is not self.pt_loop:
+            policy.set_event_loop(self.pt_loop)
         try:
             with patch_stdout(raw=True):
                 text = self.pt_app.prompt(
@@ -476,7 +606,7 @@ class TerminalInteractiveShell(InteractiveShell):
                     **self._extra_prompt_options())
         finally:
             # Restore the original event loop.
-            if old_loop is not None:
+            if old_loop is not None and old_loop is not self.pt_loop:
                 policy.set_event_loop(old_loop)
 
         return text
@@ -484,7 +614,6 @@ class TerminalInteractiveShell(InteractiveShell):
     def enable_win_unicode_console(self):
         # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
         # console by default, so WUC shouldn't be needed.
-        from warnings import warn
         warn("`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
              DeprecationWarning,
              stacklevel=2)
@@ -496,16 +625,6 @@ class TerminalInteractiveShell(InteractiveShell):
         import colorama
         colorama.init()
 
-        # For some reason we make these wrappers around stdout/stderr.
-        # For now, we need to reset them so all output gets coloured.
-        # https://github.com/ipython/ipython/issues/8669
-        # io.std* are deprecated, but don't show our own deprecation warnings
-        # during initialization of the deprecated API.
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', DeprecationWarning)
-            io.stdout = io.IOStream(sys.stdout)
-            io.stderr = io.IOStream(sys.stderr)
-
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
@@ -525,22 +644,19 @@ class TerminalInteractiveShell(InteractiveShell):
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
+        self._set_autosuggestions(self.autosuggestions_provider)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
+        self._set_formatter(self.autoformatter)
 
-        self.debugger_history = InMemoryHistory()
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
-    def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
-
-        if display_banner is not DISPLAY_BANNER_DEPRECATED:
-            warn('interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
-
+    def interact(self):
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
@@ -556,11 +672,9 @@ class TerminalInteractiveShell(InteractiveShell):
                 if code:
                     self.run_cell(code, store_history=True)
 
-    def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
+    def mainloop(self):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
-        if display_banner is not DISPLAY_BANNER_DEPRECATED:
-            warn('mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
         while True:
             try:
                 self.interact()
@@ -577,6 +691,13 @@ class TerminalInteractiveShell(InteractiveShell):
 
                 self.restore_term_title()
 
+        # try to call some at-exit operation optimistically as some things can't
+        # be done during interpreter shutdown. this is technically inaccurate as
+        # this make mainlool not re-callable, but that should be a rare if not
+        # in existent use case.
+
+        self._atexit_once()
+
 
     _inputhook = None
     def inputhook(self, context):
@@ -601,7 +722,7 @@ class TerminalInteractiveShell(InteractiveShell):
                 # When we integrate the asyncio event loop, run the UI in the
                 # same event loop as the rest of the code. don't use an actual
                 # input hook. (Asyncio is not made for nesting event loops.)
-                self.pt_loop = asyncio.get_event_loop()
+                self.pt_loop = get_asyncio_loop()
 
             elif self._inputhook:
                 # If an inputhook was set, create a new asyncio event loop with