Intermediate changes

commit_hash:fe9cb645107d4e98cea6850ff89242dd287facbb

13 files changed, 218 insertions, 34 deletions

diff --git a/contrib/python/prompt-toolkit/py3/.dist-info/METADATA b/contrib/python/prompt-toolkit/py3/.dist-info/METADATA
index eae81f9c1d..8d4f5d343d 100644
--- a/contrib/python/prompt-toolkit/py3/.dist-info/METADATA
+++ b/contrib/python/prompt-toolkit/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: prompt_toolkit
-Version: 3.0.48
+Version: 3.0.50
 Summary: Library for building powerful interactive command lines in Python
 Home-page: https://github.com/prompt-toolkit/python-prompt-toolkit
 Author: Jonathan Slenders
@@ -9,20 +9,28 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development
-Requires-Python: >=3.7.0
+Requires-Python: >=3.8.0
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: wcwidth
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 Python Prompt Toolkit
 =====================
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py
index 80da72d1ec..94727e7cb2 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py
@@ -28,7 +28,7 @@ from .formatted_text import ANSI, HTML
 from .shortcuts import PromptSession, print_formatted_text, prompt
 
 # Don't forget to update in `docs/conf.py`!
-__version__ = "3.0.48"
+__version__ = "3.0.50"
 
 assert pep440.match(__version__)
 
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py
index 7e2cf480ba..3f7eb4bd46 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py
@@ -143,8 +143,8 @@ def create_app_session(
     """
     Create a separate AppSession.
 
-    This is useful if there can be multiple individual `AppSession`s going on.
-    Like in the case of an Telnet/SSH server.
+    This is useful if there can be multiple individual ``AppSession``'s going
+    on. Like in the case of a Telnet/SSH server.
     """
     # If no input/output is specified, fall back to the current input/output,
     # if there was one that was set/created for the current session.
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py
index 18a3dadeb9..1f5e18ea78 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py
@@ -111,4 +111,7 @@ async def in_terminal(render_cli_done: bool = False) -> AsyncGenerator[None, Non
             app._request_absolute_cursor_position()
             app._redraw()
         finally:
-            new_run_in_terminal_f.set_result(None)
+            # (Check for `.done()`, because it can be that this future was
+            # cancelled.)
+            if not new_run_in_terminal_f.done():
+                new_run_in_terminal_f.set_result(None)
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py
index 410749db47..cd95424dc3 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py
@@ -81,8 +81,7 @@ class Filter(metaclass=ABCMeta):
         instead of for instance ``filter1 or Always()``.
         """
         raise ValueError(
-            "The truth value of a Filter is ambiguous. "
-            "Instead, call it as a function."
+            "The truth value of a Filter is ambiguous. Instead, call it as a function."
         )
 
 
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py
index 322d7c0d72..1ff3234a39 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py
@@ -16,7 +16,7 @@ if not SPHINX_AUTODOC_RUNNING:
     import msvcrt
     from ctypes import windll
 
-from ctypes import Array, pointer
+from ctypes import Array, byref, pointer
 from ctypes.wintypes import DWORD, HANDLE
 from typing import Callable, ContextManager, Iterable, Iterator, TextIO
 
@@ -35,6 +35,7 @@ from prompt_toolkit.win32_types import (
 
 from .ansi_escape_sequences import REVERSE_ANSI_SEQUENCES
 from .base import Input
+from .vt100_parser import Vt100Parser
 
 __all__ = [
     "Win32Input",
@@ -52,6 +53,9 @@ RIGHTMOST_BUTTON_PRESSED = 0x2
 MOUSE_MOVED = 0x0001
 MOUSE_WHEELED = 0x0004
 
+# See: https://msdn.microsoft.com/pl-pl/library/windows/desktop/ms686033(v=vs.85).aspx
+ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200
+
 
 class _Win32InputBase(Input):
     """
@@ -74,7 +78,14 @@ class Win32Input(_Win32InputBase):
 
     def __init__(self, stdin: TextIO | None = None) -> None:
         super().__init__()
-        self.console_input_reader = ConsoleInputReader()
+        self._use_virtual_terminal_input = _is_win_vt100_input_enabled()
+
+        self.console_input_reader: Vt100ConsoleInputReader | ConsoleInputReader
+
+        if self._use_virtual_terminal_input:
+            self.console_input_reader = Vt100ConsoleInputReader()
+        else:
+            self.console_input_reader = ConsoleInputReader()
 
     def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]:
         """
@@ -101,7 +112,9 @@ class Win32Input(_Win32InputBase):
         return False
 
     def raw_mode(self) -> ContextManager[None]:
-        return raw_mode()
+        return raw_mode(
+            use_win10_virtual_terminal_input=self._use_virtual_terminal_input
+        )
 
     def cooked_mode(self) -> ContextManager[None]:
         return cooked_mode()
@@ -555,6 +568,102 @@ class ConsoleInputReader:
         return [KeyPress(Keys.WindowsMouseEvent, data)]
 
 
+class Vt100ConsoleInputReader:
+    """
+    Similar to `ConsoleInputReader`, but for usage when
+    `ENABLE_VIRTUAL_TERMINAL_INPUT` is enabled. This assumes that Windows sends
+    us the right vt100 escape sequences and we parse those with our vt100
+    parser.
+
+    (Using this instead of `ConsoleInputReader` results in the "data" attribute
+    from the `KeyPress` instances to be more correct in edge cases, because
+    this responds to for instance the terminal being in application cursor keys
+    mode.)
+    """
+
+    def __init__(self) -> None:
+        self._fdcon = None
+
+        self._buffer: list[KeyPress] = []  # Buffer to collect the Key objects.
+        self._vt100_parser = Vt100Parser(
+            lambda key_press: self._buffer.append(key_press)
+        )
+
+        # When stdin is a tty, use that handle, otherwise, create a handle from
+        # CONIN$.
+        self.handle: HANDLE
+        if sys.stdin.isatty():
+            self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE))
+        else:
+            self._fdcon = os.open("CONIN$", os.O_RDWR | os.O_BINARY)
+            self.handle = HANDLE(msvcrt.get_osfhandle(self._fdcon))
+
+    def close(self) -> None:
+        "Close fdcon."
+        if self._fdcon is not None:
+            os.close(self._fdcon)
+
+    def read(self) -> Iterable[KeyPress]:
+        """
+        Return a list of `KeyPress` instances. It won't return anything when
+        there was nothing to read.  (This function doesn't block.)
+
+        http://msdn.microsoft.com/en-us/library/windows/desktop/ms684961(v=vs.85).aspx
+        """
+        max_count = 2048  # Max events to read at the same time.
+
+        read = DWORD(0)
+        arrtype = INPUT_RECORD * max_count
+        input_records = arrtype()
+
+        # Check whether there is some input to read. `ReadConsoleInputW` would
+        # block otherwise.
+        # (Actually, the event loop is responsible to make sure that this
+        # function is only called when there is something to read, but for some
+        # reason this happened in the asyncio_win32 loop, and it's better to be
+        # safe anyway.)
+        if not wait_for_handles([self.handle], timeout=0):
+            return []
+
+        # Get next batch of input event.
+        windll.kernel32.ReadConsoleInputW(
+            self.handle, pointer(input_records), max_count, pointer(read)
+        )
+
+        # First, get all the keys from the input buffer, in order to determine
+        # whether we should consider this a paste event or not.
+        for key_data in self._get_keys(read, input_records):
+            self._vt100_parser.feed(key_data)
+
+        # Return result.
+        result = self._buffer
+        self._buffer = []
+        return result
+
+    def _get_keys(
+        self, read: DWORD, input_records: Array[INPUT_RECORD]
+    ) -> Iterator[str]:
+        """
+        Generator that yields `KeyPress` objects from the input records.
+        """
+        for i in range(read.value):
+            ir = input_records[i]
+
+            # Get the right EventType from the EVENT_RECORD.
+            # (For some reason the Windows console application 'cmder'
+            # [http://gooseberrycreative.com/cmder/] can return '0' for
+            # ir.EventType. -- Just ignore that.)
+            if ir.EventType in EventTypes:
+                ev = getattr(ir.Event, EventTypes[ir.EventType])
+
+                # Process if this is a key event. (We also have mouse, menu and
+                # focus events.)
+                if isinstance(ev, KEY_EVENT_RECORD) and ev.KeyDown:
+                    u_char = ev.uChar.UnicodeChar
+                    if u_char != "\x00":
+                        yield u_char
+
+
 class _Win32Handles:
     """
     Utility to keep track of which handles are connectod to which callbacks.
@@ -700,8 +809,11 @@ class raw_mode:
     `raw_input` method of `.vt100_input`.
     """
 
-    def __init__(self, fileno: int | None = None) -> None:
+    def __init__(
+        self, fileno: int | None = None, use_win10_virtual_terminal_input: bool = False
+    ) -> None:
         self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE))
+        self.use_win10_virtual_terminal_input = use_win10_virtual_terminal_input
 
     def __enter__(self) -> None:
         # Remember original mode.
@@ -717,12 +829,15 @@ class raw_mode:
         ENABLE_LINE_INPUT = 0x0002
         ENABLE_PROCESSED_INPUT = 0x0001
 
-        windll.kernel32.SetConsoleMode(
-            self.handle,
-            self.original_mode.value
-            & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT),
+        new_mode = self.original_mode.value & ~(
+            ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT
         )
 
+        if self.use_win10_virtual_terminal_input:
+            new_mode |= ENABLE_VIRTUAL_TERMINAL_INPUT
+
+        windll.kernel32.SetConsoleMode(self.handle, new_mode)
+
     def __exit__(self, *a: object) -> None:
         # Restore original mode
         windll.kernel32.SetConsoleMode(self.handle, self.original_mode)
@@ -747,3 +862,25 @@ class cooked_mode(raw_mode):
             self.original_mode.value
             | (ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT),
         )
+
+
+def _is_win_vt100_input_enabled() -> bool:
+    """
+    Returns True when we're running Windows and VT100 escape sequences are
+    supported.
+    """
+    hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE))
+
+    # Get original console mode.
+    original_mode = DWORD(0)
+    windll.kernel32.GetConsoleMode(hconsole, byref(original_mode))
+
+    try:
+        # Try to enable VT100 sequences.
+        result: int = windll.kernel32.SetConsoleMode(
+            hconsole, DWORD(ENABLE_VIRTUAL_TERMINAL_INPUT)
+        )
+
+        return result == 1
+    finally:
+        windll.kernel32.SetConsoleMode(hconsole, original_mode)
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py
index 222e471c57..5083c8286d 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py
@@ -667,7 +667,11 @@ class BufferControl(UIControl):
 
         merged_processor = merge_processors(input_processors)
 
-        def transform(lineno: int, fragments: StyleAndTextTuples) -> _ProcessedLine:
+        def transform(
+            lineno: int,
+            fragments: StyleAndTextTuples,
+            get_line: Callable[[int], StyleAndTextTuples],
+        ) -> _ProcessedLine:
             "Transform the fragments for a given line number."
 
             # Get cursor position at this line.
@@ -679,7 +683,14 @@ class BufferControl(UIControl):
 
             transformation = merged_processor.apply_transformation(
                 TransformationInput(
-                    self, document, lineno, source_to_display, fragments, width, height
+                    self,
+                    document,
+                    lineno,
+                    source_to_display,
+                    fragments,
+                    width,
+                    height,
+                    get_line,
                 )
             )
 
@@ -697,7 +708,7 @@ class BufferControl(UIControl):
                 try:
                     return cache[i]
                 except KeyError:
-                    processed_line = transform(i, get_line(i))
+                    processed_line = transform(i, get_line(i), get_line)
                     cache[i] = processed_line
                     return processed_line
 
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py
index b10ecf7184..666e79c66d 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py
@@ -86,6 +86,9 @@ class TransformationInput:
         previous processors into account.)
     :param fragments: List of fragments that we can transform. (Received from the
         previous processor.)
+    :param get_line: Optional ; a callable that returns the fragments of another
+        line in the  current buffer; This can be used to create processors capable
+        of affecting transforms across multiple lines.
     """
 
     def __init__(
@@ -97,6 +100,7 @@ class TransformationInput:
         fragments: StyleAndTextTuples,
         width: int,
         height: int,
+        get_line: Callable[[int], StyleAndTextTuples] | None = None,
     ) -> None:
         self.buffer_control = buffer_control
         self.document = document
@@ -105,6 +109,7 @@ class TransformationInput:
         self.fragments = fragments
         self.width = width
         self.height = height
+        self.get_line = get_line
 
     def unpack(
         self,
@@ -842,9 +847,9 @@ class ReverseSearchProcessor(Processor):
     def apply_transformation(self, ti: TransformationInput) -> Transformation:
         from .controls import SearchBufferControl
 
-        assert isinstance(
-            ti.buffer_control, SearchBufferControl
-        ), "`ReverseSearchProcessor` should be applied to a `SearchBufferControl` only."
+        assert isinstance(ti.buffer_control, SearchBufferControl), (
+            "`ReverseSearchProcessor` should be applied to a `SearchBufferControl` only."
+        )
 
         source_to_display: SourceToDisplay | None
         display_to_source: DisplayToSource | None
@@ -987,6 +992,7 @@ class _MergedProcessor(Processor):
                     fragments,
                     ti.width,
                     ti.height,
+                    ti.get_line,
                 )
             )
             fragments = transformation.fragments
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py
index 069636b8c3..90df21e558 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py
@@ -436,6 +436,11 @@ class Vt100_Output(Output):
         # default, we don't change them.)
         self._cursor_shape_changed = False
 
+        # Don't hide/show the cursor when this was already done.
+        # (`None` means that we don't know whether the cursor is visible or
+        # not.)
+        self._cursor_visible: bool | None = None
+
     @classmethod
     def from_pty(
         cls,
@@ -651,10 +656,14 @@ class Vt100_Output(Output):
             self.write_raw("\x1b[%iD" % amount)
 
     def hide_cursor(self) -> None:
-        self.write_raw("\x1b[?25l")
+        if self._cursor_visible in (True, None):
+            self._cursor_visible = False
+            self.write_raw("\x1b[?25l")
 
     def show_cursor(self) -> None:
-        self.write_raw("\x1b[?12l\x1b[?25h")  # Stop blinking cursor and show.
+        if self._cursor_visible in (False, None):
+            self._cursor_visible = True
+            self.write_raw("\x1b[?12l\x1b[?25h")  # Stop blinking cursor and show.
 
     def set_cursor_shape(self, cursor_shape: CursorShape) -> None:
         if cursor_shape == CursorShape._NEVER_CHANGE:
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py
index c39f3ecfd1..2b7e596e46 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py
@@ -66,15 +66,20 @@ class Windows10_Output:
         return False  # We don't need this on Windows.
 
     def __getattr__(self, name: str) -> Any:
+        # NOTE: Now that we use "virtual terminal input" on
+        #       Windows, both input and output are done through
+        #       ANSI escape sequences on Windows. This means, we
+        #       should enable bracketed paste like on Linux, and
+        #       enable mouse support by calling the vt100_output.
         if name in (
             "get_size",
             "get_rows_below_cursor_position",
-            "enable_mouse_support",
-            "disable_mouse_support",
             "scroll_buffer_to_prompt",
             "get_win32_screen_buffer_info",
-            "enable_bracketed_paste",
-            "disable_bracketed_paste",
+            # "enable_mouse_support",
+            # "disable_mouse_support",
+            # "enable_bracketed_paste",
+            # "disable_bracketed_paste",
         ):
             return getattr(self.win32_output, name)
         else:
diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py
index 3f92303a81..8d5e03c19e 100644
--- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py
+++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py
@@ -257,7 +257,7 @@ def _output_screen_diff(
     # give weird artifacts on resize events.)
     reset_attributes()
 
-    if screen.show_cursor or is_done:
+    if screen.show_cursor:
         output.show_cursor()
 
     return current_pos, last_style
@@ -353,6 +353,11 @@ class Renderer:
         self.mouse_support = to_filter(mouse_support)
         self.cpr_not_supported_callback = cpr_not_supported_callback
 
+        # TODO: Move following state flags into `Vt100_Output`, similar to
+        #       `_cursor_shape_changed` and `_cursor_visible`. But then also
+        #       adjust the `Win32Output` to not call win32 APIs if nothing has
+        #       to be changed.
+
         self._in_alternate_screen = False
         self._mouse_support_enabled = False
         self._bracketed_paste_enabled = False
@@ -416,6 +421,7 @@ class Renderer:
             self._bracketed_paste_enabled = False
 
         self.output.reset_cursor_shape()
+        self.output.show_cursor()
 
         # NOTE: No need to set/reset cursor key mode here.
 
diff --git a/contrib/python/prompt-toolkit/py3/tests/test_cli.py b/contrib/python/prompt-toolkit/py3/tests/test_cli.py
index c155325f98..a876f2993b 100644
--- a/contrib/python/prompt-toolkit/py3/tests/test_cli.py
+++ b/contrib/python/prompt-toolkit/py3/tests/test_cli.py
@@ -870,11 +870,11 @@ def test_vi_temp_navigation_mode():
     """
     feed = partial(_feed_cli_with_input, editing_mode=EditingMode.VI)
 
-    result, cli = feed("abcde" "\x0f" "3h" "x\r")  # c-o  # 3 times to the left.
+    result, cli = feed("abcde\x0f3hx\r")  # c-o  # 3 times to the left.
     assert result.text == "axbcde"
     assert result.cursor_position == 2
 
-    result, cli = feed("abcde" "\x0f" "b" "x\r")  # c-o  # One word backwards.
+    result, cli = feed("abcde\x0fbx\r")  # c-o  # One word backwards.
     assert result.text == "xabcde"
     assert result.cursor_position == 1
 
diff --git a/contrib/python/prompt-toolkit/py3/ya.make b/contrib/python/prompt-toolkit/py3/ya.make
index b1cb3e19b0..5eed9c2519 100644
--- a/contrib/python/prompt-toolkit/py3/ya.make
+++ b/contrib/python/prompt-toolkit/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(3.0.48)
+VERSION(3.0.50)
 
 LICENSE(BSD-3-Clause)