Update contrib/python/ipython/py3 to 8.15.0

1 files changed, 30 insertions, 2 deletions

diff --git a/contrib/python/ipython/py3/IPython/utils/tokenutil.py b/contrib/python/ipython/py3/IPython/utils/tokenutil.py
index 697d2b504a..5fd8a1fbe1 100644
--- a/contrib/python/ipython/py3/IPython/utils/tokenutil.py
+++ b/contrib/python/ipython/py3/IPython/utils/tokenutil.py
@@ -21,6 +21,36 @@ def generate_tokens(readline):
         # catch EOF error
         return
 
+
+def generate_tokens_catch_errors(readline, extra_errors_to_catch=None):
+    default_errors_to_catch = [
+        "unterminated string literal",
+        "invalid non-printable character",
+        "after line continuation character",
+    ]
+    assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list)
+    errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or [])
+
+    tokens = []
+    try:
+        for token in tokenize.generate_tokens(readline):
+            tokens.append(token)
+            yield token
+    except tokenize.TokenError as exc:
+        if any(error in exc.args[0] for error in errors_to_catch):
+            if tokens:
+                start = tokens[-1].start[0], tokens[-1].end[0]
+                end = start
+                line = tokens[-1].line
+            else:
+                start = end = (1, 0)
+                line = ""
+            yield tokenize.TokenInfo(tokenize.ERRORTOKEN, "", start, end, line)
+        else:
+            # Catch EOF
+            raise
+
+
 def line_at_cursor(cell, cursor_pos=0):
     """Return the line in a cell at a given cursor position
 
@@ -123,5 +153,3 @@ def token_at_cursor(cell, cursor_pos=0):
         return names[-1]
     else:
         return ''
-    
-