From 2598ef1d0aee359b4b6d5fdd1758916d5907d04f Mon Sep 17 00:00:00 2001 From: shadchin <shadchin@yandex-team.ru> Date: Thu, 10 Feb 2022 16:44:30 +0300 Subject: Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2. --- contrib/python/ipython/py3/IPython/utils/openpy.py | 206 ++++++++++----------- 1 file changed, 103 insertions(+), 103 deletions(-) (limited to 'contrib/python/ipython/py3/IPython/utils/openpy.py') diff --git a/contrib/python/ipython/py3/IPython/utils/openpy.py b/contrib/python/ipython/py3/IPython/utils/openpy.py index c90d2b53a3..75f4601a1f 100644 --- a/contrib/python/ipython/py3/IPython/utils/openpy.py +++ b/contrib/python/ipython/py3/IPython/utils/openpy.py @@ -1,103 +1,103 @@ -""" -Tools to open .py files as Unicode, using the encoding specified within the file, -as per PEP 263. - -Much of the code is taken from the tokenize module in Python 3.2. -""" - -import io -from io import TextIOWrapper, BytesIO -import re -from tokenize import open, detect_encoding - -cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE) -cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE) - -def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): - """Converts a bytes string with python source code to unicode. - - Unicode strings are passed through unchanged. Byte strings are checked - for the python source file encoding cookie to determine encoding. - txt can be either a bytes buffer or a string containing the source - code. - """ - if isinstance(txt, str): - return txt - if isinstance(txt, bytes): - buffer = BytesIO(txt) - else: - buffer = txt - try: - encoding, _ = detect_encoding(buffer.readline) - except SyntaxError: - encoding = "ascii" - buffer.seek(0) - with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text: - text.mode = 'r' - if skip_encoding_cookie: - return u"".join(strip_encoding_cookie(text)) - else: - return text.read() - -def strip_encoding_cookie(filelike): - """Generator to pull lines from a text-mode file, skipping the encoding - cookie if it is found in the first two lines. - """ - it = iter(filelike) - try: - first = next(it) - if not cookie_comment_re.match(first): - yield first - second = next(it) - if not cookie_comment_re.match(second): - yield second - except StopIteration: - return - - for line in it: - yield line - -def read_py_file(filename, skip_encoding_cookie=True): - """Read a Python file, using the encoding declared inside the file. - - Parameters - ---------- - filename : str - The path to the file to read. - skip_encoding_cookie : bool - If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output. - - Returns - ------- - A unicode string containing the contents of the file. - """ - with open(filename) as f: # the open function defined in this module. - if skip_encoding_cookie: - return "".join(strip_encoding_cookie(f)) - else: - return f.read() - -def read_py_url(url, errors='replace', skip_encoding_cookie=True): - """Read a Python file from a URL, using the encoding declared inside the file. - - Parameters - ---------- - url : str - The URL from which to fetch the file. - errors : str - How to handle decoding errors in the file. Options are the same as for - bytes.decode(), but here 'replace' is the default. - skip_encoding_cookie : bool - If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output. - - Returns - ------- - A unicode string containing the contents of the file. - """ - # Deferred import for faster start - from urllib.request import urlopen - response = urlopen(url) - buffer = io.BytesIO(response.read()) - return source_to_unicode(buffer, errors, skip_encoding_cookie) +""" +Tools to open .py files as Unicode, using the encoding specified within the file, +as per PEP 263. + +Much of the code is taken from the tokenize module in Python 3.2. +""" + +import io +from io import TextIOWrapper, BytesIO +import re +from tokenize import open, detect_encoding + +cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE) +cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE) + +def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): + """Converts a bytes string with python source code to unicode. + + Unicode strings are passed through unchanged. Byte strings are checked + for the python source file encoding cookie to determine encoding. + txt can be either a bytes buffer or a string containing the source + code. + """ + if isinstance(txt, str): + return txt + if isinstance(txt, bytes): + buffer = BytesIO(txt) + else: + buffer = txt + try: + encoding, _ = detect_encoding(buffer.readline) + except SyntaxError: + encoding = "ascii" + buffer.seek(0) + with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text: + text.mode = 'r' + if skip_encoding_cookie: + return u"".join(strip_encoding_cookie(text)) + else: + return text.read() + +def strip_encoding_cookie(filelike): + """Generator to pull lines from a text-mode file, skipping the encoding + cookie if it is found in the first two lines. + """ + it = iter(filelike) + try: + first = next(it) + if not cookie_comment_re.match(first): + yield first + second = next(it) + if not cookie_comment_re.match(second): + yield second + except StopIteration: + return + + for line in it: + yield line + +def read_py_file(filename, skip_encoding_cookie=True): + """Read a Python file, using the encoding declared inside the file. + + Parameters + ---------- + filename : str + The path to the file to read. + skip_encoding_cookie : bool + If True (the default), and the encoding declaration is found in the first + two lines, that line will be excluded from the output. + + Returns + ------- + A unicode string containing the contents of the file. + """ + with open(filename) as f: # the open function defined in this module. + if skip_encoding_cookie: + return "".join(strip_encoding_cookie(f)) + else: + return f.read() + +def read_py_url(url, errors='replace', skip_encoding_cookie=True): + """Read a Python file from a URL, using the encoding declared inside the file. + + Parameters + ---------- + url : str + The URL from which to fetch the file. + errors : str + How to handle decoding errors in the file. Options are the same as for + bytes.decode(), but here 'replace' is the default. + skip_encoding_cookie : bool + If True (the default), and the encoding declaration is found in the first + two lines, that line will be excluded from the output. + + Returns + ------- + A unicode string containing the contents of the file. + """ + # Deferred import for faster start + from urllib.request import urlopen + response = urlopen(url) + buffer = io.BytesIO(response.read()) + return source_to_unicode(buffer, errors, skip_encoding_cookie) -- cgit v1.2.3