diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/python/parso/py3/tests | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/python/parso/py3/tests')
54 files changed, 8623 insertions, 8629 deletions
diff --git a/contrib/python/parso/py3/tests/conftest.py b/contrib/python/parso/py3/tests/conftest.py index 7ac062f433..9e7dec62df 100644 --- a/contrib/python/parso/py3/tests/conftest.py +++ b/contrib/python/parso/py3/tests/conftest.py @@ -1,148 +1,148 @@ -import re -import tempfile -import shutil -import logging -import os -from pathlib import Path - -import pytest -import yatest.common - -import parso -from parso import cache -from parso.utils import parse_version_string - -collect_ignore = ["setup.py"] - -_SUPPORTED_VERSIONS = '3.6', '3.7', '3.8', '3.9', '3.10' - - -@pytest.fixture(scope='session') -def clean_parso_cache(): - """ - Set the default cache directory to a temporary directory during tests. - - Note that you can't use built-in `tmpdir` and `monkeypatch` - fixture here because their scope is 'function', which is not used - in 'session' scope fixture. - - This fixture is activated in ../pytest.ini. - """ - old = cache._default_cache_path - tmp = tempfile.mkdtemp(prefix='parso-test-') - cache._default_cache_path = Path(tmp) - yield - cache._default_cache_path = old - shutil.rmtree(tmp) - - -def pytest_addoption(parser): - parser.addoption("--logging", "-L", action='store_true', - help="Enables the logging output.") - - -def pytest_generate_tests(metafunc): - if 'normalizer_issue_case' in metafunc.fixturenames: - base_dir = os.path.join(yatest.common.test_source_path(), 'normalizer_issue_files') - - cases = list(colllect_normalizer_tests(base_dir)) - metafunc.parametrize( - 'normalizer_issue_case', - cases, - ids=[c.name for c in cases] - ) - elif 'each_version' in metafunc.fixturenames: - metafunc.parametrize('each_version', _SUPPORTED_VERSIONS) - elif 'version_ge_py38' in metafunc.fixturenames: - ge38 = set(_SUPPORTED_VERSIONS) - {'3.6', '3.7'} - metafunc.parametrize('version_ge_py38', sorted(ge38)) - - -class NormalizerIssueCase: - """ - Static Analysis cases lie in the static_analysis folder. - The tests also start with `#!`, like the goto_definition tests. - """ - def __init__(self, path): - self.path = path - self.name = os.path.basename(path) - match = re.search(r'python([\d.]+)\.py', self.name) - self.python_version = match and match.group(1) - - -def colllect_normalizer_tests(base_dir): - for f_name in os.listdir(base_dir): - if f_name.endswith(".py"): - path = os.path.join(base_dir, f_name) - yield NormalizerIssueCase(path) - - -def pytest_configure(config): - if config.option.logging: - root = logging.getLogger() - root.setLevel(logging.DEBUG) - - #ch = logging.StreamHandler(sys.stdout) - #ch.setLevel(logging.DEBUG) - #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - #ch.setFormatter(formatter) - - #root.addHandler(ch) - - -class Checker: - def __init__(self, version, is_passing): - self.version = version - self._is_passing = is_passing - self.grammar = parso.load_grammar(version=self.version) - - def parse(self, code): - if self._is_passing: - return parso.parse(code, version=self.version, error_recovery=False) - else: - self._invalid_syntax(code) - - def _invalid_syntax(self, code): - with pytest.raises(parso.ParserSyntaxError): - module = parso.parse(code, version=self.version, error_recovery=False) - # For debugging - print(module.children) - - def get_error(self, code): - errors = list(self.grammar.iter_errors(self.grammar.parse(code))) - assert bool(errors) != self._is_passing - if errors: - return errors[0] - - def get_error_message(self, code): - error = self.get_error(code) - if error is None: - return - return error.message - - def assert_no_error_in_passing(self, code): - if self._is_passing: - module = self.grammar.parse(code) - assert not list(self.grammar.iter_errors(module)) - - -@pytest.fixture -def works_not_in_py(each_version): - return Checker(each_version, False) - - -@pytest.fixture -def works_in_py(each_version): - return Checker(each_version, True) - - -@pytest.fixture -def works_ge_py38(each_version): - version_info = parse_version_string(each_version) - return Checker(each_version, version_info >= (3, 8)) - - -@pytest.fixture -def works_ge_py39(each_version): - version_info = parse_version_string(each_version) - return Checker(each_version, version_info >= (3, 9)) +import re +import tempfile +import shutil +import logging +import os +from pathlib import Path + +import pytest +import yatest.common + +import parso +from parso import cache +from parso.utils import parse_version_string + +collect_ignore = ["setup.py"] + +_SUPPORTED_VERSIONS = '3.6', '3.7', '3.8', '3.9', '3.10' + + +@pytest.fixture(scope='session') +def clean_parso_cache(): + """ + Set the default cache directory to a temporary directory during tests. + + Note that you can't use built-in `tmpdir` and `monkeypatch` + fixture here because their scope is 'function', which is not used + in 'session' scope fixture. + + This fixture is activated in ../pytest.ini. + """ + old = cache._default_cache_path + tmp = tempfile.mkdtemp(prefix='parso-test-') + cache._default_cache_path = Path(tmp) + yield + cache._default_cache_path = old + shutil.rmtree(tmp) + + +def pytest_addoption(parser): + parser.addoption("--logging", "-L", action='store_true', + help="Enables the logging output.") + + +def pytest_generate_tests(metafunc): + if 'normalizer_issue_case' in metafunc.fixturenames: + base_dir = os.path.join(yatest.common.test_source_path(), 'normalizer_issue_files') + + cases = list(colllect_normalizer_tests(base_dir)) + metafunc.parametrize( + 'normalizer_issue_case', + cases, + ids=[c.name for c in cases] + ) + elif 'each_version' in metafunc.fixturenames: + metafunc.parametrize('each_version', _SUPPORTED_VERSIONS) + elif 'version_ge_py38' in metafunc.fixturenames: + ge38 = set(_SUPPORTED_VERSIONS) - {'3.6', '3.7'} + metafunc.parametrize('version_ge_py38', sorted(ge38)) + + +class NormalizerIssueCase: + """ + Static Analysis cases lie in the static_analysis folder. + The tests also start with `#!`, like the goto_definition tests. + """ + def __init__(self, path): + self.path = path + self.name = os.path.basename(path) + match = re.search(r'python([\d.]+)\.py', self.name) + self.python_version = match and match.group(1) + + +def colllect_normalizer_tests(base_dir): + for f_name in os.listdir(base_dir): + if f_name.endswith(".py"): + path = os.path.join(base_dir, f_name) + yield NormalizerIssueCase(path) + + +def pytest_configure(config): + if config.option.logging: + root = logging.getLogger() + root.setLevel(logging.DEBUG) + + #ch = logging.StreamHandler(sys.stdout) + #ch.setLevel(logging.DEBUG) + #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + #ch.setFormatter(formatter) + + #root.addHandler(ch) + + +class Checker: + def __init__(self, version, is_passing): + self.version = version + self._is_passing = is_passing + self.grammar = parso.load_grammar(version=self.version) + + def parse(self, code): + if self._is_passing: + return parso.parse(code, version=self.version, error_recovery=False) + else: + self._invalid_syntax(code) + + def _invalid_syntax(self, code): + with pytest.raises(parso.ParserSyntaxError): + module = parso.parse(code, version=self.version, error_recovery=False) + # For debugging + print(module.children) + + def get_error(self, code): + errors = list(self.grammar.iter_errors(self.grammar.parse(code))) + assert bool(errors) != self._is_passing + if errors: + return errors[0] + + def get_error_message(self, code): + error = self.get_error(code) + if error is None: + return + return error.message + + def assert_no_error_in_passing(self, code): + if self._is_passing: + module = self.grammar.parse(code) + assert not list(self.grammar.iter_errors(module)) + + +@pytest.fixture +def works_not_in_py(each_version): + return Checker(each_version, False) + + +@pytest.fixture +def works_in_py(each_version): + return Checker(each_version, True) + + +@pytest.fixture +def works_ge_py38(each_version): + version_info = parse_version_string(each_version) + return Checker(each_version, version_info >= (3, 8)) + + +@pytest.fixture +def works_ge_py39(each_version): + version_info = parse_version_string(each_version) + return Checker(each_version, version_info >= (3, 9)) diff --git a/contrib/python/parso/py3/tests/failing_examples.py b/contrib/python/parso/py3/tests/failing_examples.py index 09714d3902..b49abc03e7 100644 --- a/contrib/python/parso/py3/tests/failing_examples.py +++ b/contrib/python/parso/py3/tests/failing_examples.py @@ -1,415 +1,415 @@ -# -*- coding: utf-8 -*- -import sys -from textwrap import dedent - - -def indent(code): - lines = code.splitlines(True) - return ''.join([' ' * 2 + line for line in lines]) - - -def build_nested(code, depth, base='def f():\n'): - if depth == 0: - return code - - new_code = base + indent(code) - return build_nested(new_code, depth - 1, base=base) - - -FAILING_EXAMPLES = [ - '1 +', - '?', - 'continue', - 'break', - 'return', - 'yield', - - # SyntaxError from Python/ast.c - 'f(x for x in bar, 1)', - 'from foo import a,', - 'from __future__ import whatever', - 'from __future__ import braces', - 'from .__future__ import whatever', - 'def f(x=3, y): pass', - 'lambda x=3, y: x', - '__debug__ = 1', - 'with x() as __debug__: pass', - - '[]: int', - '[a, b]: int', - '(): int', - '(()): int', - '((())): int', - '{}: int', - 'True: int', - '(a, b): int', - '*star,: int', - 'a, b: int = 3', - 'foo(+a=3)', - 'f(lambda: 1=1)', - 'f(x=1, x=2)', - 'f(**x, y)', - 'f(x=2, y)', - 'f(**x, *y)', - 'f(**x, y=3, z)', - # augassign - 'a, b += 3', - '(a, b) += 3', - '[a, b] += 3', - '[a, 1] += 3', - 'f() += 1', - 'lambda x:None+=1', - '{} += 1', - '{a:b} += 1', - '{1} += 1', - '{*x} += 1', - '(x,) += 1', - '(x, y if a else q) += 1', - '[] += 1', - '[1,2] += 1', - '[] += 1', - 'None += 1', - '... += 1', - 'a > 1 += 1', - '"test" += 1', - '1 += 1', - '1.0 += 1', - '(yield) += 1', - '(yield from x) += 1', - '(x if x else y) += 1', - 'a() += 1', - 'a + b += 1', - '+a += 1', - 'a and b += 1', - '*a += 1', - 'a, b += 1', - 'f"xxx" += 1', - # All assignment tests - 'lambda a: 1 = 1', - '[x for x in y] = 1', - '{x for x in y} = 1', - '{x:x for x in y} = 1', - '(x for x in y) = 1', - 'None = 1', - '... = 1', - 'a == b = 1', - '{a, b} = 1', - '{a: b} = 1', - '1 = 1', - '"" = 1', - 'b"" = 1', - 'b"" = 1', - '"" "" = 1', - '1 | 1 = 3', - '1**1 = 3', - '~ 1 = 3', - 'not 1 = 3', - '1 and 1 = 3', - 'def foo(): (yield 1) = 3', - 'def foo(): x = yield 1 = 3', - 'async def foo(): await x = 3', - '(a if a else a) = a', - 'a, 1 = x', - 'foo() = 1', - # Cases without the equals but other assignments. - 'with x as foo(): pass', - 'del bar, 1', - 'for x, 1 in []: pass', - 'for (not 1) in []: pass', - '[x for 1 in y]', - '[x for a, 3 in y]', - '(x for 1 in y)', - '{x for 1 in y}', - '{x:x for 1 in y}', - # Unicode/Bytes issues. - r'u"\x"', - r'u"\"', - r'u"\u"', - r'u"""\U"""', - r'u"\Uffffffff"', - r"u'''\N{}'''", - r"u'\N{foo}'", - r'b"\x"', - r'b"\"', - 'b"ä"', - - '*a, *b = 3, 3', - 'async def foo(): yield from []', - 'yield from []', - '*a = 3', - 'del *a, b', - 'def x(*): pass', - '(%s *d) = x' % ('a,' * 256), - '{**{} for a in [1]}', - '(True,) = x', - '([False], a) = x', - 'def x(): from math import *', - - # invalid del statements - 'del x + y', - 'del x(y)', - 'async def foo(): del await x', - 'def foo(): del (yield x)', - 'del [x for x in range(10)]', - 'del *x', - 'del *x,', - 'del (*x,)', - 'del [*x]', - 'del x, *y', - 'del *x.y,', - 'del *x[y],', - 'del *x[y::], z', - 'del x, (y, *z)', - 'del (x, *[y, z])', - 'del [x, *(y, [*z])]', - 'del {}', - 'del {x}', - 'del {x, y}', - 'del {x, *y}', - - # invalid starred expressions - '*x', - '(*x)', - '((*x))', - '1 + (*x)', - '*x; 1', - '1; *x', - '1\n*x', - 'x = *y', - 'x: int = *y', - 'def foo(): return *x', - 'def foo(): yield *x', - 'f"{*x}"', - 'for *x in 1: pass', - '[1 for *x in 1]', - - # str/bytes combinations - '"s" b""', - '"s" b"" ""', - 'b"" "" b"" ""', - 'f"s" b""', - 'b"s" f""', - - # Parser/tokenize.c - r'"""', - r'"', - r"'''", - r"'", - r"\blub", - # IndentationError: too many levels of indentation - build_nested('pass', 100), - - # SyntaxErrors from Python/symtable.c - 'def f(x, x): pass', - 'nonlocal a', - - # IndentationError - ' foo', - 'def x():\n 1\n 2', - 'def x():\n 1\n 2', - 'if 1:\nfoo', - 'if 1: blubb\nif 1:\npass\nTrue and False', - - # f-strings - 'f"{}"', - r'f"{\}"', - 'f"{\'\\\'}"', - 'f"{#}"', - "f'{1!b}'", - "f'{1:{5:{3}}}'", - "f'{'", - "f'{'", - "f'}'", - "f'{\"}'", - "f'{\"}'", - # Now nested parsing - "f'{continue}'", - "f'{1;1}'", - "f'{a;}'", - "f'{b\"\" \"\"}'", - # f-string expression part cannot include a backslash - r'''f"{'\n'}"''', - - 'async def foo():\n yield x\n return 1', - 'async def foo():\n yield x\n return 1', - - '[*[] for a in [1]]', - 'async def bla():\n def x(): await bla()', - 'del None', - 'del True', - 'del False', - 'del ...', - - # Errors of global / nonlocal - dedent(''' - def glob(): - x = 3 - x.z - global x'''), - dedent(''' - def glob(): - x = 3 - global x'''), - dedent(''' - def glob(): - x - global x'''), - dedent(''' - def glob(): - x = 3 - x.z - nonlocal x'''), - dedent(''' - def glob(): - x = 3 - nonlocal x'''), - dedent(''' - def glob(): - x - nonlocal x'''), - # Annotation issues - dedent(''' - def glob(): - x[0]: foo - global x'''), - dedent(''' - def glob(): - x.a: foo - global x'''), - dedent(''' - def glob(): - x: foo - global x'''), - dedent(''' - def glob(): - x: foo = 5 - global x'''), - dedent(''' - def glob(): - x: foo = 5 - x - global x'''), - dedent(''' - def glob(): - global x - x: foo = 3 - '''), - # global/nonlocal + param - dedent(''' - def glob(x): - global x - '''), - dedent(''' - def glob(x): - nonlocal x - '''), - dedent(''' - def x(): - a =3 - def z(): - nonlocal a - a = 3 - nonlocal a - '''), - dedent(''' - def x(): - a = 4 - def y(): - global a - nonlocal a - '''), - # Missing binding of nonlocal - dedent(''' - def x(): - nonlocal a - '''), - dedent(''' - def x(): - def y(): - nonlocal a - '''), - dedent(''' - def x(): - a = 4 - def y(): - global a - print(a) - def z(): - nonlocal a - '''), - # Name is assigned before nonlocal declaration - dedent(''' - def x(a): - def y(): - a = 10 - nonlocal a - '''), -] - -if sys.version_info[:2] >= (3, 7): - # This is somehow ok in previous versions. - FAILING_EXAMPLES += [ - 'class X(base for base in bases): pass', - ] - -if sys.version_info[:2] < (3, 8): - FAILING_EXAMPLES += [ - # Python/compile.c - dedent('''\ - for a in [1]: - try: - pass - finally: - continue - '''), # 'continue' not supported inside 'finally' clause" - ] - -if sys.version_info[:2] >= (3, 8): - # assignment expressions from issue#89 - FAILING_EXAMPLES += [ - # Case 2 - '(lambda: x := 1)', - '((lambda: x) := 1)', - # Case 3 - '(a[i] := x)', - '((a[i]) := x)', - '(a(i) := x)', - # Case 4 - '(a.b := c)', - '[(i.i:= 0) for ((i), j) in range(5)]', - # Case 5 - '[i:= 0 for i, j in range(5)]', - '[(i:= 0) for ((i), j) in range(5)]', - '[(i:= 0) for ((i), j), in range(5)]', - '[(i:= 0) for ((i), j.i), in range(5)]', - '[[(i:= i) for j in range(5)] for i in range(5)]', - '[i for i, j in range(5) if True or (i:= 1)]', - '[False and (i:= 0) for i, j in range(5)]', - # Case 6 - '[i+1 for i in (i:= range(5))]', - '[i+1 for i in (j:= range(5))]', - '[i+1 for i in (lambda: (j:= range(5)))()]', - # Case 7 - 'class Example:\n [(j := i) for i in range(5)]', - # Not in that issue - '(await a := x)', - '((await a) := x)', - # new discoveries - '((a, b) := (1, 2))', - '([a, b] := [1, 2])', - '({a, b} := {1, 2})', - '({a: b} := {1: 2})', - '(a + b := 1)', - '(True := 1)', - '(False := 1)', - '(None := 1)', - '(__debug__ := 1)', - # Unparenthesized walrus not allowed in dict literals, dict comprehensions and slices - '{a:="a": b:=1}', - '{y:=1: 2 for x in range(5)}', - 'a[b:=0:1:2]', - ] - # f-string debugging syntax with invalid conversion character - FAILING_EXAMPLES += [ - "f'{1=!b}'", - ] +# -*- coding: utf-8 -*- +import sys +from textwrap import dedent + + +def indent(code): + lines = code.splitlines(True) + return ''.join([' ' * 2 + line for line in lines]) + + +def build_nested(code, depth, base='def f():\n'): + if depth == 0: + return code + + new_code = base + indent(code) + return build_nested(new_code, depth - 1, base=base) + + +FAILING_EXAMPLES = [ + '1 +', + '?', + 'continue', + 'break', + 'return', + 'yield', + + # SyntaxError from Python/ast.c + 'f(x for x in bar, 1)', + 'from foo import a,', + 'from __future__ import whatever', + 'from __future__ import braces', + 'from .__future__ import whatever', + 'def f(x=3, y): pass', + 'lambda x=3, y: x', + '__debug__ = 1', + 'with x() as __debug__: pass', + + '[]: int', + '[a, b]: int', + '(): int', + '(()): int', + '((())): int', + '{}: int', + 'True: int', + '(a, b): int', + '*star,: int', + 'a, b: int = 3', + 'foo(+a=3)', + 'f(lambda: 1=1)', + 'f(x=1, x=2)', + 'f(**x, y)', + 'f(x=2, y)', + 'f(**x, *y)', + 'f(**x, y=3, z)', + # augassign + 'a, b += 3', + '(a, b) += 3', + '[a, b] += 3', + '[a, 1] += 3', + 'f() += 1', + 'lambda x:None+=1', + '{} += 1', + '{a:b} += 1', + '{1} += 1', + '{*x} += 1', + '(x,) += 1', + '(x, y if a else q) += 1', + '[] += 1', + '[1,2] += 1', + '[] += 1', + 'None += 1', + '... += 1', + 'a > 1 += 1', + '"test" += 1', + '1 += 1', + '1.0 += 1', + '(yield) += 1', + '(yield from x) += 1', + '(x if x else y) += 1', + 'a() += 1', + 'a + b += 1', + '+a += 1', + 'a and b += 1', + '*a += 1', + 'a, b += 1', + 'f"xxx" += 1', + # All assignment tests + 'lambda a: 1 = 1', + '[x for x in y] = 1', + '{x for x in y} = 1', + '{x:x for x in y} = 1', + '(x for x in y) = 1', + 'None = 1', + '... = 1', + 'a == b = 1', + '{a, b} = 1', + '{a: b} = 1', + '1 = 1', + '"" = 1', + 'b"" = 1', + 'b"" = 1', + '"" "" = 1', + '1 | 1 = 3', + '1**1 = 3', + '~ 1 = 3', + 'not 1 = 3', + '1 and 1 = 3', + 'def foo(): (yield 1) = 3', + 'def foo(): x = yield 1 = 3', + 'async def foo(): await x = 3', + '(a if a else a) = a', + 'a, 1 = x', + 'foo() = 1', + # Cases without the equals but other assignments. + 'with x as foo(): pass', + 'del bar, 1', + 'for x, 1 in []: pass', + 'for (not 1) in []: pass', + '[x for 1 in y]', + '[x for a, 3 in y]', + '(x for 1 in y)', + '{x for 1 in y}', + '{x:x for 1 in y}', + # Unicode/Bytes issues. + r'u"\x"', + r'u"\"', + r'u"\u"', + r'u"""\U"""', + r'u"\Uffffffff"', + r"u'''\N{}'''", + r"u'\N{foo}'", + r'b"\x"', + r'b"\"', + 'b"ä"', + + '*a, *b = 3, 3', + 'async def foo(): yield from []', + 'yield from []', + '*a = 3', + 'del *a, b', + 'def x(*): pass', + '(%s *d) = x' % ('a,' * 256), + '{**{} for a in [1]}', + '(True,) = x', + '([False], a) = x', + 'def x(): from math import *', + + # invalid del statements + 'del x + y', + 'del x(y)', + 'async def foo(): del await x', + 'def foo(): del (yield x)', + 'del [x for x in range(10)]', + 'del *x', + 'del *x,', + 'del (*x,)', + 'del [*x]', + 'del x, *y', + 'del *x.y,', + 'del *x[y],', + 'del *x[y::], z', + 'del x, (y, *z)', + 'del (x, *[y, z])', + 'del [x, *(y, [*z])]', + 'del {}', + 'del {x}', + 'del {x, y}', + 'del {x, *y}', + + # invalid starred expressions + '*x', + '(*x)', + '((*x))', + '1 + (*x)', + '*x; 1', + '1; *x', + '1\n*x', + 'x = *y', + 'x: int = *y', + 'def foo(): return *x', + 'def foo(): yield *x', + 'f"{*x}"', + 'for *x in 1: pass', + '[1 for *x in 1]', + + # str/bytes combinations + '"s" b""', + '"s" b"" ""', + 'b"" "" b"" ""', + 'f"s" b""', + 'b"s" f""', + + # Parser/tokenize.c + r'"""', + r'"', + r"'''", + r"'", + r"\blub", + # IndentationError: too many levels of indentation + build_nested('pass', 100), + + # SyntaxErrors from Python/symtable.c + 'def f(x, x): pass', + 'nonlocal a', + + # IndentationError + ' foo', + 'def x():\n 1\n 2', + 'def x():\n 1\n 2', + 'if 1:\nfoo', + 'if 1: blubb\nif 1:\npass\nTrue and False', + + # f-strings + 'f"{}"', + r'f"{\}"', + 'f"{\'\\\'}"', + 'f"{#}"', + "f'{1!b}'", + "f'{1:{5:{3}}}'", + "f'{'", + "f'{'", + "f'}'", + "f'{\"}'", + "f'{\"}'", + # Now nested parsing + "f'{continue}'", + "f'{1;1}'", + "f'{a;}'", + "f'{b\"\" \"\"}'", + # f-string expression part cannot include a backslash + r'''f"{'\n'}"''', + + 'async def foo():\n yield x\n return 1', + 'async def foo():\n yield x\n return 1', + + '[*[] for a in [1]]', + 'async def bla():\n def x(): await bla()', + 'del None', + 'del True', + 'del False', + 'del ...', + + # Errors of global / nonlocal + dedent(''' + def glob(): + x = 3 + x.z + global x'''), + dedent(''' + def glob(): + x = 3 + global x'''), + dedent(''' + def glob(): + x + global x'''), + dedent(''' + def glob(): + x = 3 + x.z + nonlocal x'''), + dedent(''' + def glob(): + x = 3 + nonlocal x'''), + dedent(''' + def glob(): + x + nonlocal x'''), + # Annotation issues + dedent(''' + def glob(): + x[0]: foo + global x'''), + dedent(''' + def glob(): + x.a: foo + global x'''), + dedent(''' + def glob(): + x: foo + global x'''), + dedent(''' + def glob(): + x: foo = 5 + global x'''), + dedent(''' + def glob(): + x: foo = 5 + x + global x'''), + dedent(''' + def glob(): + global x + x: foo = 3 + '''), + # global/nonlocal + param + dedent(''' + def glob(x): + global x + '''), + dedent(''' + def glob(x): + nonlocal x + '''), + dedent(''' + def x(): + a =3 + def z(): + nonlocal a + a = 3 + nonlocal a + '''), + dedent(''' + def x(): + a = 4 + def y(): + global a + nonlocal a + '''), + # Missing binding of nonlocal + dedent(''' + def x(): + nonlocal a + '''), + dedent(''' + def x(): + def y(): + nonlocal a + '''), + dedent(''' + def x(): + a = 4 + def y(): + global a + print(a) + def z(): + nonlocal a + '''), + # Name is assigned before nonlocal declaration + dedent(''' + def x(a): + def y(): + a = 10 + nonlocal a + '''), +] + +if sys.version_info[:2] >= (3, 7): + # This is somehow ok in previous versions. + FAILING_EXAMPLES += [ + 'class X(base for base in bases): pass', + ] + +if sys.version_info[:2] < (3, 8): + FAILING_EXAMPLES += [ + # Python/compile.c + dedent('''\ + for a in [1]: + try: + pass + finally: + continue + '''), # 'continue' not supported inside 'finally' clause" + ] + +if sys.version_info[:2] >= (3, 8): + # assignment expressions from issue#89 + FAILING_EXAMPLES += [ + # Case 2 + '(lambda: x := 1)', + '((lambda: x) := 1)', + # Case 3 + '(a[i] := x)', + '((a[i]) := x)', + '(a(i) := x)', + # Case 4 + '(a.b := c)', + '[(i.i:= 0) for ((i), j) in range(5)]', + # Case 5 + '[i:= 0 for i, j in range(5)]', + '[(i:= 0) for ((i), j) in range(5)]', + '[(i:= 0) for ((i), j), in range(5)]', + '[(i:= 0) for ((i), j.i), in range(5)]', + '[[(i:= i) for j in range(5)] for i in range(5)]', + '[i for i, j in range(5) if True or (i:= 1)]', + '[False and (i:= 0) for i, j in range(5)]', + # Case 6 + '[i+1 for i in (i:= range(5))]', + '[i+1 for i in (j:= range(5))]', + '[i+1 for i in (lambda: (j:= range(5)))()]', + # Case 7 + 'class Example:\n [(j := i) for i in range(5)]', + # Not in that issue + '(await a := x)', + '((await a) := x)', + # new discoveries + '((a, b) := (1, 2))', + '([a, b] := [1, 2])', + '({a, b} := {1, 2})', + '({a: b} := {1: 2})', + '(a + b := 1)', + '(True := 1)', + '(False := 1)', + '(None := 1)', + '(__debug__ := 1)', + # Unparenthesized walrus not allowed in dict literals, dict comprehensions and slices + '{a:="a": b:=1}', + '{y:=1: 2 for x in range(5)}', + 'a[b:=0:1:2]', + ] + # f-string debugging syntax with invalid conversion character + FAILING_EXAMPLES += [ + "f'{1=!b}'", + ] diff --git a/contrib/python/parso/py3/tests/fuzz_diff_parser.py b/contrib/python/parso/py3/tests/fuzz_diff_parser.py index 39b93f21d5..cbf2b1e0e4 100644 --- a/contrib/python/parso/py3/tests/fuzz_diff_parser.py +++ b/contrib/python/parso/py3/tests/fuzz_diff_parser.py @@ -1,307 +1,307 @@ -""" -A script to find bugs in the diff parser. - -This script is extremely useful if changes are made to the diff parser. By -running a few thousand iterations, we can assure that the diff parser is in -good shape. - -Usage: - fuzz_diff_parser.py [--pdb|--ipdb] [-l] [-n=<nr>] [-x=<nr>] random [<path>] - fuzz_diff_parser.py [--pdb|--ipdb] [-l] redo [-o=<nr>] [-p] - fuzz_diff_parser.py -h | --help - -Options: - -h --help Show this screen - -n, --maxtries=<nr> Maximum of random tries [default: 1000] - -x, --changes=<nr> Amount of changes to be done to a file per try [default: 5] - -l, --logging Prints all the logs - -o, --only-last=<nr> Only runs the last n iterations; Defaults to running all - -p, --print-code Print all test diffs - --pdb Launch pdb when error is raised - --ipdb Launch ipdb when error is raised -""" - -from __future__ import print_function -import logging -import sys -import os -import random -import pickle - -import parso -from parso.utils import split_lines -from test.test_diff_parser import _check_error_leaves_nodes - -_latest_grammar = parso.load_grammar(version='3.8') -_python_reserved_strings = tuple( - # Keywords are ususally only interesting in combination with spaces after - # them. We don't put a space before keywords, to avoid indentation errors. - s + (' ' if s.isalpha() else '') - for s in _latest_grammar._pgen_grammar.reserved_syntax_strings.keys() -) -_random_python_fragments = _python_reserved_strings + ( - ' ', '\t', '\n', '\r', '\f', 'f"', 'F"""', "fr'", "RF'''", '"', '"""', "'", - "'''", ';', ' some_random_word ', '\\', '#', -) - - -def find_python_files_in_tree(file_path): - if not os.path.isdir(file_path): - yield file_path - return - for root, dirnames, filenames in os.walk(file_path): - if 'chardet' in root: - # Stuff like chardet/langcyrillicmodel.py is just very slow to - # parse and machine generated, so ignore those. - continue - - for name in filenames: - if name.endswith('.py'): - yield os.path.join(root, name) - - -def _print_copyable_lines(lines): - for line in lines: - line = repr(line)[1:-1] - if line.endswith(r'\n'): - line = line[:-2] + '\n' - print(line, end='') - - -def _get_first_error_start_pos_or_none(module): - error_leaf = _check_error_leaves_nodes(module) - return None if error_leaf is None else error_leaf.start_pos - - -class LineReplacement: - def __init__(self, line_nr, new_line): - self._line_nr = line_nr - self._new_line = new_line - - def apply(self, code_lines): - # print(repr(self._new_line)) - code_lines[self._line_nr] = self._new_line - - -class LineDeletion: - def __init__(self, line_nr): - self.line_nr = line_nr - - def apply(self, code_lines): - del code_lines[self.line_nr] - - -class LineCopy: - def __init__(self, copy_line, insertion_line): - self._copy_line = copy_line - self._insertion_line = insertion_line - - def apply(self, code_lines): - code_lines.insert( - self._insertion_line, - # Use some line from the file. This doesn't feel totally - # random, but for the diff parser it will feel like it. - code_lines[self._copy_line] - ) - - -class FileModification: - @classmethod - def generate(cls, code_lines, change_count, previous_file_modification=None): - if previous_file_modification is not None and random.random() > 0.5: - # We want to keep the previous modifications in some cases to make - # more complex parser issues visible. - code_lines = previous_file_modification.apply(code_lines) - added_modifications = previous_file_modification.modification_list - else: - added_modifications = [] - return cls( - added_modifications - + list(cls._generate_line_modifications(code_lines, change_count)), - # work with changed trees more than with normal ones. - check_original=random.random() > 0.8, - ) - - @staticmethod - def _generate_line_modifications(lines, change_count): - def random_line(include_end=False): - return random.randint(0, len(lines) - (not include_end)) - - lines = list(lines) - for _ in range(change_count): - rand = random.randint(1, 4) - if rand == 1: - if len(lines) == 1: - # We cannot delete every line, that doesn't make sense to - # fuzz and it would be annoying to rewrite everything here. - continue - ld = LineDeletion(random_line()) - elif rand == 2: - # Copy / Insertion - # Make it possible to insert into the first and the last line - ld = LineCopy(random_line(), random_line(include_end=True)) - elif rand in (3, 4): - # Modify a line in some weird random ways. - line_nr = random_line() - line = lines[line_nr] - column = random.randint(0, len(line)) - random_string = '' - for _ in range(random.randint(1, 3)): - if random.random() > 0.8: - # The lower characters cause way more issues. - unicode_range = 0x1f if random.randint(0, 1) else 0x3000 - random_string += chr(random.randint(0, unicode_range)) - else: - # These insertions let us understand how random - # keyword/operator insertions work. Theoretically this - # could also be done with unicode insertions, but the - # fuzzer is just way more effective here. - random_string += random.choice(_random_python_fragments) - if random.random() > 0.5: - # In this case we insert at a very random place that - # probably breaks syntax. - line = line[:column] + random_string + line[column:] - else: - # Here we have better chances to not break syntax, because - # we really replace the line with something that has - # indentation. - line = ' ' * random.randint(0, 12) + random_string + '\n' - ld = LineReplacement(line_nr, line) - ld.apply(lines) - yield ld - - def __init__(self, modification_list, check_original): - self.modification_list = modification_list - self._check_original = check_original - - def apply(self, code_lines): - changed_lines = list(code_lines) - for modification in self.modification_list: - modification.apply(changed_lines) - return changed_lines - - def run(self, grammar, code_lines, print_code): - code = ''.join(code_lines) - modified_lines = self.apply(code_lines) - modified_code = ''.join(modified_lines) - - if print_code: - if self._check_original: - print('Original:') - _print_copyable_lines(code_lines) - - print('\nModified:') - _print_copyable_lines(modified_lines) - print() - - if self._check_original: - m = grammar.parse(code, diff_cache=True) - start1 = _get_first_error_start_pos_or_none(m) - - grammar.parse(modified_code, diff_cache=True) - - if self._check_original: - # Also check if it's possible to "revert" the changes. - m = grammar.parse(code, diff_cache=True) - start2 = _get_first_error_start_pos_or_none(m) - assert start1 == start2, (start1, start2) - - -class FileTests: - def __init__(self, file_path, test_count, change_count): - self._path = file_path - with open(file_path, errors='replace') as f: - code = f.read() - self._code_lines = split_lines(code, keepends=True) - self._test_count = test_count - self._code_lines = self._code_lines - self._change_count = change_count - self._file_modifications = [] - - def _run(self, grammar, file_modifications, debugger, print_code=False): - try: - for i, fm in enumerate(file_modifications, 1): - fm.run(grammar, self._code_lines, print_code=print_code) - print('.', end='') - sys.stdout.flush() - print() - except Exception: - print("Issue in file: %s" % self._path) - if debugger: - einfo = sys.exc_info() - pdb = __import__(debugger) - pdb.post_mortem(einfo[2]) - raise - - def redo(self, grammar, debugger, only_last, print_code): - mods = self._file_modifications - if only_last is not None: - mods = mods[-only_last:] - self._run(grammar, mods, debugger, print_code=print_code) - - def run(self, grammar, debugger): - def iterate(): - fm = None - for _ in range(self._test_count): - fm = FileModification.generate( - self._code_lines, self._change_count, - previous_file_modification=fm - ) - self._file_modifications.append(fm) - yield fm - - self._run(grammar, iterate(), debugger) - - -def main(arguments): - debugger = 'pdb' if arguments['--pdb'] else \ - 'ipdb' if arguments['--ipdb'] else None - redo_file = os.path.join(os.path.dirname(__file__), 'fuzz-redo.pickle') - - if arguments['--logging']: - root = logging.getLogger() - root.setLevel(logging.DEBUG) - - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.DEBUG) - root.addHandler(ch) - - grammar = parso.load_grammar() - parso.python.diff.DEBUG_DIFF_PARSER = True - if arguments['redo']: - with open(redo_file, 'rb') as f: - file_tests_obj = pickle.load(f) - only_last = arguments['--only-last'] and int(arguments['--only-last']) - file_tests_obj.redo( - grammar, - debugger, - only_last=only_last, - print_code=arguments['--print-code'] - ) - elif arguments['random']: - # A random file is used to do diff parser checks if no file is given. - # This helps us to find errors in a lot of different files. - file_paths = list(find_python_files_in_tree(arguments['<path>'] or '.')) - max_tries = int(arguments['--maxtries']) - tries = 0 - try: - while tries < max_tries: - path = random.choice(file_paths) - print("Checking %s: %s tries" % (path, tries)) - now_tries = min(1000, max_tries - tries) - file_tests_obj = FileTests(path, now_tries, int(arguments['--changes'])) - file_tests_obj.run(grammar, debugger) - tries += now_tries - except Exception: - with open(redo_file, 'wb') as f: - pickle.dump(file_tests_obj, f) - raise - else: - raise NotImplementedError('Command is not implemented') - - -if __name__ == '__main__': - from docopt import docopt - - arguments = docopt(__doc__) - main(arguments) +""" +A script to find bugs in the diff parser. + +This script is extremely useful if changes are made to the diff parser. By +running a few thousand iterations, we can assure that the diff parser is in +good shape. + +Usage: + fuzz_diff_parser.py [--pdb|--ipdb] [-l] [-n=<nr>] [-x=<nr>] random [<path>] + fuzz_diff_parser.py [--pdb|--ipdb] [-l] redo [-o=<nr>] [-p] + fuzz_diff_parser.py -h | --help + +Options: + -h --help Show this screen + -n, --maxtries=<nr> Maximum of random tries [default: 1000] + -x, --changes=<nr> Amount of changes to be done to a file per try [default: 5] + -l, --logging Prints all the logs + -o, --only-last=<nr> Only runs the last n iterations; Defaults to running all + -p, --print-code Print all test diffs + --pdb Launch pdb when error is raised + --ipdb Launch ipdb when error is raised +""" + +from __future__ import print_function +import logging +import sys +import os +import random +import pickle + +import parso +from parso.utils import split_lines +from test.test_diff_parser import _check_error_leaves_nodes + +_latest_grammar = parso.load_grammar(version='3.8') +_python_reserved_strings = tuple( + # Keywords are ususally only interesting in combination with spaces after + # them. We don't put a space before keywords, to avoid indentation errors. + s + (' ' if s.isalpha() else '') + for s in _latest_grammar._pgen_grammar.reserved_syntax_strings.keys() +) +_random_python_fragments = _python_reserved_strings + ( + ' ', '\t', '\n', '\r', '\f', 'f"', 'F"""', "fr'", "RF'''", '"', '"""', "'", + "'''", ';', ' some_random_word ', '\\', '#', +) + + +def find_python_files_in_tree(file_path): + if not os.path.isdir(file_path): + yield file_path + return + for root, dirnames, filenames in os.walk(file_path): + if 'chardet' in root: + # Stuff like chardet/langcyrillicmodel.py is just very slow to + # parse and machine generated, so ignore those. + continue + + for name in filenames: + if name.endswith('.py'): + yield os.path.join(root, name) + + +def _print_copyable_lines(lines): + for line in lines: + line = repr(line)[1:-1] + if line.endswith(r'\n'): + line = line[:-2] + '\n' + print(line, end='') + + +def _get_first_error_start_pos_or_none(module): + error_leaf = _check_error_leaves_nodes(module) + return None if error_leaf is None else error_leaf.start_pos + + +class LineReplacement: + def __init__(self, line_nr, new_line): + self._line_nr = line_nr + self._new_line = new_line + + def apply(self, code_lines): + # print(repr(self._new_line)) + code_lines[self._line_nr] = self._new_line + + +class LineDeletion: + def __init__(self, line_nr): + self.line_nr = line_nr + + def apply(self, code_lines): + del code_lines[self.line_nr] + + +class LineCopy: + def __init__(self, copy_line, insertion_line): + self._copy_line = copy_line + self._insertion_line = insertion_line + + def apply(self, code_lines): + code_lines.insert( + self._insertion_line, + # Use some line from the file. This doesn't feel totally + # random, but for the diff parser it will feel like it. + code_lines[self._copy_line] + ) + + +class FileModification: + @classmethod + def generate(cls, code_lines, change_count, previous_file_modification=None): + if previous_file_modification is not None and random.random() > 0.5: + # We want to keep the previous modifications in some cases to make + # more complex parser issues visible. + code_lines = previous_file_modification.apply(code_lines) + added_modifications = previous_file_modification.modification_list + else: + added_modifications = [] + return cls( + added_modifications + + list(cls._generate_line_modifications(code_lines, change_count)), + # work with changed trees more than with normal ones. + check_original=random.random() > 0.8, + ) + + @staticmethod + def _generate_line_modifications(lines, change_count): + def random_line(include_end=False): + return random.randint(0, len(lines) - (not include_end)) + + lines = list(lines) + for _ in range(change_count): + rand = random.randint(1, 4) + if rand == 1: + if len(lines) == 1: + # We cannot delete every line, that doesn't make sense to + # fuzz and it would be annoying to rewrite everything here. + continue + ld = LineDeletion(random_line()) + elif rand == 2: + # Copy / Insertion + # Make it possible to insert into the first and the last line + ld = LineCopy(random_line(), random_line(include_end=True)) + elif rand in (3, 4): + # Modify a line in some weird random ways. + line_nr = random_line() + line = lines[line_nr] + column = random.randint(0, len(line)) + random_string = '' + for _ in range(random.randint(1, 3)): + if random.random() > 0.8: + # The lower characters cause way more issues. + unicode_range = 0x1f if random.randint(0, 1) else 0x3000 + random_string += chr(random.randint(0, unicode_range)) + else: + # These insertions let us understand how random + # keyword/operator insertions work. Theoretically this + # could also be done with unicode insertions, but the + # fuzzer is just way more effective here. + random_string += random.choice(_random_python_fragments) + if random.random() > 0.5: + # In this case we insert at a very random place that + # probably breaks syntax. + line = line[:column] + random_string + line[column:] + else: + # Here we have better chances to not break syntax, because + # we really replace the line with something that has + # indentation. + line = ' ' * random.randint(0, 12) + random_string + '\n' + ld = LineReplacement(line_nr, line) + ld.apply(lines) + yield ld + + def __init__(self, modification_list, check_original): + self.modification_list = modification_list + self._check_original = check_original + + def apply(self, code_lines): + changed_lines = list(code_lines) + for modification in self.modification_list: + modification.apply(changed_lines) + return changed_lines + + def run(self, grammar, code_lines, print_code): + code = ''.join(code_lines) + modified_lines = self.apply(code_lines) + modified_code = ''.join(modified_lines) + + if print_code: + if self._check_original: + print('Original:') + _print_copyable_lines(code_lines) + + print('\nModified:') + _print_copyable_lines(modified_lines) + print() + + if self._check_original: + m = grammar.parse(code, diff_cache=True) + start1 = _get_first_error_start_pos_or_none(m) + + grammar.parse(modified_code, diff_cache=True) + + if self._check_original: + # Also check if it's possible to "revert" the changes. + m = grammar.parse(code, diff_cache=True) + start2 = _get_first_error_start_pos_or_none(m) + assert start1 == start2, (start1, start2) + + +class FileTests: + def __init__(self, file_path, test_count, change_count): + self._path = file_path + with open(file_path, errors='replace') as f: + code = f.read() + self._code_lines = split_lines(code, keepends=True) + self._test_count = test_count + self._code_lines = self._code_lines + self._change_count = change_count + self._file_modifications = [] + + def _run(self, grammar, file_modifications, debugger, print_code=False): + try: + for i, fm in enumerate(file_modifications, 1): + fm.run(grammar, self._code_lines, print_code=print_code) + print('.', end='') + sys.stdout.flush() + print() + except Exception: + print("Issue in file: %s" % self._path) + if debugger: + einfo = sys.exc_info() + pdb = __import__(debugger) + pdb.post_mortem(einfo[2]) + raise + + def redo(self, grammar, debugger, only_last, print_code): + mods = self._file_modifications + if only_last is not None: + mods = mods[-only_last:] + self._run(grammar, mods, debugger, print_code=print_code) + + def run(self, grammar, debugger): + def iterate(): + fm = None + for _ in range(self._test_count): + fm = FileModification.generate( + self._code_lines, self._change_count, + previous_file_modification=fm + ) + self._file_modifications.append(fm) + yield fm + + self._run(grammar, iterate(), debugger) + + +def main(arguments): + debugger = 'pdb' if arguments['--pdb'] else \ + 'ipdb' if arguments['--ipdb'] else None + redo_file = os.path.join(os.path.dirname(__file__), 'fuzz-redo.pickle') + + if arguments['--logging']: + root = logging.getLogger() + root.setLevel(logging.DEBUG) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.DEBUG) + root.addHandler(ch) + + grammar = parso.load_grammar() + parso.python.diff.DEBUG_DIFF_PARSER = True + if arguments['redo']: + with open(redo_file, 'rb') as f: + file_tests_obj = pickle.load(f) + only_last = arguments['--only-last'] and int(arguments['--only-last']) + file_tests_obj.redo( + grammar, + debugger, + only_last=only_last, + print_code=arguments['--print-code'] + ) + elif arguments['random']: + # A random file is used to do diff parser checks if no file is given. + # This helps us to find errors in a lot of different files. + file_paths = list(find_python_files_in_tree(arguments['<path>'] or '.')) + max_tries = int(arguments['--maxtries']) + tries = 0 + try: + while tries < max_tries: + path = random.choice(file_paths) + print("Checking %s: %s tries" % (path, tries)) + now_tries = min(1000, max_tries - tries) + file_tests_obj = FileTests(path, now_tries, int(arguments['--changes'])) + file_tests_obj.run(grammar, debugger) + tries += now_tries + except Exception: + with open(redo_file, 'wb') as f: + pickle.dump(file_tests_obj, f) + raise + else: + raise NotImplementedError('Command is not implemented') + + +if __name__ == '__main__': + from docopt import docopt + + arguments = docopt(__doc__) + main(arguments) diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E10.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E10.py index 38d7a19043..396c67923c 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E10.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E10.py @@ -1,51 +1,51 @@ -for a in 'abc': - for b in 'xyz': - hello(a) # indented with 8 spaces - #: E903:0 - hello(b) # indented with 1 tab -if True: - #: E101:0 - pass - -#: E122+1 -change_2_log = \ -"""Change 2 by slamb@testclient on 2006/04/13 21:46:23 - - creation -""" - -p4change = { - 2: change_2_log, -} - - -class TestP4Poller(unittest.TestCase): - def setUp(self): - self.setUpGetProcessOutput() - return self.setUpChangeSource() - - def tearDown(self): - pass - - -# -if True: - #: E101:0 E101+1:0 - foo(1, - 2) - - -def test_keys(self): - """areas.json - All regions are accounted for.""" - expected = set([ - #: E101:0 - u'Norrbotten', - #: E101:0 - u'V\xe4sterbotten', - ]) - - -if True: - hello(""" - tab at start of this line -""") +for a in 'abc': + for b in 'xyz': + hello(a) # indented with 8 spaces + #: E903:0 + hello(b) # indented with 1 tab +if True: + #: E101:0 + pass + +#: E122+1 +change_2_log = \ +"""Change 2 by slamb@testclient on 2006/04/13 21:46:23 + + creation +""" + +p4change = { + 2: change_2_log, +} + + +class TestP4Poller(unittest.TestCase): + def setUp(self): + self.setUpGetProcessOutput() + return self.setUpChangeSource() + + def tearDown(self): + pass + + +# +if True: + #: E101:0 E101+1:0 + foo(1, + 2) + + +def test_keys(self): + """areas.json - All regions are accounted for.""" + expected = set([ + #: E101:0 + u'Norrbotten', + #: E101:0 + u'V\xe4sterbotten', + ]) + + +if True: + hello(""" + tab at start of this line +""") diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E101.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E101.py index cc24719873..bcf125345f 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E101.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E101.py @@ -1,137 +1,137 @@ -# Used to be the file for W191 - -#: E101+1 -if False: - print # indented with 1 tab - -#: E101+1 -y = x == 2 \ - or x == 3 -#: E101+5 -if ( - x == ( - 3 - ) or - y == 4): - pass -#: E101+3 -if x == 2 \ - or y > 1 \ - or x == 3: - pass -#: E101+3 -if x == 2 \ - or y > 1 \ - or x == 3: - pass - -#: E101+1 -if (foo == bar and baz == frop): - pass -#: E101+1 -if (foo == bar and baz == frop): - pass - -#: E101+2 E101+3 -if start[1] > end_col and not ( - over_indent == 4 and indent_next): - assert (0, "E121 continuation line over-" - "indented for visual indent") - - -#: E101+3 -def long_function_name( - var_one, var_two, var_three, - var_four): - hello(var_one) - - -#: E101+2 -if ((row < 0 or self.moduleCount <= row or - col < 0 or self.moduleCount <= col)): - raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) -#: E101+1 E101+2 E101+3 E101+4 E101+5 E101+6 -if bar: - assert ( - start, 'E121 lines starting with a ' - 'closing bracket should be indented ' - "to match that of the opening " - "bracket's line" - ) - -# you want vertical alignment, so use a parens -#: E101+3 -if ((foo.bar("baz") and - foo.bar("frop") - )): - hello("yes") -#: E101+3 -# also ok, but starting to look like LISP -if ((foo.bar("baz") and - foo.bar("frop"))): - hello("yes") -#: E101+1 -if (a == 2 or b == "abc def ghi" "jkl mno"): - assert True -#: E101+2 -if (a == 2 or b == """abc def ghi -jkl mno"""): - assert True -#: E101+1 E101+2 -if length > options.max_line_length: - assert options.max_line_length, \ - "E501 line too long (%d characters)" % length - - -#: E101+1 E101+2 -if os.path.exists(os.path.join(path, PEP8_BIN)): - cmd = ([os.path.join(path, PEP8_BIN)] + - self._pep8_options(targetfile)) -# TODO Tabs in docstrings shouldn't be there, use \t. -''' - multiline string with tab in it''' -# Same here. -'''multiline string - with tabs - and spaces -''' -# Okay -'''sometimes, you just need to go nuts in a multiline string - and allow all sorts of crap - like mixed tabs and spaces - -or trailing whitespace -or long long long long long long long long long long long long long long long long long lines -''' # noqa -# Okay -'''this one - will get no warning -even though the noqa comment is not immediately after the string -''' + foo # noqa - -#: E101+2 -if foo is None and bar is "frop" and \ - blah == 'yeah': - blah = 'yeahnah' - - -#: E101+1 E101+2 E101+3 -if True: - foo( - 1, - 2) - - -#: E101+1 E101+2 E101+3 E101+4 E101+5 -def test_keys(self): - """areas.json - All regions are accounted for.""" - expected = set([ - u'Norrbotten', - u'V\xe4sterbotten', - ]) - - -#: E101+1 -x = [ - 'abc' -] +# Used to be the file for W191 + +#: E101+1 +if False: + print # indented with 1 tab + +#: E101+1 +y = x == 2 \ + or x == 3 +#: E101+5 +if ( + x == ( + 3 + ) or + y == 4): + pass +#: E101+3 +if x == 2 \ + or y > 1 \ + or x == 3: + pass +#: E101+3 +if x == 2 \ + or y > 1 \ + or x == 3: + pass + +#: E101+1 +if (foo == bar and baz == frop): + pass +#: E101+1 +if (foo == bar and baz == frop): + pass + +#: E101+2 E101+3 +if start[1] > end_col and not ( + over_indent == 4 and indent_next): + assert (0, "E121 continuation line over-" + "indented for visual indent") + + +#: E101+3 +def long_function_name( + var_one, var_two, var_three, + var_four): + hello(var_one) + + +#: E101+2 +if ((row < 0 or self.moduleCount <= row or + col < 0 or self.moduleCount <= col)): + raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) +#: E101+1 E101+2 E101+3 E101+4 E101+5 E101+6 +if bar: + assert ( + start, 'E121 lines starting with a ' + 'closing bracket should be indented ' + "to match that of the opening " + "bracket's line" + ) + +# you want vertical alignment, so use a parens +#: E101+3 +if ((foo.bar("baz") and + foo.bar("frop") + )): + hello("yes") +#: E101+3 +# also ok, but starting to look like LISP +if ((foo.bar("baz") and + foo.bar("frop"))): + hello("yes") +#: E101+1 +if (a == 2 or b == "abc def ghi" "jkl mno"): + assert True +#: E101+2 +if (a == 2 or b == """abc def ghi +jkl mno"""): + assert True +#: E101+1 E101+2 +if length > options.max_line_length: + assert options.max_line_length, \ + "E501 line too long (%d characters)" % length + + +#: E101+1 E101+2 +if os.path.exists(os.path.join(path, PEP8_BIN)): + cmd = ([os.path.join(path, PEP8_BIN)] + + self._pep8_options(targetfile)) +# TODO Tabs in docstrings shouldn't be there, use \t. +''' + multiline string with tab in it''' +# Same here. +'''multiline string + with tabs + and spaces +''' +# Okay +'''sometimes, you just need to go nuts in a multiline string + and allow all sorts of crap + like mixed tabs and spaces + +or trailing whitespace +or long long long long long long long long long long long long long long long long long lines +''' # noqa +# Okay +'''this one + will get no warning +even though the noqa comment is not immediately after the string +''' + foo # noqa + +#: E101+2 +if foo is None and bar is "frop" and \ + blah == 'yeah': + blah = 'yeahnah' + + +#: E101+1 E101+2 E101+3 +if True: + foo( + 1, + 2) + + +#: E101+1 E101+2 E101+3 E101+4 E101+5 +def test_keys(self): + """areas.json - All regions are accounted for.""" + expected = set([ + u'Norrbotten', + u'V\xe4sterbotten', + ]) + + +#: E101+1 +x = [ + 'abc' +] diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E11.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E11.py index 9b97f3980c..a48216946d 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E11.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E11.py @@ -1,60 +1,60 @@ -if x > 2: - #: E111:2 - hello(x) -if True: - #: E111:5 - print - #: E111:6 - # - #: E111:2 - # what - # Comment is fine -# Comment is also fine - -if False: - pass -print -print -#: E903:0 - print -mimetype = 'application/x-directory' -#: E111:5 - # 'httpd/unix-directory' -create_date = False - - -def start(self): - # foo - #: E111:8 - # bar - if True: # Hello - self.master.start() # Comment - # try: - #: E111:12 - # self.master.start() - # except MasterExit: - #: E111:12 - # self.shutdown() - # finally: - #: E111:12 - # sys.exit() - # Dedent to the first level - #: E111:6 - # error -# Dedent to the base level -#: E111:2 - # Also wrongly indented. -# Indent is correct. - - -def start(self): # Correct comment - if True: - #: E111:0 -# try: - #: E111:0 -# self.master.start() - #: E111:0 -# except MasterExit: - #: E111:0 -# self.shutdown() - self.master.start() # comment +if x > 2: + #: E111:2 + hello(x) +if True: + #: E111:5 + print + #: E111:6 + # + #: E111:2 + # what + # Comment is fine +# Comment is also fine + +if False: + pass +print +print +#: E903:0 + print +mimetype = 'application/x-directory' +#: E111:5 + # 'httpd/unix-directory' +create_date = False + + +def start(self): + # foo + #: E111:8 + # bar + if True: # Hello + self.master.start() # Comment + # try: + #: E111:12 + # self.master.start() + # except MasterExit: + #: E111:12 + # self.shutdown() + # finally: + #: E111:12 + # sys.exit() + # Dedent to the first level + #: E111:6 + # error +# Dedent to the base level +#: E111:2 + # Also wrongly indented. +# Indent is correct. + + +def start(self): # Correct comment + if True: + #: E111:0 +# try: + #: E111:0 +# self.master.start() + #: E111:0 +# except MasterExit: + #: E111:0 +# self.shutdown() + self.master.start() # comment diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_first.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_first.py index 8dc65a5a42..d1fd165b81 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_first.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_first.py @@ -1,78 +1,78 @@ -abc = "E121", ( - #: E121:2 - "dent") -abc = "E122", ( - #: E121:0 -"dent") -my_list = [ - 1, 2, 3, - 4, 5, 6, - #: E123 - ] -abc = "E124", ("visual", - "indent_two" - #: E124:14 - ) -abc = "E124", ("visual", - "indent_five" - #: E124:0 -) -a = (123, - #: E124:0 -) -#: E129+1:4 -if (row < 0 or self.moduleCount <= row or - col < 0 or self.moduleCount <= col): - raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) - -abc = "E126", ( - #: E126:12 - "dent") -abc = "E126", ( - #: E126:8 - "dent") -abc = "E127", ("over-", - #: E127:18 - "over-indent") -abc = "E128", ("visual", - #: E128:4 - "hanging") -abc = "E128", ("under-", - #: E128:14 - "under-indent") - - -my_list = [ - 1, 2, 3, - 4, 5, 6, - #: E123:5 - ] -result = { - #: E121:3 - 'key1': 'value', - #: E121:3 - 'key2': 'value', -} -rv.update(dict.fromkeys(( - 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), - #: E128:10 - '?'), - "foo") - -abricot = 3 + \ - 4 + \ - 5 + 6 -abc = "hello", ( - - "there", - #: E126:5 - # "john", - "dude") -part = set_mimetype(( - a.get('mime_type', 'text')), - 'default') -part = set_mimetype(( - a.get('mime_type', 'text')), - #: E127:21 - 'default') +abc = "E121", ( + #: E121:2 + "dent") +abc = "E122", ( + #: E121:0 +"dent") +my_list = [ + 1, 2, 3, + 4, 5, 6, + #: E123 + ] +abc = "E124", ("visual", + "indent_two" + #: E124:14 + ) +abc = "E124", ("visual", + "indent_five" + #: E124:0 +) +a = (123, + #: E124:0 +) +#: E129+1:4 +if (row < 0 or self.moduleCount <= row or + col < 0 or self.moduleCount <= col): + raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) + +abc = "E126", ( + #: E126:12 + "dent") +abc = "E126", ( + #: E126:8 + "dent") +abc = "E127", ("over-", + #: E127:18 + "over-indent") +abc = "E128", ("visual", + #: E128:4 + "hanging") +abc = "E128", ("under-", + #: E128:14 + "under-indent") + + +my_list = [ + 1, 2, 3, + 4, 5, 6, + #: E123:5 + ] +result = { + #: E121:3 + 'key1': 'value', + #: E121:3 + 'key2': 'value', +} +rv.update(dict.fromkeys(( + 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), + #: E128:10 + '?'), + "foo") + +abricot = 3 + \ + 4 + \ + 5 + 6 +abc = "hello", ( + + "there", + #: E126:5 + # "john", + "dude") +part = set_mimetype(( + a.get('mime_type', 'text')), + 'default') +part = set_mimetype(( + a.get('mime_type', 'text')), + #: E127:21 + 'default') diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_first.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_first.py index fc3b5f9339..1a7a2569c8 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_first.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_first.py @@ -1,356 +1,356 @@ -# The issue numbers described in this file are part of the pycodestyle tracker -# and not of parso. -# Originally there were no issues in here, I (dave) added the ones that were -# necessary and IMO useful. -if ( - x == ( - 3 - ) or - y == 4): - pass - -y = x == 2 \ - or x == 3 - -#: E129+1:4 -if x == 2 \ - or y > 1 \ - or x == 3: - pass - -if x == 2 \ - or y > 1 \ - or x == 3: - pass - - -if (foo == bar and - baz == frop): - pass - -#: E129+1:4 E129+2:4 E123+3 -if ( - foo == bar and - baz == frop -): - pass - -if ( - foo == bar and - baz == frop - #: E129:4 - ): - pass - -a = ( -) - -a = (123, - ) - - -if start[1] > end_col and not ( - over_indent == 4 and indent_next): - assert (0, "E121 continuation line over-" - "indented for visual indent") - - -abc = "OK", ("visual", - "indent") - -abc = "Okay", ("visual", - "indent_three" - ) - -abc = "a-ok", ( - "there", - "dude", -) - -abc = "hello", ( - "there", - "dude") - -abc = "hello", ( - - "there", - # "john", - "dude") - -abc = "hello", ( - "there", "dude") - -abc = "hello", ( - "there", "dude", -) - -# Aligned with opening delimiter -foo = long_function_name(var_one, var_two, - var_three, var_four) - -# Extra indentation is not necessary. -foo = long_function_name( - var_one, var_two, - var_three, var_four) - - -arm = 'AAA' \ - 'BBB' \ - 'CCC' - -bbb = 'AAA' \ - 'BBB' \ - 'CCC' - -cc = ('AAA' - 'BBB' - 'CCC') - -cc = {'text': 'AAA' - 'BBB' - 'CCC'} - -cc = dict(text='AAA' - 'BBB') - -sat = 'AAA' \ - 'BBB' \ - 'iii' \ - 'CCC' - -abricot = (3 + - 4 + - 5 + 6) - -#: E122+1:4 -abricot = 3 + \ - 4 + \ - 5 + 6 - -part = [-1, 2, 3, - 4, 5, 6] - -#: E128+1:8 -part = [-1, (2, 3, - 4, 5, 6), 7, - 8, 9, 0] - -fnct(1, 2, 3, - 4, 5, 6) - -fnct(1, 2, 3, - 4, 5, 6, - 7, 8, 9, - 10, 11) - - -def long_function_name( - var_one, var_two, var_three, - var_four): - hello(var_one) - - -if ((row < 0 or self.moduleCount <= row or - col < 0 or self.moduleCount <= col)): - raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) - - -result = { - 'foo': [ - 'bar', { - 'baz': 'frop', - } - ] -} - - -foo = my.func({ - "foo": "bar", -}, "baz") - - -fooff(aaaa, - cca( - vvv, - dadd - ), fff, - ggg) - -fooff(aaaa, - abbb, - cca( - vvv, - aaa, - dadd), - "visual indentation is not a multiple of four",) - -if bar: - assert ( - start, 'E121 lines starting with a ' - 'closing bracket should be indented ' - "to match that of the opening " - "bracket's line" - ) - -# you want vertical alignment, so use a parens -if ((foo.bar("baz") and - foo.bar("frop") - )): - hello("yes") - -# also ok, but starting to look like LISP -if ((foo.bar("baz") and - foo.bar("frop"))): - hello("yes") - -#: E129+1:4 E127+2:9 -if (a == 2 or - b == "abc def ghi" - "jkl mno"): - assert True - -#: E129+1:4 -if (a == 2 or - b == """abc def ghi -jkl mno"""): - assert True - -if length > options.max_line_length: - assert options.max_line_length, \ - "E501 line too long (%d characters)" % length - - -# blub - - -asd = 'l.{line}\t{pos}\t{name}\t{text}'.format( - line=token[2][0], - pos=pos, - name=tokenize.tok_name[token[0]], - text=repr(token[1]), -) - -#: E121+1:6 E121+2:6 -hello('%-7d %s per second (%d total)' % ( - options.counters[key] / elapsed, key, - options.counters[key])) - - -if os.path.exists(os.path.join(path, PEP8_BIN)): - cmd = ([os.path.join(path, PEP8_BIN)] + - self._pep8_options(targetfile)) - - -fixed = (re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + - target[c + 1:]) - -fixed = ( - re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + - target[c + 1:] -) - - -if foo is None and bar is "frop" and \ - blah == 'yeah': - blah = 'yeahnah' - - -"""This is a multi-line - docstring.""" - - -if blah: - # is this actually readable? :) - multiline_literal = """ -while True: - if True: - 1 -""".lstrip() - multiline_literal = ( - """ -while True: - if True: - 1 -""".lstrip() - ) - multiline_literal = ( - """ -while True: - if True: - 1 -""" - .lstrip() - ) - - -if blah: - multiline_visual = (""" -while True: - if True: - 1 -""" - .lstrip()) - - -rv = {'aaa': 42} -rv.update(dict.fromkeys(( - #: E121:4 E121+1:4 - 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), '?')) - -rv.update(dict.fromkeys(('qualif_nr', 'reasonComment_en', - 'reasonComment_fr', 'reasonComment_de', - 'reasonComment_it'), '?')) - -#: E128+1:10 -rv.update(dict.fromkeys(('qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), '?')) - - -rv.update(dict.fromkeys( - ('qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), '?' - ), "foo", context={ - 'alpha': 4, 'beta': 53242234, 'gamma': 17, - }) - - -rv.update( - dict.fromkeys(( - 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), '?'), - "foo", - context={ - 'alpha': 4, 'beta': 53242234, 'gamma': 17, - }, -) - - -event_obj.write(cursor, user_id, { - 'user': user, - 'summary': text, - 'data': data, - }) - -event_obj.write(cursor, user_id, { - 'user': user, - 'summary': text, - 'data': {'aaa': 1, 'bbb': 2}, - }) - -event_obj.write(cursor, user_id, { - 'user': user, - 'summary': text, - 'data': { - 'aaa': 1, - 'bbb': 2}, - }) - -event_obj.write(cursor, user_id, { - 'user': user, - 'summary': text, - 'data': {'timestamp': now, 'content': { - 'aaa': 1, - 'bbb': 2 - }}, - }) +# The issue numbers described in this file are part of the pycodestyle tracker +# and not of parso. +# Originally there were no issues in here, I (dave) added the ones that were +# necessary and IMO useful. +if ( + x == ( + 3 + ) or + y == 4): + pass + +y = x == 2 \ + or x == 3 + +#: E129+1:4 +if x == 2 \ + or y > 1 \ + or x == 3: + pass + +if x == 2 \ + or y > 1 \ + or x == 3: + pass + + +if (foo == bar and + baz == frop): + pass + +#: E129+1:4 E129+2:4 E123+3 +if ( + foo == bar and + baz == frop +): + pass + +if ( + foo == bar and + baz == frop + #: E129:4 + ): + pass + +a = ( +) + +a = (123, + ) + + +if start[1] > end_col and not ( + over_indent == 4 and indent_next): + assert (0, "E121 continuation line over-" + "indented for visual indent") + + +abc = "OK", ("visual", + "indent") + +abc = "Okay", ("visual", + "indent_three" + ) + +abc = "a-ok", ( + "there", + "dude", +) + +abc = "hello", ( + "there", + "dude") + +abc = "hello", ( + + "there", + # "john", + "dude") + +abc = "hello", ( + "there", "dude") + +abc = "hello", ( + "there", "dude", +) + +# Aligned with opening delimiter +foo = long_function_name(var_one, var_two, + var_three, var_four) + +# Extra indentation is not necessary. +foo = long_function_name( + var_one, var_two, + var_three, var_four) + + +arm = 'AAA' \ + 'BBB' \ + 'CCC' + +bbb = 'AAA' \ + 'BBB' \ + 'CCC' + +cc = ('AAA' + 'BBB' + 'CCC') + +cc = {'text': 'AAA' + 'BBB' + 'CCC'} + +cc = dict(text='AAA' + 'BBB') + +sat = 'AAA' \ + 'BBB' \ + 'iii' \ + 'CCC' + +abricot = (3 + + 4 + + 5 + 6) + +#: E122+1:4 +abricot = 3 + \ + 4 + \ + 5 + 6 + +part = [-1, 2, 3, + 4, 5, 6] + +#: E128+1:8 +part = [-1, (2, 3, + 4, 5, 6), 7, + 8, 9, 0] + +fnct(1, 2, 3, + 4, 5, 6) + +fnct(1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 10, 11) + + +def long_function_name( + var_one, var_two, var_three, + var_four): + hello(var_one) + + +if ((row < 0 or self.moduleCount <= row or + col < 0 or self.moduleCount <= col)): + raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) + + +result = { + 'foo': [ + 'bar', { + 'baz': 'frop', + } + ] +} + + +foo = my.func({ + "foo": "bar", +}, "baz") + + +fooff(aaaa, + cca( + vvv, + dadd + ), fff, + ggg) + +fooff(aaaa, + abbb, + cca( + vvv, + aaa, + dadd), + "visual indentation is not a multiple of four",) + +if bar: + assert ( + start, 'E121 lines starting with a ' + 'closing bracket should be indented ' + "to match that of the opening " + "bracket's line" + ) + +# you want vertical alignment, so use a parens +if ((foo.bar("baz") and + foo.bar("frop") + )): + hello("yes") + +# also ok, but starting to look like LISP +if ((foo.bar("baz") and + foo.bar("frop"))): + hello("yes") + +#: E129+1:4 E127+2:9 +if (a == 2 or + b == "abc def ghi" + "jkl mno"): + assert True + +#: E129+1:4 +if (a == 2 or + b == """abc def ghi +jkl mno"""): + assert True + +if length > options.max_line_length: + assert options.max_line_length, \ + "E501 line too long (%d characters)" % length + + +# blub + + +asd = 'l.{line}\t{pos}\t{name}\t{text}'.format( + line=token[2][0], + pos=pos, + name=tokenize.tok_name[token[0]], + text=repr(token[1]), +) + +#: E121+1:6 E121+2:6 +hello('%-7d %s per second (%d total)' % ( + options.counters[key] / elapsed, key, + options.counters[key])) + + +if os.path.exists(os.path.join(path, PEP8_BIN)): + cmd = ([os.path.join(path, PEP8_BIN)] + + self._pep8_options(targetfile)) + + +fixed = (re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + + target[c + 1:]) + +fixed = ( + re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + + target[c + 1:] +) + + +if foo is None and bar is "frop" and \ + blah == 'yeah': + blah = 'yeahnah' + + +"""This is a multi-line + docstring.""" + + +if blah: + # is this actually readable? :) + multiline_literal = """ +while True: + if True: + 1 +""".lstrip() + multiline_literal = ( + """ +while True: + if True: + 1 +""".lstrip() + ) + multiline_literal = ( + """ +while True: + if True: + 1 +""" + .lstrip() + ) + + +if blah: + multiline_visual = (""" +while True: + if True: + 1 +""" + .lstrip()) + + +rv = {'aaa': 42} +rv.update(dict.fromkeys(( + #: E121:4 E121+1:4 + 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), '?')) + +rv.update(dict.fromkeys(('qualif_nr', 'reasonComment_en', + 'reasonComment_fr', 'reasonComment_de', + 'reasonComment_it'), '?')) + +#: E128+1:10 +rv.update(dict.fromkeys(('qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), '?')) + + +rv.update(dict.fromkeys( + ('qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), '?' + ), "foo", context={ + 'alpha': 4, 'beta': 53242234, 'gamma': 17, + }) + + +rv.update( + dict.fromkeys(( + 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), '?'), + "foo", + context={ + 'alpha': 4, 'beta': 53242234, 'gamma': 17, + }, +) + + +event_obj.write(cursor, user_id, { + 'user': user, + 'summary': text, + 'data': data, + }) + +event_obj.write(cursor, user_id, { + 'user': user, + 'summary': text, + 'data': {'aaa': 1, 'bbb': 2}, + }) + +event_obj.write(cursor, user_id, { + 'user': user, + 'summary': text, + 'data': { + 'aaa': 1, + 'bbb': 2}, + }) + +event_obj.write(cursor, user_id, { + 'user': user, + 'summary': text, + 'data': {'timestamp': now, 'content': { + 'aaa': 1, + 'bbb': 2 + }}, + }) diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_second.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_second.py index e7c18e0ec0..fa41e3688e 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_second.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_not_second.py @@ -1,294 +1,294 @@ - -def qualify_by_address( - self, cr, uid, ids, context=None, - params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): - """ This gets called by the web server """ - - -def qualify_by_address(self, cr, uid, ids, context=None, - params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): - """ This gets called by the web server """ - - -_ipv4_re = re.compile('^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') - - -fct(""" - AAA """ + status_2_string) - - -if context: - msg = """\ -action: GET-CONFIG -payload: - ip_address: "%(ip)s" - username: "%(username)s" -""" % context - - -if context: - msg = """\ -action: \ -GET-CONFIG -""" % context - - -if context: - #: E122+2:0 - msg = """\ -action: """\ -"""GET-CONFIG -""" % context - - -def unicode2html(s): - """Convert the characters &<>'" in string s to HTML-safe sequences. - Convert newline to <br> too.""" - #: E127+1:28 - return unicode((s or '').replace('&', '&') - .replace('\n', '<br>\n')) - - -parser.add_option('--count', action='store_true', - help="print total number of errors and warnings " - "to standard error and set exit code to 1 if " - "total is not null") - -parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, - help="exclude files or directories which match these " - "comma separated patterns (default: %s)" % - DEFAULT_EXCLUDE) - -add_option('--count', - #: E135+1 - help="print total number of errors " - "to standard error total is not null") - -add_option('--count', - #: E135+2:11 - help="print total number of errors " - "to standard error " - "total is not null") - - -help = ("print total number of errors " + - "to standard error") - -help = "print total number of errors " \ - "to standard error" - -help = u"print total number of errors " \ - u"to standard error" - -help = b"print total number of errors " \ - b"to standard error" - -#: E122+1:5 -help = br"print total number of errors " \ - br"to standard error" - -d = dict('foo', help="exclude files or directories which match these " - #: E135:9 - "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE) - -d = dict('foo', help=u"exclude files or directories which match these " - u"comma separated patterns (default: %s)" - % DEFAULT_EXCLUDE) - -#: E135+1:9 E135+2:9 -d = dict('foo', help=b"exclude files or directories which match these " - b"comma separated patterns (default: %s)" - % DEFAULT_EXCLUDE) - -d = dict('foo', help=br"exclude files or directories which match these " - br"comma separated patterns (default: %s)" % - DEFAULT_EXCLUDE) - -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s)" % - DEFAULT_EXCLUDE) - -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s, %s)" % - (DEFAULT_EXCLUDE, DEFAULT_IGNORE) - ) - -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s, %s)" % - # who knows what might happen here? - (DEFAULT_EXCLUDE, DEFAULT_IGNORE) - ) - -# parens used to allow the indenting. -troublefree_hash = { - "hash": "value", - "long": ("the quick brown fox jumps over the lazy dog before doing a " - "somersault"), - "long key that tends to happen more when you're indented": ( - "stringwithalongtoken you don't want to break" - ), -} - -# another accepted form -troublefree_hash = { - "hash": "value", - "long": "the quick brown fox jumps over the lazy dog before doing " - "a somersault", - ("long key that tends to happen more " - "when you're indented"): "stringwithalongtoken you don't want to break", -} -# confusing but accepted... don't do that -troublesome_hash = { - "hash": "value", - "long": "the quick brown fox jumps over the lazy dog before doing a " - #: E135:4 - "somersault", - "longer": - "the quick brown fox jumps over the lazy dog before doing a " - "somersaulty", - "long key that tends to happen more " - "when you're indented": "stringwithalongtoken you don't want to break", -} - -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s)" % - DEFAULT_EXCLUDE - ) -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE, - foobar="this clearly should work, because it is at " - "the right indent level", - ) - -rv.update(dict.fromkeys( - ('qualif_nr', 'reasonComment_en', 'reasonComment_fr', - 'reasonComment_de', 'reasonComment_it'), - '?'), "foo", - context={'alpha': 4, 'beta': 53242234, 'gamma': 17}) - - -def f(): - try: - if not Debug: - hello(''' -If you would like to see debugging output, -try: %s -d5 -''' % sys.argv[0]) - - -# The try statement above was not finished. -#: E901 -d = { # comment - 1: 2 -} - -# issue 138 (we won't allow this in parso) -#: E126+2:9 -[ - 12, # this is a multi-line inline - # comment -] -# issue 151 -#: E122+1:3 -if a > b and \ - c > d: - moo_like_a_cow() - -my_list = [ - 1, 2, 3, - 4, 5, 6, -] - -my_list = [1, 2, 3, - 4, 5, 6, - ] - -result = some_function_that_takes_arguments( - 'a', 'b', 'c', - 'd', 'e', 'f', -) - -result = some_function_that_takes_arguments('a', 'b', 'c', - 'd', 'e', 'f', - ) - -# issue 203 -dica = { - ('abc' - 'def'): ( - 'abc'), -} - -(abcdef[0] - [1]) = ( - 'abc') - -('abc' - 'def') == ( - 'abc') - -# issue 214 -bar( - 1).zap( - 2) - -bar( - 1).zap( - 2) - -if True: - - def example_issue254(): - return [node.copy( - ( - replacement - # First, look at all the node's current children. - for child in node.children - # Replace them. - for replacement in replace(child) - ), - dict(name=token.undefined) - )] - - -def valid_example(): - return [node.copy(properties=dict( - (key, val if val is not None else token.undefined) - for key, val in node.items() - ))] - - -foo([ - 'bug' -]) - -# issue 144, finally! -some_hash = { - "long key that tends to happen more when you're indented": - "stringwithalongtoken you don't want to break", -} - -{ - 1: - 999999 if True - else 0, -} - - -abc = dedent( - ''' - mkdir -p ./{build}/ - mv ./build/ ./{build}/%(revision)s/ - '''.format( - build='build', - # more stuff - ) -) + +def qualify_by_address( + self, cr, uid, ids, context=None, + params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): + """ This gets called by the web server """ + + +def qualify_by_address(self, cr, uid, ids, context=None, + params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): + """ This gets called by the web server """ + + +_ipv4_re = re.compile('^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' + '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' + '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.' + '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') + + +fct(""" + AAA """ + status_2_string) + + +if context: + msg = """\ +action: GET-CONFIG +payload: + ip_address: "%(ip)s" + username: "%(username)s" +""" % context + + +if context: + msg = """\ +action: \ +GET-CONFIG +""" % context + + +if context: + #: E122+2:0 + msg = """\ +action: """\ +"""GET-CONFIG +""" % context + + +def unicode2html(s): + """Convert the characters &<>'" in string s to HTML-safe sequences. + Convert newline to <br> too.""" + #: E127+1:28 + return unicode((s or '').replace('&', '&') + .replace('\n', '<br>\n')) + + +parser.add_option('--count', action='store_true', + help="print total number of errors and warnings " + "to standard error and set exit code to 1 if " + "total is not null") + +parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, + help="exclude files or directories which match these " + "comma separated patterns (default: %s)" % + DEFAULT_EXCLUDE) + +add_option('--count', + #: E135+1 + help="print total number of errors " + "to standard error total is not null") + +add_option('--count', + #: E135+2:11 + help="print total number of errors " + "to standard error " + "total is not null") + + +help = ("print total number of errors " + + "to standard error") + +help = "print total number of errors " \ + "to standard error" + +help = u"print total number of errors " \ + u"to standard error" + +help = b"print total number of errors " \ + b"to standard error" + +#: E122+1:5 +help = br"print total number of errors " \ + br"to standard error" + +d = dict('foo', help="exclude files or directories which match these " + #: E135:9 + "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE) + +d = dict('foo', help=u"exclude files or directories which match these " + u"comma separated patterns (default: %s)" + % DEFAULT_EXCLUDE) + +#: E135+1:9 E135+2:9 +d = dict('foo', help=b"exclude files or directories which match these " + b"comma separated patterns (default: %s)" + % DEFAULT_EXCLUDE) + +d = dict('foo', help=br"exclude files or directories which match these " + br"comma separated patterns (default: %s)" % + DEFAULT_EXCLUDE) + +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s)" % + DEFAULT_EXCLUDE) + +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s, %s)" % + (DEFAULT_EXCLUDE, DEFAULT_IGNORE) + ) + +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s, %s)" % + # who knows what might happen here? + (DEFAULT_EXCLUDE, DEFAULT_IGNORE) + ) + +# parens used to allow the indenting. +troublefree_hash = { + "hash": "value", + "long": ("the quick brown fox jumps over the lazy dog before doing a " + "somersault"), + "long key that tends to happen more when you're indented": ( + "stringwithalongtoken you don't want to break" + ), +} + +# another accepted form +troublefree_hash = { + "hash": "value", + "long": "the quick brown fox jumps over the lazy dog before doing " + "a somersault", + ("long key that tends to happen more " + "when you're indented"): "stringwithalongtoken you don't want to break", +} +# confusing but accepted... don't do that +troublesome_hash = { + "hash": "value", + "long": "the quick brown fox jumps over the lazy dog before doing a " + #: E135:4 + "somersault", + "longer": + "the quick brown fox jumps over the lazy dog before doing a " + "somersaulty", + "long key that tends to happen more " + "when you're indented": "stringwithalongtoken you don't want to break", +} + +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s)" % + DEFAULT_EXCLUDE + ) +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE, + foobar="this clearly should work, because it is at " + "the right indent level", + ) + +rv.update(dict.fromkeys( + ('qualif_nr', 'reasonComment_en', 'reasonComment_fr', + 'reasonComment_de', 'reasonComment_it'), + '?'), "foo", + context={'alpha': 4, 'beta': 53242234, 'gamma': 17}) + + +def f(): + try: + if not Debug: + hello(''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0]) + + +# The try statement above was not finished. +#: E901 +d = { # comment + 1: 2 +} + +# issue 138 (we won't allow this in parso) +#: E126+2:9 +[ + 12, # this is a multi-line inline + # comment +] +# issue 151 +#: E122+1:3 +if a > b and \ + c > d: + moo_like_a_cow() + +my_list = [ + 1, 2, 3, + 4, 5, 6, +] + +my_list = [1, 2, 3, + 4, 5, 6, + ] + +result = some_function_that_takes_arguments( + 'a', 'b', 'c', + 'd', 'e', 'f', +) + +result = some_function_that_takes_arguments('a', 'b', 'c', + 'd', 'e', 'f', + ) + +# issue 203 +dica = { + ('abc' + 'def'): ( + 'abc'), +} + +(abcdef[0] + [1]) = ( + 'abc') + +('abc' + 'def') == ( + 'abc') + +# issue 214 +bar( + 1).zap( + 2) + +bar( + 1).zap( + 2) + +if True: + + def example_issue254(): + return [node.copy( + ( + replacement + # First, look at all the node's current children. + for child in node.children + # Replace them. + for replacement in replace(child) + ), + dict(name=token.undefined) + )] + + +def valid_example(): + return [node.copy(properties=dict( + (key, val if val is not None else token.undefined) + for key, val in node.items() + ))] + + +foo([ + 'bug' +]) + +# issue 144, finally! +some_hash = { + "long key that tends to happen more when you're indented": + "stringwithalongtoken you don't want to break", +} + +{ + 1: + 999999 if True + else 0, +} + + +abc = dedent( + ''' + mkdir -p ./{build}/ + mv ./build/ ./{build}/%(revision)s/ + '''.format( + build='build', + # more stuff + ) +) diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_second.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_second.py index 5488ea40eb..902f956ade 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_second.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_second.py @@ -1,195 +1,195 @@ -if True: - result = some_function_that_takes_arguments( - 'a', 'b', 'c', - 'd', 'e', 'f', - #: E123:0 -) -#: E122+1 -if some_very_very_very_long_variable_name or var \ -or another_very_long_variable_name: - raise Exception() -#: E122+1 -if some_very_very_very_long_variable_name or var[0] \ -or another_very_long_variable_name: - raise Exception() -if True: - #: E122+1 - if some_very_very_very_long_variable_name or var \ - or another_very_long_variable_name: - raise Exception() -if True: - #: E122+1 - if some_very_very_very_long_variable_name or var[0] \ - or another_very_long_variable_name: - raise Exception() - -#: E901+1:8 -dictionary = [ - "is": { - # Might be a E122:4, but is not because the code is invalid Python. - "nested": yes(), - }, -] -setup('', - scripts=[''], - classifiers=[ - #: E121:6 - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - ]) - - -#: E123+2:4 E291:15 -abc = "E123", ( - "bad", "hanging", "close" - ) - -result = { - 'foo': [ - 'bar', { - 'baz': 'frop', - #: E123 - } - #: E123 - ] - #: E123 - } -result = some_function_that_takes_arguments( - 'a', 'b', 'c', - 'd', 'e', 'f', - #: E123 - ) -my_list = [1, 2, 3, - 4, 5, 6, - #: E124:0 -] -my_list = [1, 2, 3, - 4, 5, 6, - #: E124:19 - ] -#: E124+2 -result = some_function_that_takes_arguments('a', 'b', 'c', - 'd', 'e', 'f', -) -fooff(aaaa, - cca( - vvv, - dadd - ), fff, - #: E124:0 -) -fooff(aaaa, - ccaaa( - vvv, - dadd - ), - fff, - #: E124:0 -) -d = dict('foo', - help="exclude files or directories which match these " - "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE - #: E124:14 - ) - -if line_removed: - self.event(cr, uid, - #: E128:8 - name="Removing the option for contract", - #: E128:8 - description="contract line has been removed", - #: E124:8 - ) - -#: E129+1:4 -if foo is None and bar is "frop" and \ - blah == 'yeah': - blah = 'yeahnah' - - -#: E129+1:4 E129+2:4 -def long_function_name( - var_one, var_two, var_three, - var_four): - hello(var_one) - - -def qualify_by_address( - #: E129:4 E129+1:4 - self, cr, uid, ids, context=None, - params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): - """ This gets called by the web server """ - - -#: E129+1:4 E129+2:4 -if (a == 2 or - b == "abc def ghi" - "jkl mno"): - True - -my_list = [ - 1, 2, 3, - 4, 5, 6, - #: E123:8 - ] - -abris = 3 + \ - 4 + \ - 5 + 6 - -fixed = re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + \ - target[c + 1:] - -rv.update(dict.fromkeys(( - 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', - #: E121:12 - 'reasonComment_de', 'reasonComment_it'), - '?'), - #: E128:4 - "foo") -#: E126+1:8 -eat_a_dict_a_day({ - "foo": "bar", -}) -#: E129+1:4 -if ( - x == ( - 3 - #: E129:4 - ) or - y == 4): - pass -#: E129+1:4 E121+2:8 E129+3:4 -if ( - x == ( - 3 - ) or - x == ( - # This one has correct indentation. - 3 - #: E129:4 - ) or - y == 4): - pass -troublesome_hash = { - "hash": "value", - #: E135+1:8 - "long": "the quick brown fox jumps over the lazy dog before doing a " - "somersault", -} - -# Arguments on first line forbidden when not using vertical alignment -#: E128+1:4 -foo = long_function_name(var_one, var_two, - var_three, var_four) - -#: E128+1:4 -hello('l.%s\t%s\t%s\t%r' % - (token[2][0], pos, tokenize.tok_name[token[0]], token[1])) - - -def qualify_by_address(self, cr, uid, ids, context=None, - #: E128:8 - params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): - """ This gets called by the web server """ +if True: + result = some_function_that_takes_arguments( + 'a', 'b', 'c', + 'd', 'e', 'f', + #: E123:0 +) +#: E122+1 +if some_very_very_very_long_variable_name or var \ +or another_very_long_variable_name: + raise Exception() +#: E122+1 +if some_very_very_very_long_variable_name or var[0] \ +or another_very_long_variable_name: + raise Exception() +if True: + #: E122+1 + if some_very_very_very_long_variable_name or var \ + or another_very_long_variable_name: + raise Exception() +if True: + #: E122+1 + if some_very_very_very_long_variable_name or var[0] \ + or another_very_long_variable_name: + raise Exception() + +#: E901+1:8 +dictionary = [ + "is": { + # Might be a E122:4, but is not because the code is invalid Python. + "nested": yes(), + }, +] +setup('', + scripts=[''], + classifiers=[ + #: E121:6 + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + ]) + + +#: E123+2:4 E291:15 +abc = "E123", ( + "bad", "hanging", "close" + ) + +result = { + 'foo': [ + 'bar', { + 'baz': 'frop', + #: E123 + } + #: E123 + ] + #: E123 + } +result = some_function_that_takes_arguments( + 'a', 'b', 'c', + 'd', 'e', 'f', + #: E123 + ) +my_list = [1, 2, 3, + 4, 5, 6, + #: E124:0 +] +my_list = [1, 2, 3, + 4, 5, 6, + #: E124:19 + ] +#: E124+2 +result = some_function_that_takes_arguments('a', 'b', 'c', + 'd', 'e', 'f', +) +fooff(aaaa, + cca( + vvv, + dadd + ), fff, + #: E124:0 +) +fooff(aaaa, + ccaaa( + vvv, + dadd + ), + fff, + #: E124:0 +) +d = dict('foo', + help="exclude files or directories which match these " + "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE + #: E124:14 + ) + +if line_removed: + self.event(cr, uid, + #: E128:8 + name="Removing the option for contract", + #: E128:8 + description="contract line has been removed", + #: E124:8 + ) + +#: E129+1:4 +if foo is None and bar is "frop" and \ + blah == 'yeah': + blah = 'yeahnah' + + +#: E129+1:4 E129+2:4 +def long_function_name( + var_one, var_two, var_three, + var_four): + hello(var_one) + + +def qualify_by_address( + #: E129:4 E129+1:4 + self, cr, uid, ids, context=None, + params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): + """ This gets called by the web server """ + + +#: E129+1:4 E129+2:4 +if (a == 2 or + b == "abc def ghi" + "jkl mno"): + True + +my_list = [ + 1, 2, 3, + 4, 5, 6, + #: E123:8 + ] + +abris = 3 + \ + 4 + \ + 5 + 6 + +fixed = re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + \ + target[c + 1:] + +rv.update(dict.fromkeys(( + 'qualif_nr', 'reasonComment_en', 'reasonComment_fr', + #: E121:12 + 'reasonComment_de', 'reasonComment_it'), + '?'), + #: E128:4 + "foo") +#: E126+1:8 +eat_a_dict_a_day({ + "foo": "bar", +}) +#: E129+1:4 +if ( + x == ( + 3 + #: E129:4 + ) or + y == 4): + pass +#: E129+1:4 E121+2:8 E129+3:4 +if ( + x == ( + 3 + ) or + x == ( + # This one has correct indentation. + 3 + #: E129:4 + ) or + y == 4): + pass +troublesome_hash = { + "hash": "value", + #: E135+1:8 + "long": "the quick brown fox jumps over the lazy dog before doing a " + "somersault", +} + +# Arguments on first line forbidden when not using vertical alignment +#: E128+1:4 +foo = long_function_name(var_one, var_two, + var_three, var_four) + +#: E128+1:4 +hello('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], token[1])) + + +def qualify_by_address(self, cr, uid, ids, context=None, + #: E128:8 + params_to_check=frozenset(QUALIF_BY_ADDRESS_PARAM)): + """ This gets called by the web server """ diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_third.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_third.py index 26697fed73..7105ad6486 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E12_third.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E12_third.py @@ -1,116 +1,116 @@ -#: E128+1 -foo(1, 2, 3, -4, 5, 6) -#: E128+1:1 -foo(1, 2, 3, - 4, 5, 6) -#: E128+1:2 -foo(1, 2, 3, - 4, 5, 6) -#: E128+1:3 -foo(1, 2, 3, - 4, 5, 6) -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:5 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:6 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:7 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:8 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:9 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:10 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:11 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:12 -foo(1, 2, 3, - 4, 5, 6) -#: E127+1:13 -foo(1, 2, 3, - 4, 5, 6) -if line_removed: - #: E128+1:14 E128+2:14 - self.event(cr, uid, - name="Removing the option for contract", - description="contract line has been removed", - ) - -if line_removed: - self.event(cr, uid, - #: E127:16 - name="Removing the option for contract", - #: E127:16 - description="contract line has been removed", - #: E124:16 - ) -rv.update(d=('a', 'b', 'c'), - #: E127:13 - e=42) - -#: E135+2:17 -rv.update(d=('a' + 'b', 'c'), - e=42, f=42 - + 42) -rv.update(d=('a' + 'b', 'c'), - e=42, f=42 - + 42) -#: E127+1:26 -input1 = {'a': {'calc': 1 + 2}, 'b': 1 - + 42} -#: E128+2:17 -rv.update(d=('a' + 'b', 'c'), - e=42, f=(42 - + 42)) - -if True: - def example_issue254(): - #: - return [node.copy( - ( - #: E121:16 E121+3:20 - replacement - # First, look at all the node's current children. - for child in node.children - for replacement in replace(child) - ), - dict(name=token.undefined) - )] -# TODO multiline docstring are currently not handled. E125+1:4? -if (""" - """): - pass - -# TODO same -for foo in """ - abc - 123 - """.strip().split(): - hello(foo) -abc = dedent( - ''' - mkdir -p ./{build}/ - mv ./build/ ./{build}/%(revision)s/ - '''.format( - #: E121:4 E121+1:4 E123+2:0 - build='build', - # more stuff -) -) -#: E701+1: E122+1 -if True:\ -hello(True) - -#: E128+1 -foobar(a -, end=' ') +#: E128+1 +foo(1, 2, 3, +4, 5, 6) +#: E128+1:1 +foo(1, 2, 3, + 4, 5, 6) +#: E128+1:2 +foo(1, 2, 3, + 4, 5, 6) +#: E128+1:3 +foo(1, 2, 3, + 4, 5, 6) +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:5 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:6 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:7 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:8 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:9 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:10 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:11 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:12 +foo(1, 2, 3, + 4, 5, 6) +#: E127+1:13 +foo(1, 2, 3, + 4, 5, 6) +if line_removed: + #: E128+1:14 E128+2:14 + self.event(cr, uid, + name="Removing the option for contract", + description="contract line has been removed", + ) + +if line_removed: + self.event(cr, uid, + #: E127:16 + name="Removing the option for contract", + #: E127:16 + description="contract line has been removed", + #: E124:16 + ) +rv.update(d=('a', 'b', 'c'), + #: E127:13 + e=42) + +#: E135+2:17 +rv.update(d=('a' + 'b', 'c'), + e=42, f=42 + + 42) +rv.update(d=('a' + 'b', 'c'), + e=42, f=42 + + 42) +#: E127+1:26 +input1 = {'a': {'calc': 1 + 2}, 'b': 1 + + 42} +#: E128+2:17 +rv.update(d=('a' + 'b', 'c'), + e=42, f=(42 + + 42)) + +if True: + def example_issue254(): + #: + return [node.copy( + ( + #: E121:16 E121+3:20 + replacement + # First, look at all the node's current children. + for child in node.children + for replacement in replace(child) + ), + dict(name=token.undefined) + )] +# TODO multiline docstring are currently not handled. E125+1:4? +if (""" + """): + pass + +# TODO same +for foo in """ + abc + 123 + """.strip().split(): + hello(foo) +abc = dedent( + ''' + mkdir -p ./{build}/ + mv ./build/ ./{build}/%(revision)s/ + '''.format( + #: E121:4 E121+1:4 E123+2:0 + build='build', + # more stuff +) +) +#: E701+1: E122+1 +if True:\ +hello(True) + +#: E128+1 +foobar(a +, end=' ') diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E20.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E20.py index 44986fa963..dca322227c 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E20.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E20.py @@ -1,52 +1,52 @@ -#: E201:5 -spam( ham[1], {eggs: 2}) -#: E201:9 -spam(ham[ 1], {eggs: 2}) -#: E201:14 -spam(ham[1], { eggs: 2}) - -# Okay -spam(ham[1], {eggs: 2}) - - -#: E202:22 -spam(ham[1], {eggs: 2} ) -#: E202:21 -spam(ham[1], {eggs: 2 }) -#: E202:10 -spam(ham[1 ], {eggs: 2}) -# Okay -spam(ham[1], {eggs: 2}) - -result = func( - arg1='some value', - arg2='another value', -) - -result = func( - arg1='some value', - arg2='another value' -) - -result = [ - item for item in items - if item > 5 -] - -#: E203:9 -if x == 4 : - foo(x, y) - x, y = y, x -if x == 4: - #: E203:12 E702:13 - a = x, y ; x, y = y, x -if x == 4: - foo(x, y) - #: E203:12 - x, y = y , x -# Okay -if x == 4: - foo(x, y) - x, y = y, x -a[b1, :1] == 3 -b = a[:, b1] +#: E201:5 +spam( ham[1], {eggs: 2}) +#: E201:9 +spam(ham[ 1], {eggs: 2}) +#: E201:14 +spam(ham[1], { eggs: 2}) + +# Okay +spam(ham[1], {eggs: 2}) + + +#: E202:22 +spam(ham[1], {eggs: 2} ) +#: E202:21 +spam(ham[1], {eggs: 2 }) +#: E202:10 +spam(ham[1 ], {eggs: 2}) +# Okay +spam(ham[1], {eggs: 2}) + +result = func( + arg1='some value', + arg2='another value', +) + +result = func( + arg1='some value', + arg2='another value' +) + +result = [ + item for item in items + if item > 5 +] + +#: E203:9 +if x == 4 : + foo(x, y) + x, y = y, x +if x == 4: + #: E203:12 E702:13 + a = x, y ; x, y = y, x +if x == 4: + foo(x, y) + #: E203:12 + x, y = y , x +# Okay +if x == 4: + foo(x, y) + x, y = y, x +a[b1, :1] == 3 +b = a[:, b1] diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E21.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E21.py index f65616e8ab..d0d79c64b8 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E21.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E21.py @@ -1,16 +1,16 @@ -#: E211:4 -spam (1) -#: E211:4 E211:19 -dict ['key'] = list [index] -#: E211:11 -dict['key'] ['subkey'] = list[index] -# Okay -spam(1) -dict['key'] = list[index] - - -# This is not prohibited by PEP8, but avoid it. -# Dave: I think this is extremely stupid. Use the same convention everywhere. -#: E211:9 -class Foo (Bar, Baz): - pass +#: E211:4 +spam (1) +#: E211:4 E211:19 +dict ['key'] = list [index] +#: E211:11 +dict['key'] ['subkey'] = list[index] +# Okay +spam(1) +dict['key'] = list[index] + + +# This is not prohibited by PEP8, but avoid it. +# Dave: I think this is extremely stupid. Use the same convention everywhere. +#: E211:9 +class Foo (Bar, Baz): + pass diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E22.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E22.py index 82ff6a440a..d4b48263d7 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E22.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E22.py @@ -1,156 +1,156 @@ -a = 12 + 3 -#: E221:5 E229:8 -b = 4 + 5 -#: E221:1 -x = 1 -#: E221:1 -y = 2 -long_variable = 3 -#: E221:4 -x[0] = 1 -#: E221:4 -x[1] = 2 -long_variable = 3 -#: E221:8 E229:19 -x = f(x) + 1 -y = long_variable + 2 -#: E221:8 E229:19 -z = x[0] + 3 -#: E221+2:13 -text = """ - bar - foo %s""" % rofl -# Okay -x = 1 -y = 2 -long_variable = 3 - - -#: E221:7 -a = a + 1 -b = b + 10 -#: E221:3 -x = -1 -#: E221:3 -y = -2 -long_variable = 3 -#: E221:6 -x[0] = 1 -#: E221:6 -x[1] = 2 -long_variable = 3 - - -#: E223+1:1 -foobart = 4 -a = 3 # aligned with tab - - -#: E223:4 -a += 1 -b += 1000 - - -#: E225:12 -submitted +=1 -#: E225:9 -submitted+= 1 -#: E225:3 -c =-1 -#: E229:7 -x = x /2 - 1 -#: E229:11 -c = alpha -4 -#: E229:10 -c = alpha- 4 -#: E229:8 -z = x **y -#: E229:14 -z = (x + 1) **y -#: E229:13 -z = (x + 1)** y -#: E227:14 -_1kB = _1MB >>10 -#: E227:11 -_1kB = _1MB>> 10 -#: E225:1 E225:2 E229:4 -i=i+ 1 -#: E225:1 E225:2 E229:5 -i=i +1 -#: E225:1 E225:2 -i=i+1 -#: E225:3 -i =i+1 -#: E225:1 -i= i+1 -#: E229:8 -c = (a +b)*(a - b) -#: E229:7 -c = (a+ b)*(a - b) - -z = 2//30 -c = (a+b) * (a-b) -x = x*2 - 1 -x = x/2 - 1 -# TODO whitespace should be the other way around according to pep8. -x = x / 2-1 - -hypot2 = x*x + y*y -c = (a + b)*(a - b) - - -def halves(n): - return (i//2 for i in range(n)) - - -#: E227:11 E227:13 -_1kB = _1MB>>10 -#: E227:11 E227:13 -_1MB = _1kB<<10 -#: E227:5 E227:6 -a = b|c -#: E227:5 E227:6 -b = c&a -#: E227:5 E227:6 -c = b^a -#: E228:5 E228:6 -a = b%c -#: E228:9 E228:10 -msg = fmt%(errno, errmsg) -#: E228:25 E228:26 -msg = "Error %d occurred"%errno - -#: E228:7 -a = b %c -a = b % c - -# Okay -i = i + 1 -submitted += 1 -x = x * 2 - 1 -hypot2 = x * x + y * y -c = (a + b) * (a - b) -_1MiB = 2 ** 20 -_1TiB = 2**30 -foo(bar, key='word', *args, **kwargs) -baz(**kwargs) -negative = -1 -spam(-1) --negative -func1(lambda *args, **kw: (args, kw)) -func2(lambda a, b=h[:], c=0: (a, b, c)) -if not -5 < x < +5: - #: E227:12 - print >>sys.stderr, "x is out of range." -print >> sys.stdout, "x is an integer." -x = x / 2 - 1 - - -def squares(n): - return (i**2 for i in range(n)) - - -ENG_PREFIXES = { - -6: "\u03bc", # Greek letter mu - -3: "m", -} +a = 12 + 3 +#: E221:5 E229:8 +b = 4 + 5 +#: E221:1 +x = 1 +#: E221:1 +y = 2 +long_variable = 3 +#: E221:4 +x[0] = 1 +#: E221:4 +x[1] = 2 +long_variable = 3 +#: E221:8 E229:19 +x = f(x) + 1 +y = long_variable + 2 +#: E221:8 E229:19 +z = x[0] + 3 +#: E221+2:13 +text = """ + bar + foo %s""" % rofl +# Okay +x = 1 +y = 2 +long_variable = 3 + + +#: E221:7 +a = a + 1 +b = b + 10 +#: E221:3 +x = -1 +#: E221:3 +y = -2 +long_variable = 3 +#: E221:6 +x[0] = 1 +#: E221:6 +x[1] = 2 +long_variable = 3 + + +#: E223+1:1 +foobart = 4 +a = 3 # aligned with tab + + +#: E223:4 +a += 1 +b += 1000 + + +#: E225:12 +submitted +=1 +#: E225:9 +submitted+= 1 +#: E225:3 +c =-1 +#: E229:7 +x = x /2 - 1 +#: E229:11 +c = alpha -4 +#: E229:10 +c = alpha- 4 +#: E229:8 +z = x **y +#: E229:14 +z = (x + 1) **y +#: E229:13 +z = (x + 1)** y +#: E227:14 +_1kB = _1MB >>10 +#: E227:11 +_1kB = _1MB>> 10 +#: E225:1 E225:2 E229:4 +i=i+ 1 +#: E225:1 E225:2 E229:5 +i=i +1 +#: E225:1 E225:2 +i=i+1 +#: E225:3 +i =i+1 +#: E225:1 +i= i+1 +#: E229:8 +c = (a +b)*(a - b) +#: E229:7 +c = (a+ b)*(a - b) + +z = 2//30 +c = (a+b) * (a-b) +x = x*2 - 1 +x = x/2 - 1 +# TODO whitespace should be the other way around according to pep8. +x = x / 2-1 + +hypot2 = x*x + y*y +c = (a + b)*(a - b) + + +def halves(n): + return (i//2 for i in range(n)) + + +#: E227:11 E227:13 +_1kB = _1MB>>10 +#: E227:11 E227:13 +_1MB = _1kB<<10 +#: E227:5 E227:6 +a = b|c +#: E227:5 E227:6 +b = c&a +#: E227:5 E227:6 +c = b^a +#: E228:5 E228:6 +a = b%c +#: E228:9 E228:10 +msg = fmt%(errno, errmsg) +#: E228:25 E228:26 +msg = "Error %d occurred"%errno + +#: E228:7 +a = b %c +a = b % c + +# Okay +i = i + 1 +submitted += 1 +x = x * 2 - 1 +hypot2 = x * x + y * y +c = (a + b) * (a - b) +_1MiB = 2 ** 20 +_1TiB = 2**30 +foo(bar, key='word', *args, **kwargs) +baz(**kwargs) +negative = -1 +spam(-1) +-negative +func1(lambda *args, **kw: (args, kw)) +func2(lambda a, b=h[:], c=0: (a, b, c)) +if not -5 < x < +5: + #: E227:12 + print >>sys.stderr, "x is out of range." +print >> sys.stdout, "x is an integer." +x = x / 2 - 1 + + +def squares(n): + return (i**2 for i in range(n)) + + +ENG_PREFIXES = { + -6: "\u03bc", # Greek letter mu + -3: "m", +} diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E23.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E23.py index 47f1447a23..66f9ca17d6 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E23.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E23.py @@ -1,16 +1,16 @@ -#: E231:7 -a = (1,2) -#: E231:5 -a[b1,:] -#: E231:10 -a = [{'a':''}] -# Okay -a = (4,) -#: E202:7 -b = (5, ) -c = {'text': text[5:]} - -result = { - 'key1': 'value', - 'key2': 'value', -} +#: E231:7 +a = (1,2) +#: E231:5 +a[b1,:] +#: E231:10 +a = [{'a':''}] +# Okay +a = (4,) +#: E202:7 +b = (5, ) +c = {'text': text[5:]} + +result = { + 'key1': 'value', + 'key2': 'value', +} diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E25.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E25.py index 8cf53147f7..49c008faee 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E25.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E25.py @@ -1,36 +1,36 @@ -#: E251:11 E251:13 -def foo(bar = False): - '''Test function with an error in declaration''' - pass - - -#: E251:8 -foo(bar= True) -#: E251:7 -foo(bar =True) -#: E251:7 E251:9 -foo(bar = True) -#: E251:13 -y = bar(root= "sdasd") -parser.add_argument('--long-option', - #: E135+1:20 - default= - "/rather/long/filesystem/path/here/blah/blah/blah") -parser.add_argument('--long-option', - default= - "/rather/long/filesystem") -# TODO this looks so stupid. -parser.add_argument('--long-option', default - ="/rather/long/filesystem/path/here/blah/blah/blah") -#: E251+2:7 E251+2:9 -foo(True, - baz=(1, 2), - biz = 'foo' - ) -# Okay -foo(bar=(1 == 1)) -foo(bar=(1 != 1)) -foo(bar=(1 >= 1)) -foo(bar=(1 <= 1)) -(options, args) = parser.parse_args() -d[type(None)] = _deepcopy_atomic +#: E251:11 E251:13 +def foo(bar = False): + '''Test function with an error in declaration''' + pass + + +#: E251:8 +foo(bar= True) +#: E251:7 +foo(bar =True) +#: E251:7 E251:9 +foo(bar = True) +#: E251:13 +y = bar(root= "sdasd") +parser.add_argument('--long-option', + #: E135+1:20 + default= + "/rather/long/filesystem/path/here/blah/blah/blah") +parser.add_argument('--long-option', + default= + "/rather/long/filesystem") +# TODO this looks so stupid. +parser.add_argument('--long-option', default + ="/rather/long/filesystem/path/here/blah/blah/blah") +#: E251+2:7 E251+2:9 +foo(True, + baz=(1, 2), + biz = 'foo' + ) +# Okay +foo(bar=(1 == 1)) +foo(bar=(1 != 1)) +foo(bar=(1 >= 1)) +foo(bar=(1 <= 1)) +(options, args) = parser.parse_args() +d[type(None)] = _deepcopy_atomic diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E26.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E26.py index 4774852a07..75b39d22b5 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E26.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E26.py @@ -1,78 +1,78 @@ -#: E261:4 -pass # an inline comment -#: E261:4 -pass# an inline comment - -# Okay -pass # an inline comment -pass # an inline comment -#: E262:11 -x = x + 1 #Increment x -#: E262:11 -x = x + 1 # Increment x -#: E262:11 -x = y + 1 #: Increment x -#: E265 -#Block comment -a = 1 -#: E265+1 -m = 42 -#! This is important -mx = 42 - 42 - -# Comment without anything is not an issue. -# -# However if there are comments at the end without anything it obviously -# doesn't make too much sense. -#: E262:9 -foo = 1 # - - -#: E266+2:4 E266+5:4 -def how_it_feel(r): - - ### This is a variable ### - a = 42 - - ### Of course it is unused - return - - -#: E266 E266+1 -##if DEBUG: -## logging.error() -#: E266 -######################################### - -# Not at the beginning of a file -#: E265 -#!/usr/bin/env python - -# Okay - -pass # an inline comment -x = x + 1 # Increment x -y = y + 1 #: Increment x - -# Block comment -a = 1 - -# Block comment1 - -# Block comment2 -aaa = 1 - - -# example of docstring (not parsed) -def oof(): - """ - #foo not parsed - """ - - ########################################################################### - # A SEPARATOR # - ########################################################################### - - # ####################################################################### # - # ########################## another separator ########################## # - # ####################################################################### # +#: E261:4 +pass # an inline comment +#: E261:4 +pass# an inline comment + +# Okay +pass # an inline comment +pass # an inline comment +#: E262:11 +x = x + 1 #Increment x +#: E262:11 +x = x + 1 # Increment x +#: E262:11 +x = y + 1 #: Increment x +#: E265 +#Block comment +a = 1 +#: E265+1 +m = 42 +#! This is important +mx = 42 - 42 + +# Comment without anything is not an issue. +# +# However if there are comments at the end without anything it obviously +# doesn't make too much sense. +#: E262:9 +foo = 1 # + + +#: E266+2:4 E266+5:4 +def how_it_feel(r): + + ### This is a variable ### + a = 42 + + ### Of course it is unused + return + + +#: E266 E266+1 +##if DEBUG: +## logging.error() +#: E266 +######################################### + +# Not at the beginning of a file +#: E265 +#!/usr/bin/env python + +# Okay + +pass # an inline comment +x = x + 1 # Increment x +y = y + 1 #: Increment x + +# Block comment +a = 1 + +# Block comment1 + +# Block comment2 +aaa = 1 + + +# example of docstring (not parsed) +def oof(): + """ + #foo not parsed + """ + + ########################################################################### + # A SEPARATOR # + ########################################################################### + + # ####################################################################### # + # ########################## another separator ########################## # + # ####################################################################### # diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E27.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E27.py index 9149f0aa52..40965de2c8 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E27.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E27.py @@ -1,49 +1,49 @@ -# Okay -from u import (a, b) -from v import c, d -#: E221:13 -from w import (e, f) -#: E275:13 -from w import(e, f) -#: E275:29 -from importable.module import(e, f) -try: - #: E275:33 - from importable.module import(e, f) -except ImportError: - pass -# Okay -True and False -#: E221:8 -True and False -#: E221:4 -True and False -#: E221:2 -if 1: - pass -# Syntax Error, no indentation -#: E903+1 -if 1: -pass -#: E223:8 -True and False -#: E223:4 E223:9 -True and False -#: E221:5 -a and b -#: E221:5 -1 and b -#: E221:5 -a and 2 -#: E221:1 E221:6 -1 and b -#: E221:1 E221:6 -a and 2 -#: E221:4 -this and False -#: E223:5 -a and b -#: E223:1 -a and b -#: E223:4 E223:9 -this and False +# Okay +from u import (a, b) +from v import c, d +#: E221:13 +from w import (e, f) +#: E275:13 +from w import(e, f) +#: E275:29 +from importable.module import(e, f) +try: + #: E275:33 + from importable.module import(e, f) +except ImportError: + pass +# Okay +True and False +#: E221:8 +True and False +#: E221:4 +True and False +#: E221:2 +if 1: + pass +# Syntax Error, no indentation +#: E903+1 +if 1: +pass +#: E223:8 +True and False +#: E223:4 E223:9 +True and False +#: E221:5 +a and b +#: E221:5 +1 and b +#: E221:5 +a and 2 +#: E221:1 E221:6 +1 and b +#: E221:1 E221:6 +a and 2 +#: E221:4 +this and False +#: E223:5 +a and b +#: E223:1 +a and b +#: E223:4 E223:9 +this and False diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E29.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E29.py index cebbb7bba1..877f35c103 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E29.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E29.py @@ -1,15 +1,15 @@ -# Okay -# 情 -#: W291:5 -print - - -#: W291+1 -class Foo(object): - - bang = 12 - - -#: W291+1:34 -'''multiline -string with trailing whitespace''' +# Okay +# 情 +#: W291:5 +print + + +#: W291+1 +class Foo(object): + + bang = 12 + + +#: W291+1:34 +'''multiline +string with trailing whitespace''' diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E30.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E30.py index 31e241cd44..2439d40b10 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E30.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E30.py @@ -1,177 +1,177 @@ -#: E301+4 -class X: - - def a(): - pass - def b(): - pass - - -#: E301+5 -class X: - - def a(): - pass - # comment - def b(): - pass - - -# -*- coding: utf-8 -*- -def a(): - pass - - -#: E302+1:0 -"""Main module.""" -def _main(): - pass - - -#: E302+1:0 -foo = 1 -def get_sys_path(): - return sys.path - - -#: E302+3:0 -def a(): - pass - -def b(): - pass - - -#: E302+5:0 -def a(): - pass - -# comment - -def b(): - pass - - -#: E303+3:0 -print - - - -#: E303+3:0 E303+4:0 -print - - - - -print -#: E303+3:0 -print - - - -# comment - -print - - -#: E303+3 E303+6 -def a(): - print - - - # comment - - - # another comment - - print - - -#: E302+2 -a = 3 -#: E304+1 -@decorator - -def function(): - pass - - -#: E303+3 -# something - - - -"""This class docstring comes on line 5. -It gives error E303: too many blank lines (3) -""" - - -#: E302+6 -def a(): - print - - # comment - - # another comment -a() - - -#: E302+7 -def a(): - print - - # comment - - # another comment - -try: - a() -except Exception: - pass - - -#: E302+4 -def a(): - print - -# Two spaces before comments, too. -if a(): - a() - - -#: E301+2 -def a(): - x = 1 - def b(): - pass - - -#: E301+2 E301+4 -def a(): - x = 2 - def b(): - x = 1 - def c(): - pass - - -#: E301+2 E301+4 E301+5 -def a(): - x = 1 - class C: - pass - x = 2 - def b(): - pass - - -#: E302+7 -# Example from https://github.com/PyCQA/pycodestyle/issues/400 -foo = 2 - - -def main(): - blah, blah - -if __name__ == '__main__': - main() +#: E301+4 +class X: + + def a(): + pass + def b(): + pass + + +#: E301+5 +class X: + + def a(): + pass + # comment + def b(): + pass + + +# -*- coding: utf-8 -*- +def a(): + pass + + +#: E302+1:0 +"""Main module.""" +def _main(): + pass + + +#: E302+1:0 +foo = 1 +def get_sys_path(): + return sys.path + + +#: E302+3:0 +def a(): + pass + +def b(): + pass + + +#: E302+5:0 +def a(): + pass + +# comment + +def b(): + pass + + +#: E303+3:0 +print + + + +#: E303+3:0 E303+4:0 +print + + + + +print +#: E303+3:0 +print + + + +# comment + +print + + +#: E303+3 E303+6 +def a(): + print + + + # comment + + + # another comment + + print + + +#: E302+2 +a = 3 +#: E304+1 +@decorator + +def function(): + pass + + +#: E303+3 +# something + + + +"""This class docstring comes on line 5. +It gives error E303: too many blank lines (3) +""" + + +#: E302+6 +def a(): + print + + # comment + + # another comment +a() + + +#: E302+7 +def a(): + print + + # comment + + # another comment + +try: + a() +except Exception: + pass + + +#: E302+4 +def a(): + print + +# Two spaces before comments, too. +if a(): + a() + + +#: E301+2 +def a(): + x = 1 + def b(): + pass + + +#: E301+2 E301+4 +def a(): + x = 2 + def b(): + x = 1 + def c(): + pass + + +#: E301+2 E301+4 E301+5 +def a(): + x = 1 + class C: + pass + x = 2 + def b(): + pass + + +#: E302+7 +# Example from https://github.com/PyCQA/pycodestyle/issues/400 +foo = 2 + + +def main(): + blah, blah + +if __name__ == '__main__': + main() diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E30not.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E30not.py index c0c005ccd2..0abb6124ff 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E30not.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E30not.py @@ -1,175 +1,175 @@ -# Okay -class X: - pass -# Okay - - -def foo(): - pass - - -# Okay -# -*- coding: utf-8 -*- -class X: - pass - - -# Okay -# -*- coding: utf-8 -*- -def foo(): - pass - - -# Okay -class X: - - def a(): - pass - - # comment - def b(): - pass - - # This is a - # ... multi-line comment - - def c(): - pass - - -# This is a -# ... multi-line comment - -@some_decorator -class Y: - - def a(): - pass - - # comment - - def b(): - pass - - @property - def c(): - pass - - -try: - from nonexistent import Bar -except ImportError: - class Bar(object): - """This is a Bar replacement""" - - -def with_feature(f): - """Some decorator""" - wrapper = f - if has_this_feature(f): - def wrapper(*args): - call_feature(args[0]) - return f(*args) - return wrapper - - -try: - next -except NameError: - def next(iterator, default): - for item in iterator: - return item - return default - - -def a(): - pass - - -class Foo(): - """Class Foo""" - - def b(): - - pass - - -# comment -def c(): - pass - - -# comment - - -def d(): - pass - -# This is a -# ... multi-line comment - -# And this one is -# ... a second paragraph -# ... which spans on 3 lines - - -# Function `e` is below -# NOTE: Hey this is a testcase - -def e(): - pass - - -def a(): - print - - # comment - - print - - print - -# Comment 1 - -# Comment 2 - - -# Comment 3 - -def b(): - - pass - - -# Okay -def foo(): - pass - - -def bar(): - pass - - -class Foo(object): - pass - - -class Bar(object): - pass - - -if __name__ == '__main__': - foo() -# Okay -classification_errors = None -# Okay -defined_properly = True -# Okay -defaults = {} -defaults.update({}) - - -# Okay -def foo(x): - classification = x - definitely = not classification +# Okay +class X: + pass +# Okay + + +def foo(): + pass + + +# Okay +# -*- coding: utf-8 -*- +class X: + pass + + +# Okay +# -*- coding: utf-8 -*- +def foo(): + pass + + +# Okay +class X: + + def a(): + pass + + # comment + def b(): + pass + + # This is a + # ... multi-line comment + + def c(): + pass + + +# This is a +# ... multi-line comment + +@some_decorator +class Y: + + def a(): + pass + + # comment + + def b(): + pass + + @property + def c(): + pass + + +try: + from nonexistent import Bar +except ImportError: + class Bar(object): + """This is a Bar replacement""" + + +def with_feature(f): + """Some decorator""" + wrapper = f + if has_this_feature(f): + def wrapper(*args): + call_feature(args[0]) + return f(*args) + return wrapper + + +try: + next +except NameError: + def next(iterator, default): + for item in iterator: + return item + return default + + +def a(): + pass + + +class Foo(): + """Class Foo""" + + def b(): + + pass + + +# comment +def c(): + pass + + +# comment + + +def d(): + pass + +# This is a +# ... multi-line comment + +# And this one is +# ... a second paragraph +# ... which spans on 3 lines + + +# Function `e` is below +# NOTE: Hey this is a testcase + +def e(): + pass + + +def a(): + print + + # comment + + print + + print + +# Comment 1 + +# Comment 2 + + +# Comment 3 + +def b(): + + pass + + +# Okay +def foo(): + pass + + +def bar(): + pass + + +class Foo(object): + pass + + +class Bar(object): + pass + + +if __name__ == '__main__': + foo() +# Okay +classification_errors = None +# Okay +defined_properly = True +# Okay +defaults = {} +defaults.update({}) + + +# Okay +def foo(x): + classification = x + definitely = not classification diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E40.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E40.py index 93a2ccf386..708412864c 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E40.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E40.py @@ -1,39 +1,39 @@ -#: E401:7 -import os, sys -# Okay -import os -import sys - -from subprocess import Popen, PIPE - -from myclass import MyClass -from foo.bar.yourclass import YourClass - -import myclass -import foo.bar.yourclass -# All Okay from here until the definition of VERSION -__all__ = ['abc'] - -import foo -__version__ = "42" - -import foo -__author__ = "Simon Gomizelj" - -import foo -try: - import foo -except ImportError: - pass -else: - hello('imported foo') -finally: - hello('made attempt to import foo') - -import bar -VERSION = '1.2.3' - -#: E402 -import foo -#: E402 -import foo +#: E401:7 +import os, sys +# Okay +import os +import sys + +from subprocess import Popen, PIPE + +from myclass import MyClass +from foo.bar.yourclass import YourClass + +import myclass +import foo.bar.yourclass +# All Okay from here until the definition of VERSION +__all__ = ['abc'] + +import foo +__version__ = "42" + +import foo +__author__ = "Simon Gomizelj" + +import foo +try: + import foo +except ImportError: + pass +else: + hello('imported foo') +finally: + hello('made attempt to import foo') + +import bar +VERSION = '1.2.3' + +#: E402 +import foo +#: E402 +import foo diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E50.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E50.py index 67fd55833c..6dac742802 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E50.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E50.py @@ -1,126 +1,126 @@ -#: E501:4 -a = '12345678901234567890123456789012345678901234567890123456789012345678901234567890' -#: E501:80 -a = '1234567890123456789012345678901234567890123456789012345678901234567890' or \ - 6 -#: E501+1:80 -a = 7 or \ - '1234567890123456789012345678901234567890123456789012345678901234567890' or \ - 6 -#: E501+1:80 E501+2:80 -a = 7 or \ - '1234567890123456789012345678901234567890123456789012345678901234567890' or \ - '1234567890123456789012345678901234567890123456789012345678901234567890' or \ - 6 -#: E501:78 -a = '1234567890123456789012345678901234567890123456789012345678901234567890' # \ -#: E502:78 -a = ('123456789012345678901234567890123456789012345678901234567890123456789' \ - '01234567890') -#: E502+1:11 -a = ('AAA \ - BBB' \ - 'CCC') -#: E502:38 -if (foo is None and bar is "e000" and \ - blah == 'yeah'): - blah = 'yeahnah' -# -# Okay -a = ('AAA' - 'BBB') - -a = ('AAA \ - BBB' - 'CCC') - -a = 'AAA' \ - 'BBB' \ - 'CCC' - -a = ('AAA\ -BBBBBBBBB\ -CCCCCCCCC\ -DDDDDDDDD') -# -# Okay -if aaa: - pass -elif bbb or \ - ccc: - pass - -ddd = \ - ccc - -('\ - ' + ' \ -') -(''' - ''' + ' \ -') -#: E501:67 E225:21 E225:22 -very_long_identifiers=and_terrible_whitespace_habits(are_no_excuse+for_long_lines) -# -# TODO Long multiline strings are not handled. E501? -'''multiline string -with a long long long long long long long long long long long long long long long long line -''' -#: E501 -'''same thing, but this time without a terminal newline in the string -long long long long long long long long long long long long long long long long line''' -# -# issue 224 (unavoidable long lines in docstrings) -# Okay -""" -I'm some great documentation. Because I'm some great documentation, I'm -going to give you a reference to some valuable information about some API -that I'm calling: - - http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx -""" -#: E501 -""" -longnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaces""" - - -# Regression test for #622 -def foo(): - """Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis pulvinar vitae - """ - - -# Okay -""" -This - almost_empty_line -""" - -""" -This - almost_empty_line -""" -# A basic comment -#: E501 -# with a long long long long long long long long long long long long long long long long line - -# -# Okay -# I'm some great comment. Because I'm so great, I'm going to give you a -# reference to some valuable information about some API that I'm calling: -# -# http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx - -x = 3 - -# longnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaces - -# -# Okay -# This -# almost_empty_line - -# -#: E501+1 -# This -# almost_empty_line +#: E501:4 +a = '12345678901234567890123456789012345678901234567890123456789012345678901234567890' +#: E501:80 +a = '1234567890123456789012345678901234567890123456789012345678901234567890' or \ + 6 +#: E501+1:80 +a = 7 or \ + '1234567890123456789012345678901234567890123456789012345678901234567890' or \ + 6 +#: E501+1:80 E501+2:80 +a = 7 or \ + '1234567890123456789012345678901234567890123456789012345678901234567890' or \ + '1234567890123456789012345678901234567890123456789012345678901234567890' or \ + 6 +#: E501:78 +a = '1234567890123456789012345678901234567890123456789012345678901234567890' # \ +#: E502:78 +a = ('123456789012345678901234567890123456789012345678901234567890123456789' \ + '01234567890') +#: E502+1:11 +a = ('AAA \ + BBB' \ + 'CCC') +#: E502:38 +if (foo is None and bar is "e000" and \ + blah == 'yeah'): + blah = 'yeahnah' +# +# Okay +a = ('AAA' + 'BBB') + +a = ('AAA \ + BBB' + 'CCC') + +a = 'AAA' \ + 'BBB' \ + 'CCC' + +a = ('AAA\ +BBBBBBBBB\ +CCCCCCCCC\ +DDDDDDDDD') +# +# Okay +if aaa: + pass +elif bbb or \ + ccc: + pass + +ddd = \ + ccc + +('\ + ' + ' \ +') +(''' + ''' + ' \ +') +#: E501:67 E225:21 E225:22 +very_long_identifiers=and_terrible_whitespace_habits(are_no_excuse+for_long_lines) +# +# TODO Long multiline strings are not handled. E501? +'''multiline string +with a long long long long long long long long long long long long long long long long line +''' +#: E501 +'''same thing, but this time without a terminal newline in the string +long long long long long long long long long long long long long long long long line''' +# +# issue 224 (unavoidable long lines in docstrings) +# Okay +""" +I'm some great documentation. Because I'm some great documentation, I'm +going to give you a reference to some valuable information about some API +that I'm calling: + + http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx +""" +#: E501 +""" +longnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaces""" + + +# Regression test for #622 +def foo(): + """Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis pulvinar vitae + """ + + +# Okay +""" +This + almost_empty_line +""" + +""" +This + almost_empty_line +""" +# A basic comment +#: E501 +# with a long long long long long long long long long long long long long long long long line + +# +# Okay +# I'm some great comment. Because I'm so great, I'm going to give you a +# reference to some valuable information about some API that I'm calling: +# +# http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx + +x = 3 + +# longnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaceslongnospaces + +# +# Okay +# This +# almost_empty_line + +# +#: E501+1 +# This +# almost_empty_line diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E70.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E70.py index be11fb1def..96ffe35ac5 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E70.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E70.py @@ -1,25 +1,25 @@ -#: E701:6 -if a: a = False -#: E701:41 -if not header or header[:6] != 'bytes=': pass -#: E702:9 -a = False; b = True -#: E702:16 E402 -import bdist_egg; bdist_egg.write_safety_flag(cmd.egg_info, safe) -#: E703:12 E402 -import shlex; -#: E702:8 E703:22 -del a[:]; a.append(42); - - -#: E704:10 -def f(x): return 2 - - -#: E704:10 -def f(x): return 2 * x - - -while all is round: - #: E704:14 - def f(x): return 2 * x +#: E701:6 +if a: a = False +#: E701:41 +if not header or header[:6] != 'bytes=': pass +#: E702:9 +a = False; b = True +#: E702:16 E402 +import bdist_egg; bdist_egg.write_safety_flag(cmd.egg_info, safe) +#: E703:12 E402 +import shlex; +#: E702:8 E703:22 +del a[:]; a.append(42); + + +#: E704:10 +def f(x): return 2 + + +#: E704:10 +def f(x): return 2 * x + + +while all is round: + #: E704:14 + def f(x): return 2 * x diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E71.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E71.py index 109dcd6c77..2ea464cafc 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E71.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E71.py @@ -1,93 +1,93 @@ -#: E711:7 -if res == None: - pass -#: E711:7 -if res != None: - pass -#: E711:8 -if None == res: - pass -#: E711:8 -if None != res: - pass -#: E711:10 -if res[1] == None: - pass -#: E711:10 -if res[1] != None: - pass -#: E711:8 -if None != res[1]: - pass -#: E711:8 -if None == res[1]: - pass - -# -#: E712:7 -if res == True: - pass -#: E712:7 -if res != False: - pass -#: E712:8 -if True != res: - pass -#: E712:9 -if False == res: - pass -#: E712:10 -if res[1] == True: - pass -#: E712:10 -if res[1] != False: - pass - -if x is False: - pass - -# -#: E713:9 -if not X in Y: - pass -#: E713:11 -if not X.B in Y: - pass -#: E713:9 -if not X in Y and Z == "zero": - pass -#: E713:24 -if X == "zero" or not Y in Z: - pass - -# -#: E714:9 -if not X is Y: - pass -#: E714:11 -if not X.B is Y: - pass - -# -# Okay -if x not in y: - pass - -if not (X in Y or X is Z): - pass - -if not (X in Y): - pass - -if x is not y: - pass - -if TrueElement.get_element(True) == TrueElement.get_element(False): - pass - -if (True) == TrueElement or x == TrueElement: - pass - -assert (not foo) in bar -assert {'x': not foo} in bar -assert [42, not foo] in bar +#: E711:7 +if res == None: + pass +#: E711:7 +if res != None: + pass +#: E711:8 +if None == res: + pass +#: E711:8 +if None != res: + pass +#: E711:10 +if res[1] == None: + pass +#: E711:10 +if res[1] != None: + pass +#: E711:8 +if None != res[1]: + pass +#: E711:8 +if None == res[1]: + pass + +# +#: E712:7 +if res == True: + pass +#: E712:7 +if res != False: + pass +#: E712:8 +if True != res: + pass +#: E712:9 +if False == res: + pass +#: E712:10 +if res[1] == True: + pass +#: E712:10 +if res[1] != False: + pass + +if x is False: + pass + +# +#: E713:9 +if not X in Y: + pass +#: E713:11 +if not X.B in Y: + pass +#: E713:9 +if not X in Y and Z == "zero": + pass +#: E713:24 +if X == "zero" or not Y in Z: + pass + +# +#: E714:9 +if not X is Y: + pass +#: E714:11 +if not X.B is Y: + pass + +# +# Okay +if x not in y: + pass + +if not (X in Y or X is Z): + pass + +if not (X in Y): + pass + +if x is not y: + pass + +if TrueElement.get_element(True) == TrueElement.get_element(False): + pass + +if (True) == TrueElement or x == TrueElement: + pass + +assert (not foo) in bar +assert {'x': not foo} in bar +assert [42, not foo] in bar diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E72.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E72.py index 2e9ef9151d..545348e60b 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E72.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E72.py @@ -1,79 +1,79 @@ -#: E721:3 -if type(res) == type(42): - pass -#: E721:3 -if type(res) != type(""): - pass - -import types - -if res == types.IntType: - pass - -import types - -#: E721:3 -if type(res) is not types.ListType: - pass -#: E721:7 E721:35 -assert type(res) == type(False) or type(res) == type(None) -#: E721:7 -assert type(res) == type([]) -#: E721:7 -assert type(res) == type(()) -#: E721:7 -assert type(res) == type((0,)) -#: E721:7 -assert type(res) == type((0)) -#: E721:7 -assert type(res) != type((1,)) -#: E721:7 -assert type(res) is type((1,)) -#: E721:7 -assert type(res) is not type((1,)) - -# Okay -#: E402 -import types - -if isinstance(res, int): - pass -if isinstance(res, str): - pass -if isinstance(res, types.MethodType): - pass - -#: E721:3 E721:25 -if type(a) != type(b) or type(a) == type(ccc): - pass -#: E721 -type(a) != type(b) -#: E721 -1 != type(b) -#: E721 -type(b) != 1 -1 != 1 - -try: - pass -#: E722 -except: - pass -try: - pass -except Exception: - pass -#: E722 -except: - pass -# Okay -fake_code = """" -try: - do_something() -except: - pass -""" -try: - pass -except Exception: - pass +#: E721:3 +if type(res) == type(42): + pass +#: E721:3 +if type(res) != type(""): + pass + +import types + +if res == types.IntType: + pass + +import types + +#: E721:3 +if type(res) is not types.ListType: + pass +#: E721:7 E721:35 +assert type(res) == type(False) or type(res) == type(None) +#: E721:7 +assert type(res) == type([]) +#: E721:7 +assert type(res) == type(()) +#: E721:7 +assert type(res) == type((0,)) +#: E721:7 +assert type(res) == type((0)) +#: E721:7 +assert type(res) != type((1,)) +#: E721:7 +assert type(res) is type((1,)) +#: E721:7 +assert type(res) is not type((1,)) + +# Okay +#: E402 +import types + +if isinstance(res, int): + pass +if isinstance(res, str): + pass +if isinstance(res, types.MethodType): + pass + +#: E721:3 E721:25 +if type(a) != type(b) or type(a) == type(ccc): + pass +#: E721 +type(a) != type(b) +#: E721 +1 != type(b) +#: E721 +type(b) != 1 +1 != 1 + +try: + pass +#: E722 +except: + pass +try: + pass +except Exception: + pass +#: E722 +except: + pass +# Okay +fake_code = """" +try: + do_something() +except: + pass +""" +try: + pass +except Exception: + pass diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/E73.py b/contrib/python/parso/py3/tests/normalizer_issue_files/E73.py index 77e2e9043a..1eaf5e8f00 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/E73.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/E73.py @@ -1,16 +1,16 @@ -#: E731:4 -f = lambda x: 2 * x -while False: - #: E731:10 - foo = lambda y, z: 2 * x -# Okay -f = object() -f.method = lambda: 'Method' - -f = {} -f['a'] = lambda x: x ** 2 - -f = [] -f.append(lambda x: x ** 2) - -lambda: 'no-op' +#: E731:4 +f = lambda x: 2 * x +while False: + #: E731:10 + foo = lambda y, z: 2 * x +# Okay +f = object() +f.method = lambda: 'Method' + +f = {} +f['a'] = lambda x: x ** 2 + +f = [] +f.append(lambda x: x ** 2) + +lambda: 'no-op' diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/LICENSE b/contrib/python/parso/py3/tests/normalizer_issue_files/LICENSE index 142a508a63..4e6aa1fc31 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/LICENSE +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/LICENSE @@ -1,29 +1,29 @@ -Copyright © 2006-2009 Johann C. Rocholl <johann@rocholl.net> -Copyright © 2009-2014 Florent Xicluna <florent.xicluna@gmail.com> -Copyright © 2014-2016 Ian Lee <IanLee1521@gmail.com> -Copyright © 2017-???? Dave Halter <davidhalter88@gmail.com> - -Dave: The files in this folder were ported from pydocstyle and some -modifications where made. - -Licensed under the terms of the Expat License - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation files -(the "Software"), to deal in the Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, -and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Copyright © 2006-2009 Johann C. Rocholl <johann@rocholl.net> +Copyright © 2009-2014 Florent Xicluna <florent.xicluna@gmail.com> +Copyright © 2014-2016 Ian Lee <IanLee1521@gmail.com> +Copyright © 2017-???? Dave Halter <davidhalter88@gmail.com> + +Dave: The files in this folder were ported from pydocstyle and some +modifications where made. + +Licensed under the terms of the Expat License + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/allowed_syntax.py b/contrib/python/parso/py3/tests/normalizer_issue_files/allowed_syntax.py index 9cccf619cb..e67a838c24 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/allowed_syntax.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/allowed_syntax.py @@ -1,163 +1,163 @@ -""" -Some syntax errors are a bit complicated and need exact checking. Here we -gather some of the potentially dangerous ones. -""" - -from __future__ import division - -# With a dot it's not a future import anymore. -from .__future__ import absolute_import - -'' '' -''r''u'' -b'' BR'' - - -for x in [1]: - break - continue - -try: - pass -except ZeroDivisionError: - pass - #: E722:0 -except: - pass - -try: - pass - #: E722:0 E901:0 -except: - pass -except ZeroDivisionError: - pass - - -r'\n' -r'\x' -b'\n' - - -a = 3 - - -def x(b=a): - global a - - -def x(*args, c=2, d): - pass - - -def x(*, c=2, d): - pass - - -def x(a, b=1, *args, c=2, d): - pass - - -def x(a, b=1, *, c=2, d): - pass - - -lambda *args, c=2, d: (c, d) -lambda *, c=2, d: (c, d) -lambda a, b=1, *args, c=2, d: (c, d) -lambda a, b=1, *, c=2, d: (c, d) - - -*foo, a = (1,) -*foo[0], a = (1,) -*[], a = (1,) - - -async def foo(): - await bar() - #: E901 - yield from [] - return - #: E901 - return '' - - -# With decorator it's a different statement. -@bla -async def foo(): - await bar() - #: E901 - yield from [] - return - #: E901 - return '' - - -foo: int = 4 -(foo): int = 3 -((foo)): int = 3 -foo.bar: int -foo[3]: int - - -def glob(): - global x - y: foo = x - - -def c(): - a = 3 - - def d(): - class X(): - nonlocal a - - -def x(): - a = 3 - - def y(): - nonlocal a - - -def x(): - def y(): - nonlocal a - - a = 3 - - -def x(): - a = 3 - - def y(): - class z(): - nonlocal a - - -def x(a): - def y(): - nonlocal a - - -def x(a, b): - def y(): - nonlocal b - nonlocal a - - -def x(a): - def y(): - def z(): - nonlocal a - - -def x(): - def y(a): - def z(): - nonlocal a - - -a = *args, *args -error[(*args, *args)] = 3 -*args, *args +""" +Some syntax errors are a bit complicated and need exact checking. Here we +gather some of the potentially dangerous ones. +""" + +from __future__ import division + +# With a dot it's not a future import anymore. +from .__future__ import absolute_import + +'' '' +''r''u'' +b'' BR'' + + +for x in [1]: + break + continue + +try: + pass +except ZeroDivisionError: + pass + #: E722:0 +except: + pass + +try: + pass + #: E722:0 E901:0 +except: + pass +except ZeroDivisionError: + pass + + +r'\n' +r'\x' +b'\n' + + +a = 3 + + +def x(b=a): + global a + + +def x(*args, c=2, d): + pass + + +def x(*, c=2, d): + pass + + +def x(a, b=1, *args, c=2, d): + pass + + +def x(a, b=1, *, c=2, d): + pass + + +lambda *args, c=2, d: (c, d) +lambda *, c=2, d: (c, d) +lambda a, b=1, *args, c=2, d: (c, d) +lambda a, b=1, *, c=2, d: (c, d) + + +*foo, a = (1,) +*foo[0], a = (1,) +*[], a = (1,) + + +async def foo(): + await bar() + #: E901 + yield from [] + return + #: E901 + return '' + + +# With decorator it's a different statement. +@bla +async def foo(): + await bar() + #: E901 + yield from [] + return + #: E901 + return '' + + +foo: int = 4 +(foo): int = 3 +((foo)): int = 3 +foo.bar: int +foo[3]: int + + +def glob(): + global x + y: foo = x + + +def c(): + a = 3 + + def d(): + class X(): + nonlocal a + + +def x(): + a = 3 + + def y(): + nonlocal a + + +def x(): + def y(): + nonlocal a + + a = 3 + + +def x(): + a = 3 + + def y(): + class z(): + nonlocal a + + +def x(a): + def y(): + nonlocal a + + +def x(a, b): + def y(): + nonlocal b + nonlocal a + + +def x(a): + def y(): + def z(): + nonlocal a + + +def x(): + def y(a): + def z(): + nonlocal a + + +a = *args, *args +error[(*args, *args)] = 3 +*args, *args diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/latin-1.py b/contrib/python/parso/py3/tests/normalizer_issue_files/latin-1.py deleted file mode 100644 index 8328cfba9e..0000000000 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/latin-1.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: latin-1 -*- -# Test non-UTF8 encoding -latin1 = ('' - '') - -c = ("w") diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/python.py b/contrib/python/parso/py3/tests/normalizer_issue_files/python.py index 566e90360a..9afc3821b3 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/python.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/python.py @@ -1,90 +1,90 @@ -#!/usr/bin/env python3 -from typing import ClassVar, List - -print(1, 2) - - -# Annotated function (Issue #29) -def foo(x: int) -> int: - return x + 1 - - -# Annotated variables #575 -CONST: int = 42 - - -class Class: - cls_var: ClassVar[str] - - def m(self): - xs: List[int] = [] - - -# True and False are keywords in Python 3 and therefore need a space. -#: E275:13 E275:14 -norman = True+False - - -#: E302+3:0 -def a(): - pass - -async def b(): - pass - - -# Okay -async def add(a: int = 0, b: int = 0) -> int: - return a + b - - -# Previously E251 four times -#: E221:5 -async def add(a: int = 0, b: int = 0) -> int: - return a + b - - -# Previously just E272+1:5 E272+4:5 -#: E302+3 E221:5 E221+3:5 -async def x(): - pass - -async def x(y: int = 1): - pass - - -#: E704:16 -async def f(x): return 2 - - -a[b1, :] == a[b1, ...] - - -# Annotated Function Definitions -# Okay -def munge(input: AnyStr, sep: AnyStr = None, limit=1000, - extra: Union[str, dict] = None) -> AnyStr: - pass - - -#: E225:24 E225:26 -def x(b: tuple = (1, 2))->int: - return a + b - - -#: E252:11 E252:12 E231:8 -def b(a:int=1): - pass - - -if alpha[:-i]: - *a, b = (1, 2, 3) - - -# Named only arguments -def foo(*, asdf): - pass - - -def foo2(bar, *, asdf=2): - pass +#!/usr/bin/env python3 +from typing import ClassVar, List + +print(1, 2) + + +# Annotated function (Issue #29) +def foo(x: int) -> int: + return x + 1 + + +# Annotated variables #575 +CONST: int = 42 + + +class Class: + cls_var: ClassVar[str] + + def m(self): + xs: List[int] = [] + + +# True and False are keywords in Python 3 and therefore need a space. +#: E275:13 E275:14 +norman = True+False + + +#: E302+3:0 +def a(): + pass + +async def b(): + pass + + +# Okay +async def add(a: int = 0, b: int = 0) -> int: + return a + b + + +# Previously E251 four times +#: E221:5 +async def add(a: int = 0, b: int = 0) -> int: + return a + b + + +# Previously just E272+1:5 E272+4:5 +#: E302+3 E221:5 E221+3:5 +async def x(): + pass + +async def x(y: int = 1): + pass + + +#: E704:16 +async def f(x): return 2 + + +a[b1, :] == a[b1, ...] + + +# Annotated Function Definitions +# Okay +def munge(input: AnyStr, sep: AnyStr = None, limit=1000, + extra: Union[str, dict] = None) -> AnyStr: + pass + + +#: E225:24 E225:26 +def x(b: tuple = (1, 2))->int: + return a + b + + +#: E252:11 E252:12 E231:8 +def b(a:int=1): + pass + + +if alpha[:-i]: + *a, b = (1, 2, 3) + + +# Named only arguments +def foo(*, asdf): + pass + + +def foo2(bar, *, asdf=2): + pass diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8-bom.py b/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8-bom.py index 9c065c9494..e690e82ad6 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8-bom.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8-bom.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -hello = 'こんにちわ' - -# EOF +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +hello = 'こんにちわ' + +# EOF diff --git a/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8.py b/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8.py index 73ea9a2827..9bc17dc3bf 100644 --- a/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8.py +++ b/contrib/python/parso/py3/tests/normalizer_issue_files/utf-8.py @@ -1,35 +1,35 @@ -# -*- coding: utf-8 -*- - -# Some random text with multi-byte characters (utf-8 encoded) -# -# Εδώ μάτσο κειμένων τη, τρόπο πιθανό διευθυντές ώρα μη. Νέων απλό παράγει ροή -# κι, το επί δεδομένη καθορίζουν. Πάντως ζητήσεις περιβάλλοντος ένα με, τη -# ξέχασε αρπάζεις φαινόμενο όλη. Τρέξει εσφαλμένη χρησιμοποίησέ νέα τι. Θα όρο -# πετάνε φακέλους, άρα με διακοπής λαμβάνουν εφαμοργής. Λες κι μειώσει -# καθυστερεί. - -# 79 narrow chars -# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 [79] - -# 78 narrow chars (Na) + 1 wide char (W) -# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8情 - -# 3 narrow chars (Na) + 40 wide chars (W) -# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 - -# 3 narrow chars (Na) + 76 wide chars (W) -# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 - -# -# 80 narrow chars (Na) -#: E501 -# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 [80] -# -# 78 narrow chars (Na) + 2 wide char (W) -#: E501 -# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8情情 -# -# 3 narrow chars (Na) + 77 wide chars (W) -#: E501 -# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 -# +# -*- coding: utf-8 -*- + +# Some random text with multi-byte characters (utf-8 encoded) +# +# Εδώ μάτσο κειμένων τη, τρόπο πιθανό διευθυντές ώρα μη. Νέων απλό παράγει ροή +# κι, το επί δεδομένη καθορίζουν. Πάντως ζητήσεις περιβάλλοντος ένα με, τη +# ξέχασε αρπάζεις φαινόμενο όλη. Τρέξει εσφαλμένη χρησιμοποίησέ νέα τι. Θα όρο +# πετάνε φακέλους, άρα με διακοπής λαμβάνουν εφαμοργής. Λες κι μειώσει +# καθυστερεί. + +# 79 narrow chars +# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 [79] + +# 78 narrow chars (Na) + 1 wide char (W) +# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8情 + +# 3 narrow chars (Na) + 40 wide chars (W) +# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 + +# 3 narrow chars (Na) + 76 wide chars (W) +# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 + +# +# 80 narrow chars (Na) +#: E501 +# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 [80] +# +# 78 narrow chars (Na) + 2 wide char (W) +#: E501 +# 01 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8情情 +# +# 3 narrow chars (Na) + 77 wide chars (W) +#: E501 +# 情 情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情情 +# diff --git a/contrib/python/parso/py3/tests/test_cache.py b/contrib/python/parso/py3/tests/test_cache.py index d19a872e14..c1338e1cf5 100644 --- a/contrib/python/parso/py3/tests/test_cache.py +++ b/contrib/python/parso/py3/tests/test_cache.py @@ -1,197 +1,197 @@ -""" -Test all things related to the ``jedi.cache`` module. -""" - -import os -import pytest -import time -from pathlib import Path - -from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG, - _get_cache_clear_lock_path, _get_hashed_path, - _load_from_file_system, _NodeCacheItem, - _remove_cache_and_update_lock, _save_to_file_system, - load_module, parser_cache, try_to_save_module) -from parso._compatibility import is_pypy -from parso import load_grammar -from parso import cache -from parso import file_io -from parso import parse - -skip_pypy = pytest.mark.skipif( - is_pypy, - reason="pickling in pypy is slow, since we don't pickle," - "we never go into path of auto-collecting garbage" -) - - -@pytest.fixture() -def isolated_parso_cache(monkeypatch, tmpdir): - """Set `parso.cache._default_cache_path` to a temporary directory - during the test. """ - cache_path = Path(str(tmpdir), "__parso_cache") - monkeypatch.setattr(cache, '_default_cache_path', cache_path) - return cache_path - - -@pytest.mark.skip("SUBBOTNIK-2721 Disable load cache from disk") -def test_modulepickling_change_cache_dir(tmpdir): - """ - ParserPickling should not save old cache when cache_directory is changed. - - See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_ - """ - dir_1 = Path(str(tmpdir.mkdir('first'))) - dir_2 = Path(str(tmpdir.mkdir('second'))) - - item_1 = _NodeCacheItem('bla', []) - item_2 = _NodeCacheItem('bla', []) - path_1 = Path('fake path 1') - path_2 = Path('fake path 2') - - hashed_grammar = load_grammar()._hashed - _save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1) - parser_cache.clear() - cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1) - assert cached == item_1.node - - _save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2) - cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2) - assert cached is None - - -def load_stored_item(hashed_grammar, path, item, cache_path): - """Load `item` stored at `path` in `cache`.""" - item = _load_from_file_system(hashed_grammar, path, item.change_time - 1, cache_path) - return item - - -@pytest.mark.usefixtures("isolated_parso_cache") -def test_modulepickling_simulate_deleted_cache(tmpdir): - """ - Tests loading from a cache file after it is deleted. - According to macOS `dev docs`__, - - Note that the system may delete the Caches/ directory to free up disk - space, so your app must be able to re-create or download these files as - needed. - - It is possible that other supported platforms treat cache files the same - way. - - __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html - """ # noqa - grammar = load_grammar() - module = 'fake parser' - - # Create the file - path = Path(str(tmpdir.dirname), 'some_path') - with open(path, 'w'): - pass - io = file_io.FileIO(path) - - try_to_save_module(grammar._hashed, io, module, lines=[]) - assert load_module(grammar._hashed, io) == module - - os.unlink(_get_hashed_path(grammar._hashed, path)) - parser_cache.clear() - - cached2 = load_module(grammar._hashed, io) - assert cached2 is None - - -@pytest.mark.skip -def test_cache_limit(): - def cache_size(): - return sum(len(v) for v in parser_cache.values()) - - try: - parser_cache.clear() - future_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() + 10e6) - old_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() - 10e4) - parser_cache['some_hash_old'] = { - '/path/%s' % i: old_node_cache_item for i in range(300) - } - parser_cache['some_hash_new'] = { - '/path/%s' % i: future_node_cache_item for i in range(300) - } - assert cache_size() == 600 - parse('somecode', cache=True, path='/path/somepath') - assert cache_size() == 301 - finally: - parser_cache.clear() - - -class _FixedTimeFileIO(file_io.KnownContentFileIO): - def __init__(self, path, content, last_modified): - super().__init__(path, content) - self._last_modified = last_modified - - def get_last_modified(self): - return self._last_modified - - -@pytest.mark.skip -@pytest.mark.parametrize('diff_cache', [False, True]) -@pytest.mark.parametrize('use_file_io', [False, True]) -def test_cache_last_used_update(diff_cache, use_file_io): - p = Path('/path/last-used') - parser_cache.clear() # Clear, because then it's easier to find stuff. - parse('somecode', cache=True, path=p) - node_cache_item = next(iter(parser_cache.values()))[p] - now = time.time() - assert node_cache_item.last_used <= now - - if use_file_io: - f = _FixedTimeFileIO(p, 'code', node_cache_item.last_used - 10) - parse(file_io=f, cache=True, diff_cache=diff_cache) - else: - parse('somecode2', cache=True, path=p, diff_cache=diff_cache) - - node_cache_item = next(iter(parser_cache.values()))[p] - assert now <= node_cache_item.last_used <= time.time() - - -@skip_pypy -def test_inactive_cache(tmpdir, isolated_parso_cache): - parser_cache.clear() - test_subjects = "abcdef" - for path in test_subjects: - parse('somecode', cache=True, path=os.path.join(str(tmpdir), path)) - raw_cache_path = isolated_parso_cache.joinpath(_VERSION_TAG) - assert raw_cache_path.exists() - dir_names = os.listdir(raw_cache_path) - a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL - old_paths = set() - for dir_name in dir_names[:len(test_subjects) // 2]: # make certain number of paths old - os.utime(raw_cache_path.joinpath(dir_name), (a_while_ago, a_while_ago)) - old_paths.add(dir_name) - # nothing should be cleared while the lock is on - assert _get_cache_clear_lock_path().exists() - _remove_cache_and_update_lock() # it shouldn't clear anything - assert len(os.listdir(raw_cache_path)) == len(test_subjects) - assert old_paths.issubset(os.listdir(raw_cache_path)) - - os.utime(_get_cache_clear_lock_path(), (a_while_ago, a_while_ago)) - _remove_cache_and_update_lock() - assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2 - assert not old_paths.intersection(os.listdir(raw_cache_path)) - - -@pytest.mark.skip -@skip_pypy -def test_permission_error(monkeypatch): - def save(*args, **kwargs): - nonlocal was_called - was_called = True - raise PermissionError - - was_called = False - - monkeypatch.setattr(cache, '_save_to_file_system', save) - try: - with pytest.warns(Warning): - parse(path=__file__, cache=True, diff_cache=True) - assert was_called - finally: - parser_cache.clear() +""" +Test all things related to the ``jedi.cache`` module. +""" + +import os +import pytest +import time +from pathlib import Path + +from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG, + _get_cache_clear_lock_path, _get_hashed_path, + _load_from_file_system, _NodeCacheItem, + _remove_cache_and_update_lock, _save_to_file_system, + load_module, parser_cache, try_to_save_module) +from parso._compatibility import is_pypy +from parso import load_grammar +from parso import cache +from parso import file_io +from parso import parse + +skip_pypy = pytest.mark.skipif( + is_pypy, + reason="pickling in pypy is slow, since we don't pickle," + "we never go into path of auto-collecting garbage" +) + + +@pytest.fixture() +def isolated_parso_cache(monkeypatch, tmpdir): + """Set `parso.cache._default_cache_path` to a temporary directory + during the test. """ + cache_path = Path(str(tmpdir), "__parso_cache") + monkeypatch.setattr(cache, '_default_cache_path', cache_path) + return cache_path + + +@pytest.mark.skip("SUBBOTNIK-2721 Disable load cache from disk") +def test_modulepickling_change_cache_dir(tmpdir): + """ + ParserPickling should not save old cache when cache_directory is changed. + + See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_ + """ + dir_1 = Path(str(tmpdir.mkdir('first'))) + dir_2 = Path(str(tmpdir.mkdir('second'))) + + item_1 = _NodeCacheItem('bla', []) + item_2 = _NodeCacheItem('bla', []) + path_1 = Path('fake path 1') + path_2 = Path('fake path 2') + + hashed_grammar = load_grammar()._hashed + _save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1) + parser_cache.clear() + cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1) + assert cached == item_1.node + + _save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2) + cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2) + assert cached is None + + +def load_stored_item(hashed_grammar, path, item, cache_path): + """Load `item` stored at `path` in `cache`.""" + item = _load_from_file_system(hashed_grammar, path, item.change_time - 1, cache_path) + return item + + +@pytest.mark.usefixtures("isolated_parso_cache") +def test_modulepickling_simulate_deleted_cache(tmpdir): + """ + Tests loading from a cache file after it is deleted. + According to macOS `dev docs`__, + + Note that the system may delete the Caches/ directory to free up disk + space, so your app must be able to re-create or download these files as + needed. + + It is possible that other supported platforms treat cache files the same + way. + + __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html + """ # noqa + grammar = load_grammar() + module = 'fake parser' + + # Create the file + path = Path(str(tmpdir.dirname), 'some_path') + with open(path, 'w'): + pass + io = file_io.FileIO(path) + + try_to_save_module(grammar._hashed, io, module, lines=[]) + assert load_module(grammar._hashed, io) == module + + os.unlink(_get_hashed_path(grammar._hashed, path)) + parser_cache.clear() + + cached2 = load_module(grammar._hashed, io) + assert cached2 is None + + +@pytest.mark.skip +def test_cache_limit(): + def cache_size(): + return sum(len(v) for v in parser_cache.values()) + + try: + parser_cache.clear() + future_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() + 10e6) + old_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() - 10e4) + parser_cache['some_hash_old'] = { + '/path/%s' % i: old_node_cache_item for i in range(300) + } + parser_cache['some_hash_new'] = { + '/path/%s' % i: future_node_cache_item for i in range(300) + } + assert cache_size() == 600 + parse('somecode', cache=True, path='/path/somepath') + assert cache_size() == 301 + finally: + parser_cache.clear() + + +class _FixedTimeFileIO(file_io.KnownContentFileIO): + def __init__(self, path, content, last_modified): + super().__init__(path, content) + self._last_modified = last_modified + + def get_last_modified(self): + return self._last_modified + + +@pytest.mark.skip +@pytest.mark.parametrize('diff_cache', [False, True]) +@pytest.mark.parametrize('use_file_io', [False, True]) +def test_cache_last_used_update(diff_cache, use_file_io): + p = Path('/path/last-used') + parser_cache.clear() # Clear, because then it's easier to find stuff. + parse('somecode', cache=True, path=p) + node_cache_item = next(iter(parser_cache.values()))[p] + now = time.time() + assert node_cache_item.last_used <= now + + if use_file_io: + f = _FixedTimeFileIO(p, 'code', node_cache_item.last_used - 10) + parse(file_io=f, cache=True, diff_cache=diff_cache) + else: + parse('somecode2', cache=True, path=p, diff_cache=diff_cache) + + node_cache_item = next(iter(parser_cache.values()))[p] + assert now <= node_cache_item.last_used <= time.time() + + +@skip_pypy +def test_inactive_cache(tmpdir, isolated_parso_cache): + parser_cache.clear() + test_subjects = "abcdef" + for path in test_subjects: + parse('somecode', cache=True, path=os.path.join(str(tmpdir), path)) + raw_cache_path = isolated_parso_cache.joinpath(_VERSION_TAG) + assert raw_cache_path.exists() + dir_names = os.listdir(raw_cache_path) + a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL + old_paths = set() + for dir_name in dir_names[:len(test_subjects) // 2]: # make certain number of paths old + os.utime(raw_cache_path.joinpath(dir_name), (a_while_ago, a_while_ago)) + old_paths.add(dir_name) + # nothing should be cleared while the lock is on + assert _get_cache_clear_lock_path().exists() + _remove_cache_and_update_lock() # it shouldn't clear anything + assert len(os.listdir(raw_cache_path)) == len(test_subjects) + assert old_paths.issubset(os.listdir(raw_cache_path)) + + os.utime(_get_cache_clear_lock_path(), (a_while_ago, a_while_ago)) + _remove_cache_and_update_lock() + assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2 + assert not old_paths.intersection(os.listdir(raw_cache_path)) + + +@pytest.mark.skip +@skip_pypy +def test_permission_error(monkeypatch): + def save(*args, **kwargs): + nonlocal was_called + was_called = True + raise PermissionError + + was_called = False + + monkeypatch.setattr(cache, '_save_to_file_system', save) + try: + with pytest.warns(Warning): + parse(path=__file__, cache=True, diff_cache=True) + assert was_called + finally: + parser_cache.clear() diff --git a/contrib/python/parso/py3/tests/test_diff_parser.py b/contrib/python/parso/py3/tests/test_diff_parser.py index 222236e7e8..66356aad3e 100644 --- a/contrib/python/parso/py3/tests/test_diff_parser.py +++ b/contrib/python/parso/py3/tests/test_diff_parser.py @@ -1,1746 +1,1746 @@ -# -*- coding: utf-8 -*- -from textwrap import dedent -import logging - -import pytest - -from parso.utils import split_lines -from parso import cache -from parso import load_grammar -from parso.python.diff import DiffParser, _assert_valid_graph, _assert_nodes_are_equal -from parso import parse - -ANY = object() - - -def test_simple(): - """ - The diff parser reuses modules. So check for that. - """ - grammar = load_grammar() - module_a = grammar.parse('a', diff_cache=True) - assert grammar.parse('b', diff_cache=True) == module_a - - -def _check_error_leaves_nodes(node): - if node.type in ('error_leaf', 'error_node'): - return node - - try: - children = node.children - except AttributeError: - pass - else: - for child in children: - x_node = _check_error_leaves_nodes(child) - if x_node is not None: - return x_node - return None - - -class Differ: - grammar = load_grammar() - - def initialize(self, code): - logging.debug('differ: initialize') - try: - del cache.parser_cache[self.grammar._hashed][None] - except KeyError: - pass - - self.lines = split_lines(code, keepends=True) - self.module = parse(code, diff_cache=True, cache=True) - assert code == self.module.get_code() - _assert_valid_graph(self.module) - return self.module - - def parse(self, code, copies=0, parsers=0, expect_error_leaves=False): - logging.debug('differ: parse copies=%s parsers=%s', copies, parsers) - lines = split_lines(code, keepends=True) - diff_parser = DiffParser( - self.grammar._pgen_grammar, - self.grammar._tokenizer, - self.module, - ) - new_module = diff_parser.update(self.lines, lines) - self.lines = lines - assert code == new_module.get_code() - - _assert_valid_graph(new_module) - - without_diff_parser_module = parse(code) - _assert_nodes_are_equal(new_module, without_diff_parser_module) - - error_node = _check_error_leaves_nodes(new_module) - assert expect_error_leaves == (error_node is not None), error_node - if parsers is not ANY: - assert diff_parser._parser_count == parsers - if copies is not ANY: - assert diff_parser._copy_count == copies - return new_module - - -@pytest.fixture() -def differ(): - return Differ() - - -def test_change_and_undo(differ): - func_before = 'def func():\n pass\n' - # Parse the function and a. - differ.initialize(func_before + 'a') - # Parse just b. - differ.parse(func_before + 'b', copies=1, parsers=2) - # b has changed to a again, so parse that. - differ.parse(func_before + 'a', copies=1, parsers=2) - # Same as before parsers should not be used. Just a simple copy. - differ.parse(func_before + 'a', copies=1) - - # Now that we have a newline at the end, everything is easier in Python - # syntax, we can parse once and then get a copy. - differ.parse(func_before + 'a\n', copies=1, parsers=2) - differ.parse(func_before + 'a\n', copies=1) - - # Getting rid of an old parser: Still no parsers used. - differ.parse('a\n', copies=1) - # Now the file has completely changed and we need to parse. - differ.parse('b\n', parsers=1) - # And again. - differ.parse('a\n', parsers=1) - - -def test_positions(differ): - func_before = 'class A:\n pass\n' - m = differ.initialize(func_before + 'a') - assert m.start_pos == (1, 0) - assert m.end_pos == (3, 1) - - m = differ.parse('a', copies=1) - assert m.start_pos == (1, 0) - assert m.end_pos == (1, 1) - - m = differ.parse('a\n\n', parsers=1) - assert m.end_pos == (3, 0) - m = differ.parse('a\n\n ', copies=1, parsers=2) - assert m.end_pos == (3, 1) - m = differ.parse('a ', parsers=1) - assert m.end_pos == (1, 2) - - -def test_if_simple(differ): - src = dedent('''\ - if 1: - a = 3 - ''') - else_ = "else:\n a = ''\n" - - differ.initialize(src + 'a') - differ.parse(src + else_ + "a", copies=0, parsers=1) - - differ.parse(else_, parsers=2, expect_error_leaves=True) - differ.parse(src + else_, parsers=1) - - -def test_func_with_for_and_comment(differ): - # The first newline is important, leave it. It should not trigger another - # parser split. - src = dedent("""\ - - def func(): - pass - - - for a in [1]: - # COMMENT - a""") - differ.initialize(src) - differ.parse('a\n' + src, copies=1, parsers=3) - - -def test_one_statement_func(differ): - src = dedent("""\ - first - def func(): a - """) - differ.initialize(src + 'second') - differ.parse(src + 'def second():\n a', parsers=1, copies=1) - - -def test_for_on_one_line(differ): - src = dedent("""\ - foo = 1 - for x in foo: pass - - def hi(): - pass - """) - differ.initialize(src) - - src = dedent("""\ - def hi(): - for x in foo: pass - pass - - pass - """) - differ.parse(src, parsers=2) - - src = dedent("""\ - def hi(): - for x in foo: pass - pass - - def nested(): - pass - """) - # The second parser is for parsing the `def nested()` which is an `equal` - # operation in the SequenceMatcher. - differ.parse(src, parsers=1, copies=1) - - -def test_open_parentheses(differ): - func = 'def func():\n a\n' - code = 'isinstance(\n\n' + func - new_code = 'isinstance(\n' + func - differ.initialize(code) - - differ.parse(new_code, parsers=1, expect_error_leaves=True) - - new_code = 'a = 1\n' + new_code - differ.parse(new_code, parsers=2, expect_error_leaves=True) - - func += 'def other_func():\n pass\n' - differ.initialize('isinstance(\n' + func) - # Cannot copy all, because the prefix of the function is once a newline and - # once not. - differ.parse('isinstance()\n' + func, parsers=2, copies=1) - - -def test_open_parentheses_at_end(differ): - code = "a['" - differ.initialize(code) - differ.parse(code, parsers=1, expect_error_leaves=True) - - -def test_backslash(differ): - src = dedent(r""" - a = 1\ - if 1 else 2 - def x(): - pass - """) - differ.initialize(src) - - src = dedent(r""" - def x(): - a = 1\ - if 1 else 2 - def y(): - pass - """) - differ.parse(src, parsers=1) - - src = dedent(r""" - def first(): - if foo \ - and bar \ - or baz: - pass - def second(): - pass - """) - differ.parse(src, parsers=2) - - -def test_full_copy(differ): - code = 'def foo(bar, baz):\n pass\n bar' - differ.initialize(code) - differ.parse(code, copies=1) - - -def test_wrong_whitespace(differ): - code = ''' - hello - ''' - differ.initialize(code) - differ.parse(code + 'bar\n ', parsers=2, expect_error_leaves=True) - - code += """abc(\npass\n """ - differ.parse(code, parsers=2, expect_error_leaves=True) - - -def test_issues_with_error_leaves(differ): - code = dedent(''' - def ints(): - str.. - str - ''') - code2 = dedent(''' - def ints(): - str. - str - ''') - differ.initialize(code) - differ.parse(code2, parsers=1, expect_error_leaves=True) - - -def test_unfinished_nodes(differ): - code = dedent(''' - class a(): - def __init__(self, a): - self.a = a - def p(self): - a(1) - ''') - code2 = dedent(''' - class a(): - def __init__(self, a): - self.a = a - def p(self): - self - a(1) - ''') - differ.initialize(code) - differ.parse(code2, parsers=2, copies=2) - - -def test_nested_if_and_scopes(differ): - code = dedent(''' - class a(): - if 1: - def b(): - 2 - ''') - code2 = code + ' else:\n 3' - differ.initialize(code) - differ.parse(code2, parsers=1, copies=0) - - -def test_word_before_def(differ): - code1 = 'blub def x():\n' - code2 = code1 + ' s' - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=0, expect_error_leaves=True) - - -def test_classes_with_error_leaves(differ): - code1 = dedent(''' - class X(): - def x(self): - blablabla - assert 3 - self. - - class Y(): - pass - ''') - code2 = dedent(''' - class X(): - def x(self): - blablabla - assert 3 - str( - - class Y(): - pass - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - - -def test_totally_wrong_whitespace(differ): - code1 = ''' - class X(): - raise n - - class Y(): - pass - ''' - code2 = ''' - class X(): - raise n - str( - - class Y(): - pass - ''' - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True) - - -def test_node_insertion(differ): - code1 = dedent(''' - class X(): - def y(self): - a = 1 - b = 2 - - c = 3 - d = 4 - ''') - code2 = dedent(''' - class X(): - def y(self): - a = 1 - b = 2 - str - - c = 3 - d = 4 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=2) - - -def test_whitespace_at_end(differ): - code = dedent('str\n\n') - - differ.initialize(code) - differ.parse(code + '\n', parsers=1, copies=1) - - -def test_endless_while_loop(differ): - """ - This was a bug in Jedi #878. - """ - code = '#dead' - differ.initialize(code) - module = differ.parse(code, parsers=1) - assert module.end_pos == (1, 5) - - code = '#dead\n' - differ.initialize(code) - module = differ.parse(code + '\n', parsers=1) - assert module.end_pos == (3, 0) - - -def test_in_class_movements(differ): - code1 = dedent("""\ - class PlaybookExecutor: - p - b - def run(self): - 1 - try: - x - except: - pass - """) - code2 = dedent("""\ - class PlaybookExecutor: - b - def run(self): - 1 - try: - x - except: - pass - """) - - differ.initialize(code1) - differ.parse(code2, parsers=1) - - -def test_in_parentheses_newlines(differ): - code1 = dedent(""" - x = str( - True) - - a = 1 - - def foo(): - pass - - b = 2""") - - code2 = dedent(""" - x = str(True) - - a = 1 - - def foo(): - pass - - b = 2""") - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1) - - -def test_indentation_issue(differ): - code1 = dedent(""" - import module - """) - - code2 = dedent(""" - class L1: - class L2: - class L3: - def f(): pass - def f(): pass - def f(): pass - def f(): pass - """) - - differ.initialize(code1) - differ.parse(code2, parsers=2) - - -def test_endmarker_newline(differ): - code1 = dedent('''\ - docu = None - # some comment - result = codet - incomplete_dctassign = { - "module" - - if "a": - x = 3 # asdf - ''') - - code2 = code1.replace('codet', 'coded') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - - -def test_newlines_at_end(differ): - differ.initialize('a\n\n') - differ.parse('a\n', copies=1) - - -def test_end_newline_with_decorator(differ): - code = dedent('''\ - @staticmethod - def spam(): - import json - json.l''') - - differ.initialize(code) - module = differ.parse(code + '\n', copies=1, parsers=1) - decorated, endmarker = module.children - assert decorated.type == 'decorated' - decorator, func = decorated.children - suite = func.children[-1] - assert suite.type == 'suite' - newline, first_stmt, second_stmt = suite.children - assert first_stmt.get_code() == ' import json\n' - assert second_stmt.get_code() == ' json.l\n' - - -def test_invalid_to_valid_nodes(differ): - code1 = dedent('''\ - def a(): - foo = 3 - def b(): - la = 3 - else: - la - return - foo - base - ''') - code2 = dedent('''\ - def a(): - foo = 3 - def b(): - la = 3 - if foo: - latte = 3 - else: - la - return - foo - base - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=3) - - -def test_if_removal_and_reappearence(differ): - code1 = dedent('''\ - la = 3 - if foo: - latte = 3 - else: - la - pass - ''') - - code2 = dedent('''\ - la = 3 - latte = 3 - else: - la - pass - ''') - - code3 = dedent('''\ - la = 3 - if foo: - latte = 3 - else: - la - ''') - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=2, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=1) - differ.parse(code3, parsers=1, copies=1) - - -def test_add_error_indentation(differ): - code = 'if x:\n 1\n' - differ.initialize(code) - differ.parse(code + ' 2\n', parsers=1, copies=0, expect_error_leaves=True) - - -def test_differing_docstrings(differ): - code1 = dedent('''\ - def foobar(x, y): - 1 - return x - - def bazbiz(): - foobar() - lala - ''') - - code2 = dedent('''\ - def foobar(x, y): - 2 - return x + y - - def bazbiz(): - z = foobar() - lala - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1) - differ.parse(code1, parsers=2, copies=1) - - -def test_one_call_in_function_change(differ): - code1 = dedent('''\ - def f(self): - mro = [self] - for a in something: - yield a - - def g(self): - return C( - a=str, - b=self, - ) - ''') - - code2 = dedent('''\ - def f(self): - mro = [self] - - def g(self): - return C( - a=str, - t - b=self, - ) - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=2, copies=1) - - -def test_function_deletion(differ): - code1 = dedent('''\ - class C(list): - def f(self): - def iterate(): - for x in b: - break - - return list(iterate()) - ''') - - code2 = dedent('''\ - class C(): - def f(self): - for x in b: - break - - return list(iterate()) - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=0, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=0) - - -def test_docstring_removal(differ): - code1 = dedent('''\ - class E(Exception): - """ - 1 - 2 - 3 - """ - - class S(object): - @property - def f(self): - return cmd - def __repr__(self): - return cmd2 - ''') - - code2 = dedent('''\ - class E(Exception): - """ - 1 - 3 - """ - - class S(object): - @property - def f(self): - return cmd - return cmd2 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=2) - differ.parse(code1, parsers=3, copies=1) - - -def test_paren_in_strange_position(differ): - code1 = dedent('''\ - class C: - """ ha """ - def __init__(self, message): - self.message = message - ''') - - code2 = dedent('''\ - class C: - """ ha """ - ) - def __init__(self, message): - self.message = message - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=2, expect_error_leaves=True) - differ.parse(code1, parsers=0, copies=2) - - -def insert_line_into_code(code, index, line): - lines = split_lines(code, keepends=True) - lines.insert(index, line) - return ''.join(lines) - - -def test_paren_before_docstring(differ): - code1 = dedent('''\ - # comment - """ - The - """ - from parso import tree - from parso import python - ''') - - code2 = insert_line_into_code(code1, 1, ' ' * 16 + 'raise InternalParseError(\n') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=2, copies=1) - - -def test_parentheses_before_method(differ): - code1 = dedent('''\ - class A: - def a(self): - pass - - class B: - def b(self): - if 1: - pass - ''') - - code2 = dedent('''\ - class A: - def a(self): - pass - Exception.__init__(self, "x" % - - def b(self): - if 1: - pass - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=2, copies=1) - - -def test_indentation_issues(differ): - code1 = dedent('''\ - class C: - def f(): - 1 - if 2: - return 3 - - def g(): - to_be_removed - pass - ''') - - code2 = dedent('''\ - class C: - def f(): - 1 - ``something``, very ``weird``). - if 2: - return 3 - - def g(): - to_be_removed - pass - ''') - - code3 = dedent('''\ - class C: - def f(): - 1 - if 2: - return 3 - - def g(): - pass - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=2) - differ.parse(code3, parsers=2, copies=1) - differ.parse(code1, parsers=2, copies=1) - - -def test_error_dedent_issues(differ): - code1 = dedent('''\ - while True: - try: - 1 - except KeyError: - if 2: - 3 - except IndexError: - 4 - - 5 - ''') - - code2 = dedent('''\ - while True: - try: - except KeyError: - 1 - except KeyError: - if 2: - 3 - except IndexError: - 4 - - something_inserted - 5 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=0, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=0) - - -def test_random_text_insertion(differ): - code1 = dedent('''\ -class C: - def f(): - return node - - def g(): - try: - 1 - except KeyError: - 2 - ''') - - code2 = dedent('''\ -class C: - def f(): - return node -Some'random text: yeah - for push in plan.dfa_pushes: - - def g(): - try: - 1 - except KeyError: - 2 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=2, copies=1) - - -def test_many_nested_ifs(differ): - code1 = dedent('''\ - class C: - def f(self): - def iterate(): - if 1: - yield t - else: - yield - return - - def g(): - 3 - ''') - - code2 = dedent('''\ - def f(self): - def iterate(): - if 1: - yield t - hahahaha - if 2: - else: - yield - return - - def g(): - 3 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=1) - - -@pytest.mark.parametrize('prefix', ['', 'async ']) -def test_with_and_funcdef_in_call(differ, prefix): - code1 = prefix + dedent('''\ - with x: - la = C( - a=1, - b=2, - c=3, - ) - ''') - - code2 = insert_line_into_code(code1, 3, 'def y(self, args):\n') - - differ.initialize(code1) - differ.parse(code2, parsers=1, expect_error_leaves=True) - differ.parse(code1, parsers=1) - - -def test_wrong_backslash(differ): - code1 = dedent('''\ - def y(): - 1 - for x in y: - continue - ''') - - code2 = insert_line_into_code(code1, 3, '\\.whl$\n') - - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=1) - - -def test_random_unicode_characters(differ): - """ - Those issues were all found with the fuzzer. - """ - differ.initialize('') - differ.parse('\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1, - expect_error_leaves=True) - differ.parse('\r\r', parsers=1) - differ.parse("˟Ę\x05À\r rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True) - differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1) - s = ' if not (self, "_fi\x02\x0e\x08\n\nle"):' - differ.parse(s, parsers=1, expect_error_leaves=True) - differ.parse('') - differ.parse(s + '\n', parsers=1, expect_error_leaves=True) - differ.parse(' result = (\r\f\x17\t\x11res)', parsers=1, expect_error_leaves=True) - differ.parse('') - differ.parse(' a( # xx\ndef', parsers=1, expect_error_leaves=True) - - -def test_dedent_end_positions(differ): - code1 = dedent('''\ - if 1: - if b: - 2 - c = { - 5} - ''') - code2 = dedent('''\ - if 1: - if ⌟ഒᜈྡྷṭb: - 2 - 'l': ''} - c = { - 5} - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, expect_error_leaves=True) - differ.parse(code1, parsers=1) - - -def test_special_no_newline_ending(differ): - code1 = dedent('''\ - 1 - ''') - code2 = dedent('''\ - 1 - is ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=0) - - -def test_random_character_insertion(differ): - code1 = dedent('''\ - def create(self): - 1 - if self.path is not None: - return - # 3 - # 4 - ''') - code2 = dedent('''\ - def create(self): - 1 - if 2: - x return - # 3 - # 4 - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=1) - - -def test_import_opening_bracket(differ): - code1 = dedent('''\ - 1 - 2 - from bubu import (X, - ''') - code2 = dedent('''\ - 11 - 2 - from bubu import (X, - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=2, expect_error_leaves=True) - - -def test_opening_bracket_at_end(differ): - code1 = dedent('''\ - class C: - 1 - [ - ''') - code2 = dedent('''\ - 3 - class C: - 1 - [ - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) - - -def test_all_sorts_of_indentation(differ): - code1 = dedent('''\ - class C: - 1 - def f(): - 'same' - - if foo: - a = b - end - ''') - code2 = dedent('''\ - class C: - 1 - def f(yield await %|( - 'same' - - \x02\x06\x0f\x1c\x11 - if foo: - a = b - - end - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) - - code3 = dedent('''\ - if 1: - a - b - c - d - \x00 - ''') - differ.parse(code3, parsers=1, expect_error_leaves=True) - differ.parse('') - - -def test_dont_copy_dedents_in_beginning(differ): - code1 = dedent('''\ - a - 4 - ''') - code2 = dedent('''\ - 1 - 2 - 3 - 4 - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=1) - - -def test_dont_copy_error_leaves(differ): - code1 = dedent('''\ - def f(n): - x - if 2: - 3 - ''') - code2 = dedent('''\ - def f(n): - def if 1: - indent - x - if 2: - 3 - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, expect_error_leaves=True) - differ.parse(code1, parsers=1) - - -def test_error_dedent_in_between(differ): - code1 = dedent('''\ - class C: - def f(): - a - if something: - x - z - ''') - code2 = dedent('''\ - class C: - def f(): - a - dedent - if other_thing: - b - if something: - x - z - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=2) - - -def test_some_other_indentation_issues(differ): - code1 = dedent('''\ - class C: - x - def f(): - "" - copied - a - ''') - code2 = dedent('''\ - try: - de - a - b - c - d - def f(): - "" - copied - a - ''') - differ.initialize(code1) - differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=1) - - -def test_open_bracket_case1(differ): - code1 = dedent('''\ - class C: - 1 - 2 # ha - ''') - code2 = insert_line_into_code(code1, 2, ' [str\n') - code3 = insert_line_into_code(code2, 4, ' str\n') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code3, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=1, parsers=1) - - -def test_open_bracket_case2(differ): - code1 = dedent('''\ - class C: - def f(self): - ( - b - c - - def g(self): - d - ''') - code2 = dedent('''\ - class C: - def f(self): - ( - b - c - self. - - def g(self): - d - ''') - differ.initialize(code1) - differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True) - differ.parse(code1, copies=0, parsers=1, expect_error_leaves=True) - - -def test_some_weird_removals(differ): - code1 = dedent('''\ - class C: - 1 - ''') - code2 = dedent('''\ - class C: - 1 - @property - A - return - # x - omega - ''') - code3 = dedent('''\ - class C: - 1 - ; - omega - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) - differ.parse(code3, copies=1, parsers=3, expect_error_leaves=True) - differ.parse(code1, copies=1) - - -def test_async_copy(differ): - code1 = dedent('''\ - async def main(): - x = 3 - print( - ''') - code2 = dedent('''\ - async def main(): - x = 3 - print() - ''') - differ.initialize(code1) - differ.parse(code2, copies=1, parsers=1) - differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) - - -def test_parent_on_decorator(differ): - code1 = dedent('''\ - class AClass: - @decorator() - def b_test(self): - print("Hello") - print("world") - - def a_test(self): - pass''') - code2 = dedent('''\ - class AClass: - @decorator() - def b_test(self): - print("Hello") - print("world") - - def a_test(self): - pass''') - differ.initialize(code1) - module_node = differ.parse(code2, parsers=1) - cls = module_node.children[0] - cls_suite = cls.children[-1] - assert len(cls_suite.children) == 3 - - -def test_wrong_indent_in_def(differ): - code1 = dedent('''\ - def x(): - a - b - ''') - - code2 = dedent('''\ - def x(): - // - b - c - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, expect_error_leaves=True) - differ.parse(code1, parsers=1) - - -def test_backslash_issue(differ): - code1 = dedent(''' - pre = ( - '') - after = 'instead' - ''') - code2 = dedent(''' - pre = ( - '') - \\if - ''') # noqa - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=1, copies=1) - - -def test_paren_with_indentation(differ): - code1 = dedent(''' - class C: - def f(self, fullname, path=None): - x - - def load_module(self, fullname): - a - for prefix in self.search_path: - try: - b - except ImportError: - c - else: - raise - def x(): - pass - ''') - code2 = dedent(''' - class C: - def f(self, fullname, path=None): - x - - ( - a - for prefix in self.search_path: - try: - b - except ImportError: - c - else: - raise - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=3, copies=1) - - -def test_error_dedent_in_function(differ): - code1 = dedent('''\ - def x(): - a - b - c - d - ''') - code2 = dedent('''\ - def x(): - a - b - c - d - e - ''') - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - - -def test_with_formfeed(differ): - code1 = dedent('''\ - @bla - async def foo(): - 1 - yield from [] - return - return '' - ''') - code2 = dedent('''\ - @bla - async def foo(): - 1 - \x0cimport - return - return '' - ''') # noqa - differ.initialize(code1) - differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) - - -def test_repeating_invalid_indent(differ): - code1 = dedent('''\ - def foo(): - return - - @bla - a - def foo(): - a - b - c - ''') - code2 = dedent('''\ - def foo(): - return - - @bla - a - b - c - ''') - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - - -def test_another_random_indent(differ): - code1 = dedent('''\ - def foo(): - a - b - c - return - def foo(): - d - ''') - code2 = dedent('''\ - def foo(): - a - c - return - def foo(): - d - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=3) - - -def test_invalid_function(differ): - code1 = dedent('''\ - a - def foo(): - def foo(): - b - ''') - code2 = dedent('''\ - a - def foo(): - def foo(): - b - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - - -def test_async_func2(differ): - code1 = dedent('''\ - async def foo(): - return '' - @bla - async def foo(): - x - ''') - code2 = dedent('''\ - async def foo(): - return '' - - { - @bla - async def foo(): - x - y - ''') - differ.initialize(code1) - differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) - - -def test_weird_ending(differ): - code1 = dedent('''\ - def foo(): - a - return - ''') - code2 = dedent('''\ - def foo(): - a - nonlocal xF""" - y"""''') - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - - -def test_nested_class(differ): - code1 = dedent('''\ -def c(): - a = 3 - class X: - b - ''') - code2 = dedent('''\ -def c(): - a = 3 - class X: - elif - ''') - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - - -def test_class_with_paren_breaker(differ): - code1 = dedent('''\ -class Grammar: - x - def parse(): - y - parser( - ) - z - ''') - code2 = dedent('''\ -class Grammar: - x - def parse(): - y - parser( - finally ; - ) - z - ''') - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) - - -def test_byte_order_mark(differ): - code2 = dedent('''\ - - x - \ufeff - else : - ''') - differ.initialize('\n') - differ.parse(code2, parsers=2, expect_error_leaves=True) - - code3 = dedent('''\ - \ufeff - if: - - x - ''') - differ.initialize('\n') - differ.parse(code3, parsers=2, expect_error_leaves=True) - - -def test_byte_order_mark2(differ): - code = '\ufeff# foo' - differ.initialize(code) - differ.parse(code + 'x', parsers=ANY) - - -def test_byte_order_mark3(differ): - code1 = "\ufeff#\ny\n" - code2 = 'x\n\ufeff#\n\ufeff#\ny\n' - differ.initialize(code1) - differ.parse(code2, expect_error_leaves=True, parsers=ANY, copies=ANY) - differ.parse(code1, parsers=1) - - -def test_backslash_insertion(differ): - code1 = dedent(''' - def f(): - x - def g(): - base = "" \\ - "" - return - ''') - code2 = dedent(''' - def f(): - x - def g(): - base = "" \\ - def h(): - "" - return - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) - differ.parse(code1, parsers=2, copies=1) - - -def test_fstring_with_error_leaf(differ): - code1 = dedent("""\ - def f(): - x - def g(): - y - """) - code2 = dedent("""\ - def f(): - x - F''' - def g(): - y - {a - \x01 - """) - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - - -def test_yet_another_backslash(differ): - code1 = dedent('''\ - def f(): - x - def g(): - y - base = "" \\ - "" % to - return - ''') - code2 = dedent('''\ - def f(): - x - def g(): - y - base = "" \\ - \x0f - return - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) - differ.parse(code1, parsers=ANY, copies=ANY) - - -def test_backslash_before_def(differ): - code1 = dedent('''\ - def f(): - x - - def g(): - y - z - ''') - code2 = dedent('''\ - def f(): - x - >\\ - def g(): - y - x - z - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) - - -def test_backslash_with_imports(differ): - code1 = dedent('''\ - from x import y, \\ - ''') - code2 = dedent('''\ - from x import y, \\ - z - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1) - differ.parse(code1, parsers=1) - - -def test_one_line_function_error_recovery(differ): - code1 = dedent('''\ - class X: - x - def y(): word """ - # a - # b - c(self) - ''') - code2 = dedent('''\ - class X: - x - def y(): word """ - # a - # b - c(\x01+self) - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) - - -def test_one_line_property_error_recovery(differ): - code1 = dedent('''\ - class X: - x - @property - def encoding(self): True - - return 1 - ''') - code2 = dedent('''\ - class X: - x - @property - def encoding(self): True - - return 1 - ''') - - differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) +# -*- coding: utf-8 -*- +from textwrap import dedent +import logging + +import pytest + +from parso.utils import split_lines +from parso import cache +from parso import load_grammar +from parso.python.diff import DiffParser, _assert_valid_graph, _assert_nodes_are_equal +from parso import parse + +ANY = object() + + +def test_simple(): + """ + The diff parser reuses modules. So check for that. + """ + grammar = load_grammar() + module_a = grammar.parse('a', diff_cache=True) + assert grammar.parse('b', diff_cache=True) == module_a + + +def _check_error_leaves_nodes(node): + if node.type in ('error_leaf', 'error_node'): + return node + + try: + children = node.children + except AttributeError: + pass + else: + for child in children: + x_node = _check_error_leaves_nodes(child) + if x_node is not None: + return x_node + return None + + +class Differ: + grammar = load_grammar() + + def initialize(self, code): + logging.debug('differ: initialize') + try: + del cache.parser_cache[self.grammar._hashed][None] + except KeyError: + pass + + self.lines = split_lines(code, keepends=True) + self.module = parse(code, diff_cache=True, cache=True) + assert code == self.module.get_code() + _assert_valid_graph(self.module) + return self.module + + def parse(self, code, copies=0, parsers=0, expect_error_leaves=False): + logging.debug('differ: parse copies=%s parsers=%s', copies, parsers) + lines = split_lines(code, keepends=True) + diff_parser = DiffParser( + self.grammar._pgen_grammar, + self.grammar._tokenizer, + self.module, + ) + new_module = diff_parser.update(self.lines, lines) + self.lines = lines + assert code == new_module.get_code() + + _assert_valid_graph(new_module) + + without_diff_parser_module = parse(code) + _assert_nodes_are_equal(new_module, without_diff_parser_module) + + error_node = _check_error_leaves_nodes(new_module) + assert expect_error_leaves == (error_node is not None), error_node + if parsers is not ANY: + assert diff_parser._parser_count == parsers + if copies is not ANY: + assert diff_parser._copy_count == copies + return new_module + + +@pytest.fixture() +def differ(): + return Differ() + + +def test_change_and_undo(differ): + func_before = 'def func():\n pass\n' + # Parse the function and a. + differ.initialize(func_before + 'a') + # Parse just b. + differ.parse(func_before + 'b', copies=1, parsers=2) + # b has changed to a again, so parse that. + differ.parse(func_before + 'a', copies=1, parsers=2) + # Same as before parsers should not be used. Just a simple copy. + differ.parse(func_before + 'a', copies=1) + + # Now that we have a newline at the end, everything is easier in Python + # syntax, we can parse once and then get a copy. + differ.parse(func_before + 'a\n', copies=1, parsers=2) + differ.parse(func_before + 'a\n', copies=1) + + # Getting rid of an old parser: Still no parsers used. + differ.parse('a\n', copies=1) + # Now the file has completely changed and we need to parse. + differ.parse('b\n', parsers=1) + # And again. + differ.parse('a\n', parsers=1) + + +def test_positions(differ): + func_before = 'class A:\n pass\n' + m = differ.initialize(func_before + 'a') + assert m.start_pos == (1, 0) + assert m.end_pos == (3, 1) + + m = differ.parse('a', copies=1) + assert m.start_pos == (1, 0) + assert m.end_pos == (1, 1) + + m = differ.parse('a\n\n', parsers=1) + assert m.end_pos == (3, 0) + m = differ.parse('a\n\n ', copies=1, parsers=2) + assert m.end_pos == (3, 1) + m = differ.parse('a ', parsers=1) + assert m.end_pos == (1, 2) + + +def test_if_simple(differ): + src = dedent('''\ + if 1: + a = 3 + ''') + else_ = "else:\n a = ''\n" + + differ.initialize(src + 'a') + differ.parse(src + else_ + "a", copies=0, parsers=1) + + differ.parse(else_, parsers=2, expect_error_leaves=True) + differ.parse(src + else_, parsers=1) + + +def test_func_with_for_and_comment(differ): + # The first newline is important, leave it. It should not trigger another + # parser split. + src = dedent("""\ + + def func(): + pass + + + for a in [1]: + # COMMENT + a""") + differ.initialize(src) + differ.parse('a\n' + src, copies=1, parsers=3) + + +def test_one_statement_func(differ): + src = dedent("""\ + first + def func(): a + """) + differ.initialize(src + 'second') + differ.parse(src + 'def second():\n a', parsers=1, copies=1) + + +def test_for_on_one_line(differ): + src = dedent("""\ + foo = 1 + for x in foo: pass + + def hi(): + pass + """) + differ.initialize(src) + + src = dedent("""\ + def hi(): + for x in foo: pass + pass + + pass + """) + differ.parse(src, parsers=2) + + src = dedent("""\ + def hi(): + for x in foo: pass + pass + + def nested(): + pass + """) + # The second parser is for parsing the `def nested()` which is an `equal` + # operation in the SequenceMatcher. + differ.parse(src, parsers=1, copies=1) + + +def test_open_parentheses(differ): + func = 'def func():\n a\n' + code = 'isinstance(\n\n' + func + new_code = 'isinstance(\n' + func + differ.initialize(code) + + differ.parse(new_code, parsers=1, expect_error_leaves=True) + + new_code = 'a = 1\n' + new_code + differ.parse(new_code, parsers=2, expect_error_leaves=True) + + func += 'def other_func():\n pass\n' + differ.initialize('isinstance(\n' + func) + # Cannot copy all, because the prefix of the function is once a newline and + # once not. + differ.parse('isinstance()\n' + func, parsers=2, copies=1) + + +def test_open_parentheses_at_end(differ): + code = "a['" + differ.initialize(code) + differ.parse(code, parsers=1, expect_error_leaves=True) + + +def test_backslash(differ): + src = dedent(r""" + a = 1\ + if 1 else 2 + def x(): + pass + """) + differ.initialize(src) + + src = dedent(r""" + def x(): + a = 1\ + if 1 else 2 + def y(): + pass + """) + differ.parse(src, parsers=1) + + src = dedent(r""" + def first(): + if foo \ + and bar \ + or baz: + pass + def second(): + pass + """) + differ.parse(src, parsers=2) + + +def test_full_copy(differ): + code = 'def foo(bar, baz):\n pass\n bar' + differ.initialize(code) + differ.parse(code, copies=1) + + +def test_wrong_whitespace(differ): + code = ''' + hello + ''' + differ.initialize(code) + differ.parse(code + 'bar\n ', parsers=2, expect_error_leaves=True) + + code += """abc(\npass\n """ + differ.parse(code, parsers=2, expect_error_leaves=True) + + +def test_issues_with_error_leaves(differ): + code = dedent(''' + def ints(): + str.. + str + ''') + code2 = dedent(''' + def ints(): + str. + str + ''') + differ.initialize(code) + differ.parse(code2, parsers=1, expect_error_leaves=True) + + +def test_unfinished_nodes(differ): + code = dedent(''' + class a(): + def __init__(self, a): + self.a = a + def p(self): + a(1) + ''') + code2 = dedent(''' + class a(): + def __init__(self, a): + self.a = a + def p(self): + self + a(1) + ''') + differ.initialize(code) + differ.parse(code2, parsers=2, copies=2) + + +def test_nested_if_and_scopes(differ): + code = dedent(''' + class a(): + if 1: + def b(): + 2 + ''') + code2 = code + ' else:\n 3' + differ.initialize(code) + differ.parse(code2, parsers=1, copies=0) + + +def test_word_before_def(differ): + code1 = 'blub def x():\n' + code2 = code1 + ' s' + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=0, expect_error_leaves=True) + + +def test_classes_with_error_leaves(differ): + code1 = dedent(''' + class X(): + def x(self): + blablabla + assert 3 + self. + + class Y(): + pass + ''') + code2 = dedent(''' + class X(): + def x(self): + blablabla + assert 3 + str( + + class Y(): + pass + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + + +def test_totally_wrong_whitespace(differ): + code1 = ''' + class X(): + raise n + + class Y(): + pass + ''' + code2 = ''' + class X(): + raise n + str( + + class Y(): + pass + ''' + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True) + + +def test_node_insertion(differ): + code1 = dedent(''' + class X(): + def y(self): + a = 1 + b = 2 + + c = 3 + d = 4 + ''') + code2 = dedent(''' + class X(): + def y(self): + a = 1 + b = 2 + str + + c = 3 + d = 4 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=2) + + +def test_whitespace_at_end(differ): + code = dedent('str\n\n') + + differ.initialize(code) + differ.parse(code + '\n', parsers=1, copies=1) + + +def test_endless_while_loop(differ): + """ + This was a bug in Jedi #878. + """ + code = '#dead' + differ.initialize(code) + module = differ.parse(code, parsers=1) + assert module.end_pos == (1, 5) + + code = '#dead\n' + differ.initialize(code) + module = differ.parse(code + '\n', parsers=1) + assert module.end_pos == (3, 0) + + +def test_in_class_movements(differ): + code1 = dedent("""\ + class PlaybookExecutor: + p + b + def run(self): + 1 + try: + x + except: + pass + """) + code2 = dedent("""\ + class PlaybookExecutor: + b + def run(self): + 1 + try: + x + except: + pass + """) + + differ.initialize(code1) + differ.parse(code2, parsers=1) + + +def test_in_parentheses_newlines(differ): + code1 = dedent(""" + x = str( + True) + + a = 1 + + def foo(): + pass + + b = 2""") + + code2 = dedent(""" + x = str(True) + + a = 1 + + def foo(): + pass + + b = 2""") + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1) + + +def test_indentation_issue(differ): + code1 = dedent(""" + import module + """) + + code2 = dedent(""" + class L1: + class L2: + class L3: + def f(): pass + def f(): pass + def f(): pass + def f(): pass + """) + + differ.initialize(code1) + differ.parse(code2, parsers=2) + + +def test_endmarker_newline(differ): + code1 = dedent('''\ + docu = None + # some comment + result = codet + incomplete_dctassign = { + "module" + + if "a": + x = 3 # asdf + ''') + + code2 = code1.replace('codet', 'coded') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + + +def test_newlines_at_end(differ): + differ.initialize('a\n\n') + differ.parse('a\n', copies=1) + + +def test_end_newline_with_decorator(differ): + code = dedent('''\ + @staticmethod + def spam(): + import json + json.l''') + + differ.initialize(code) + module = differ.parse(code + '\n', copies=1, parsers=1) + decorated, endmarker = module.children + assert decorated.type == 'decorated' + decorator, func = decorated.children + suite = func.children[-1] + assert suite.type == 'suite' + newline, first_stmt, second_stmt = suite.children + assert first_stmt.get_code() == ' import json\n' + assert second_stmt.get_code() == ' json.l\n' + + +def test_invalid_to_valid_nodes(differ): + code1 = dedent('''\ + def a(): + foo = 3 + def b(): + la = 3 + else: + la + return + foo + base + ''') + code2 = dedent('''\ + def a(): + foo = 3 + def b(): + la = 3 + if foo: + latte = 3 + else: + la + return + foo + base + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=3) + + +def test_if_removal_and_reappearence(differ): + code1 = dedent('''\ + la = 3 + if foo: + latte = 3 + else: + la + pass + ''') + + code2 = dedent('''\ + la = 3 + latte = 3 + else: + la + pass + ''') + + code3 = dedent('''\ + la = 3 + if foo: + latte = 3 + else: + la + ''') + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=2, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=1) + differ.parse(code3, parsers=1, copies=1) + + +def test_add_error_indentation(differ): + code = 'if x:\n 1\n' + differ.initialize(code) + differ.parse(code + ' 2\n', parsers=1, copies=0, expect_error_leaves=True) + + +def test_differing_docstrings(differ): + code1 = dedent('''\ + def foobar(x, y): + 1 + return x + + def bazbiz(): + foobar() + lala + ''') + + code2 = dedent('''\ + def foobar(x, y): + 2 + return x + y + + def bazbiz(): + z = foobar() + lala + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1) + differ.parse(code1, parsers=2, copies=1) + + +def test_one_call_in_function_change(differ): + code1 = dedent('''\ + def f(self): + mro = [self] + for a in something: + yield a + + def g(self): + return C( + a=str, + b=self, + ) + ''') + + code2 = dedent('''\ + def f(self): + mro = [self] + + def g(self): + return C( + a=str, + t + b=self, + ) + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=2, copies=1) + + +def test_function_deletion(differ): + code1 = dedent('''\ + class C(list): + def f(self): + def iterate(): + for x in b: + break + + return list(iterate()) + ''') + + code2 = dedent('''\ + class C(): + def f(self): + for x in b: + break + + return list(iterate()) + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=0, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=0) + + +def test_docstring_removal(differ): + code1 = dedent('''\ + class E(Exception): + """ + 1 + 2 + 3 + """ + + class S(object): + @property + def f(self): + return cmd + def __repr__(self): + return cmd2 + ''') + + code2 = dedent('''\ + class E(Exception): + """ + 1 + 3 + """ + + class S(object): + @property + def f(self): + return cmd + return cmd2 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=2) + differ.parse(code1, parsers=3, copies=1) + + +def test_paren_in_strange_position(differ): + code1 = dedent('''\ + class C: + """ ha """ + def __init__(self, message): + self.message = message + ''') + + code2 = dedent('''\ + class C: + """ ha """ + ) + def __init__(self, message): + self.message = message + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=2, expect_error_leaves=True) + differ.parse(code1, parsers=0, copies=2) + + +def insert_line_into_code(code, index, line): + lines = split_lines(code, keepends=True) + lines.insert(index, line) + return ''.join(lines) + + +def test_paren_before_docstring(differ): + code1 = dedent('''\ + # comment + """ + The + """ + from parso import tree + from parso import python + ''') + + code2 = insert_line_into_code(code1, 1, ' ' * 16 + 'raise InternalParseError(\n') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=2, copies=1) + + +def test_parentheses_before_method(differ): + code1 = dedent('''\ + class A: + def a(self): + pass + + class B: + def b(self): + if 1: + pass + ''') + + code2 = dedent('''\ + class A: + def a(self): + pass + Exception.__init__(self, "x" % + + def b(self): + if 1: + pass + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=2, copies=1) + + +def test_indentation_issues(differ): + code1 = dedent('''\ + class C: + def f(): + 1 + if 2: + return 3 + + def g(): + to_be_removed + pass + ''') + + code2 = dedent('''\ + class C: + def f(): + 1 + ``something``, very ``weird``). + if 2: + return 3 + + def g(): + to_be_removed + pass + ''') + + code3 = dedent('''\ + class C: + def f(): + 1 + if 2: + return 3 + + def g(): + pass + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=2) + differ.parse(code3, parsers=2, copies=1) + differ.parse(code1, parsers=2, copies=1) + + +def test_error_dedent_issues(differ): + code1 = dedent('''\ + while True: + try: + 1 + except KeyError: + if 2: + 3 + except IndexError: + 4 + + 5 + ''') + + code2 = dedent('''\ + while True: + try: + except KeyError: + 1 + except KeyError: + if 2: + 3 + except IndexError: + 4 + + something_inserted + 5 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=0, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=0) + + +def test_random_text_insertion(differ): + code1 = dedent('''\ +class C: + def f(): + return node + + def g(): + try: + 1 + except KeyError: + 2 + ''') + + code2 = dedent('''\ +class C: + def f(): + return node +Some'random text: yeah + for push in plan.dfa_pushes: + + def g(): + try: + 1 + except KeyError: + 2 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=2, copies=1) + + +def test_many_nested_ifs(differ): + code1 = dedent('''\ + class C: + def f(self): + def iterate(): + if 1: + yield t + else: + yield + return + + def g(): + 3 + ''') + + code2 = dedent('''\ + def f(self): + def iterate(): + if 1: + yield t + hahahaha + if 2: + else: + yield + return + + def g(): + 3 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=1) + + +@pytest.mark.parametrize('prefix', ['', 'async ']) +def test_with_and_funcdef_in_call(differ, prefix): + code1 = prefix + dedent('''\ + with x: + la = C( + a=1, + b=2, + c=3, + ) + ''') + + code2 = insert_line_into_code(code1, 3, 'def y(self, args):\n') + + differ.initialize(code1) + differ.parse(code2, parsers=1, expect_error_leaves=True) + differ.parse(code1, parsers=1) + + +def test_wrong_backslash(differ): + code1 = dedent('''\ + def y(): + 1 + for x in y: + continue + ''') + + code2 = insert_line_into_code(code1, 3, '\\.whl$\n') + + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=1) + + +def test_random_unicode_characters(differ): + """ + Those issues were all found with the fuzzer. + """ + differ.initialize('') + differ.parse('\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1, + expect_error_leaves=True) + differ.parse('\r\r', parsers=1) + differ.parse("˟Ę\x05À\r rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True) + differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1) + s = ' if not (self, "_fi\x02\x0e\x08\n\nle"):' + differ.parse(s, parsers=1, expect_error_leaves=True) + differ.parse('') + differ.parse(s + '\n', parsers=1, expect_error_leaves=True) + differ.parse(' result = (\r\f\x17\t\x11res)', parsers=1, expect_error_leaves=True) + differ.parse('') + differ.parse(' a( # xx\ndef', parsers=1, expect_error_leaves=True) + + +def test_dedent_end_positions(differ): + code1 = dedent('''\ + if 1: + if b: + 2 + c = { + 5} + ''') + code2 = dedent('''\ + if 1: + if ⌟ഒᜈྡྷṭb: + 2 + 'l': ''} + c = { + 5} + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, expect_error_leaves=True) + differ.parse(code1, parsers=1) + + +def test_special_no_newline_ending(differ): + code1 = dedent('''\ + 1 + ''') + code2 = dedent('''\ + 1 + is ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=0) + + +def test_random_character_insertion(differ): + code1 = dedent('''\ + def create(self): + 1 + if self.path is not None: + return + # 3 + # 4 + ''') + code2 = dedent('''\ + def create(self): + 1 + if 2: + x return + # 3 + # 4 + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=1) + + +def test_import_opening_bracket(differ): + code1 = dedent('''\ + 1 + 2 + from bubu import (X, + ''') + code2 = dedent('''\ + 11 + 2 + from bubu import (X, + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=2, expect_error_leaves=True) + + +def test_opening_bracket_at_end(differ): + code1 = dedent('''\ + class C: + 1 + [ + ''') + code2 = dedent('''\ + 3 + class C: + 1 + [ + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) + + +def test_all_sorts_of_indentation(differ): + code1 = dedent('''\ + class C: + 1 + def f(): + 'same' + + if foo: + a = b + end + ''') + code2 = dedent('''\ + class C: + 1 + def f(yield await %|( + 'same' + + \x02\x06\x0f\x1c\x11 + if foo: + a = b + + end + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) + + code3 = dedent('''\ + if 1: + a + b + c + d + \x00 + ''') + differ.parse(code3, parsers=1, expect_error_leaves=True) + differ.parse('') + + +def test_dont_copy_dedents_in_beginning(differ): + code1 = dedent('''\ + a + 4 + ''') + code2 = dedent('''\ + 1 + 2 + 3 + 4 + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=1) + + +def test_dont_copy_error_leaves(differ): + code1 = dedent('''\ + def f(n): + x + if 2: + 3 + ''') + code2 = dedent('''\ + def f(n): + def if 1: + indent + x + if 2: + 3 + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, expect_error_leaves=True) + differ.parse(code1, parsers=1) + + +def test_error_dedent_in_between(differ): + code1 = dedent('''\ + class C: + def f(): + a + if something: + x + z + ''') + code2 = dedent('''\ + class C: + def f(): + a + dedent + if other_thing: + b + if something: + x + z + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=2) + + +def test_some_other_indentation_issues(differ): + code1 = dedent('''\ + class C: + x + def f(): + "" + copied + a + ''') + code2 = dedent('''\ + try: + de + a + b + c + d + def f(): + "" + copied + a + ''') + differ.initialize(code1) + differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=1) + + +def test_open_bracket_case1(differ): + code1 = dedent('''\ + class C: + 1 + 2 # ha + ''') + code2 = insert_line_into_code(code1, 2, ' [str\n') + code3 = insert_line_into_code(code2, 4, ' str\n') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code3, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=1) + + +def test_open_bracket_case2(differ): + code1 = dedent('''\ + class C: + def f(self): + ( + b + c + + def g(self): + d + ''') + code2 = dedent('''\ + class C: + def f(self): + ( + b + c + self. + + def g(self): + d + ''') + differ.initialize(code1) + differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True) + differ.parse(code1, copies=0, parsers=1, expect_error_leaves=True) + + +def test_some_weird_removals(differ): + code1 = dedent('''\ + class C: + 1 + ''') + code2 = dedent('''\ + class C: + 1 + @property + A + return + # x + omega + ''') + code3 = dedent('''\ + class C: + 1 + ; + omega + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True) + differ.parse(code3, copies=1, parsers=3, expect_error_leaves=True) + differ.parse(code1, copies=1) + + +def test_async_copy(differ): + code1 = dedent('''\ + async def main(): + x = 3 + print( + ''') + code2 = dedent('''\ + async def main(): + x = 3 + print() + ''') + differ.initialize(code1) + differ.parse(code2, copies=1, parsers=1) + differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) + + +def test_parent_on_decorator(differ): + code1 = dedent('''\ + class AClass: + @decorator() + def b_test(self): + print("Hello") + print("world") + + def a_test(self): + pass''') + code2 = dedent('''\ + class AClass: + @decorator() + def b_test(self): + print("Hello") + print("world") + + def a_test(self): + pass''') + differ.initialize(code1) + module_node = differ.parse(code2, parsers=1) + cls = module_node.children[0] + cls_suite = cls.children[-1] + assert len(cls_suite.children) == 3 + + +def test_wrong_indent_in_def(differ): + code1 = dedent('''\ + def x(): + a + b + ''') + + code2 = dedent('''\ + def x(): + // + b + c + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, expect_error_leaves=True) + differ.parse(code1, parsers=1) + + +def test_backslash_issue(differ): + code1 = dedent(''' + pre = ( + '') + after = 'instead' + ''') + code2 = dedent(''' + pre = ( + '') + \\if + ''') # noqa + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=1, copies=1) + + +def test_paren_with_indentation(differ): + code1 = dedent(''' + class C: + def f(self, fullname, path=None): + x + + def load_module(self, fullname): + a + for prefix in self.search_path: + try: + b + except ImportError: + c + else: + raise + def x(): + pass + ''') + code2 = dedent(''' + class C: + def f(self, fullname, path=None): + x + + ( + a + for prefix in self.search_path: + try: + b + except ImportError: + c + else: + raise + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=3, copies=1) + + +def test_error_dedent_in_function(differ): + code1 = dedent('''\ + def x(): + a + b + c + d + ''') + code2 = dedent('''\ + def x(): + a + b + c + d + e + ''') + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + + +def test_with_formfeed(differ): + code1 = dedent('''\ + @bla + async def foo(): + 1 + yield from [] + return + return '' + ''') + code2 = dedent('''\ + @bla + async def foo(): + 1 + \x0cimport + return + return '' + ''') # noqa + differ.initialize(code1) + differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) + + +def test_repeating_invalid_indent(differ): + code1 = dedent('''\ + def foo(): + return + + @bla + a + def foo(): + a + b + c + ''') + code2 = dedent('''\ + def foo(): + return + + @bla + a + b + c + ''') + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + + +def test_another_random_indent(differ): + code1 = dedent('''\ + def foo(): + a + b + c + return + def foo(): + d + ''') + code2 = dedent('''\ + def foo(): + a + c + return + def foo(): + d + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=3) + + +def test_invalid_function(differ): + code1 = dedent('''\ + a + def foo(): + def foo(): + b + ''') + code2 = dedent('''\ + a + def foo(): + def foo(): + b + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + + +def test_async_func2(differ): + code1 = dedent('''\ + async def foo(): + return '' + @bla + async def foo(): + x + ''') + code2 = dedent('''\ + async def foo(): + return '' + + { + @bla + async def foo(): + x + y + ''') + differ.initialize(code1) + differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) + + +def test_weird_ending(differ): + code1 = dedent('''\ + def foo(): + a + return + ''') + code2 = dedent('''\ + def foo(): + a + nonlocal xF""" + y"""''') + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + + +def test_nested_class(differ): + code1 = dedent('''\ +def c(): + a = 3 + class X: + b + ''') + code2 = dedent('''\ +def c(): + a = 3 + class X: + elif + ''') + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + + +def test_class_with_paren_breaker(differ): + code1 = dedent('''\ +class Grammar: + x + def parse(): + y + parser( + ) + z + ''') + code2 = dedent('''\ +class Grammar: + x + def parse(): + y + parser( + finally ; + ) + z + ''') + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) + + +def test_byte_order_mark(differ): + code2 = dedent('''\ + + x + \ufeff + else : + ''') + differ.initialize('\n') + differ.parse(code2, parsers=2, expect_error_leaves=True) + + code3 = dedent('''\ + \ufeff + if: + + x + ''') + differ.initialize('\n') + differ.parse(code3, parsers=2, expect_error_leaves=True) + + +def test_byte_order_mark2(differ): + code = '\ufeff# foo' + differ.initialize(code) + differ.parse(code + 'x', parsers=ANY) + + +def test_byte_order_mark3(differ): + code1 = "\ufeff#\ny\n" + code2 = 'x\n\ufeff#\n\ufeff#\ny\n' + differ.initialize(code1) + differ.parse(code2, expect_error_leaves=True, parsers=ANY, copies=ANY) + differ.parse(code1, parsers=1) + + +def test_backslash_insertion(differ): + code1 = dedent(''' + def f(): + x + def g(): + base = "" \\ + "" + return + ''') + code2 = dedent(''' + def f(): + x + def g(): + base = "" \\ + def h(): + "" + return + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + differ.parse(code1, parsers=2, copies=1) + + +def test_fstring_with_error_leaf(differ): + code1 = dedent("""\ + def f(): + x + def g(): + y + """) + code2 = dedent("""\ + def f(): + x + F''' + def g(): + y + {a + \x01 + """) + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + + +def test_yet_another_backslash(differ): + code1 = dedent('''\ + def f(): + x + def g(): + y + base = "" \\ + "" % to + return + ''') + code2 = dedent('''\ + def f(): + x + def g(): + y + base = "" \\ + \x0f + return + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True) + differ.parse(code1, parsers=ANY, copies=ANY) + + +def test_backslash_before_def(differ): + code1 = dedent('''\ + def f(): + x + + def g(): + y + z + ''') + code2 = dedent('''\ + def f(): + x + >\\ + def g(): + y + x + z + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True) + + +def test_backslash_with_imports(differ): + code1 = dedent('''\ + from x import y, \\ + ''') + code2 = dedent('''\ + from x import y, \\ + z + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1) + differ.parse(code1, parsers=1) + + +def test_one_line_function_error_recovery(differ): + code1 = dedent('''\ + class X: + x + def y(): word """ + # a + # b + c(self) + ''') + code2 = dedent('''\ + class X: + x + def y(): word """ + # a + # b + c(\x01+self) + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + + +def test_one_line_property_error_recovery(differ): + code1 = dedent('''\ + class X: + x + @property + def encoding(self): True - + return 1 + ''') + code2 = dedent('''\ + class X: + x + @property + def encoding(self): True - + return 1 + ''') + + differ.initialize(code1) + differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) diff --git a/contrib/python/parso/py3/tests/test_dump_tree.py b/contrib/python/parso/py3/tests/test_dump_tree.py index d2d7259f73..df98c9e714 100644 --- a/contrib/python/parso/py3/tests/test_dump_tree.py +++ b/contrib/python/parso/py3/tests/test_dump_tree.py @@ -1,182 +1,182 @@ -from textwrap import dedent - -import pytest - -from parso import parse -# Using star import for easier eval testing below. -from parso.python.tree import * # noqa: F403 -from parso.tree import * # noqa: F403 -from parso.tree import ErrorLeaf, TypedLeaf - - -@pytest.mark.parametrize( - 'indent,expected_dump', [ - (None, "Module([" - "Lambda([" - "Keyword('lambda', (1, 0)), " - "Param([" - "Name('x', (1, 7), prefix=' '), " - "Operator(',', (1, 8)), " - "]), " - "Param([" - "Name('y', (1, 10), prefix=' '), " - "]), " - "Operator(':', (1, 11)), " - "PythonNode('arith_expr', [" - "Name('x', (1, 13), prefix=' '), " - "Operator('+', (1, 15), prefix=' '), " - "Name('y', (1, 17), prefix=' '), " - "]), " - "]), " - "EndMarker('', (1, 18)), " - "])"), - (0, dedent('''\ - Module([ - Lambda([ - Keyword('lambda', (1, 0)), - Param([ - Name('x', (1, 7), prefix=' '), - Operator(',', (1, 8)), - ]), - Param([ - Name('y', (1, 10), prefix=' '), - ]), - Operator(':', (1, 11)), - PythonNode('arith_expr', [ - Name('x', (1, 13), prefix=' '), - Operator('+', (1, 15), prefix=' '), - Name('y', (1, 17), prefix=' '), - ]), - ]), - EndMarker('', (1, 18)), - ])''')), - (4, dedent('''\ - Module([ - Lambda([ - Keyword('lambda', (1, 0)), - Param([ - Name('x', (1, 7), prefix=' '), - Operator(',', (1, 8)), - ]), - Param([ - Name('y', (1, 10), prefix=' '), - ]), - Operator(':', (1, 11)), - PythonNode('arith_expr', [ - Name('x', (1, 13), prefix=' '), - Operator('+', (1, 15), prefix=' '), - Name('y', (1, 17), prefix=' '), - ]), - ]), - EndMarker('', (1, 18)), - ])''')), - ('\t', dedent('''\ - Module([ - \tLambda([ - \t\tKeyword('lambda', (1, 0)), - \t\tParam([ - \t\t\tName('x', (1, 7), prefix=' '), - \t\t\tOperator(',', (1, 8)), - \t\t]), - \t\tParam([ - \t\t\tName('y', (1, 10), prefix=' '), - \t\t]), - \t\tOperator(':', (1, 11)), - \t\tPythonNode('arith_expr', [ - \t\t\tName('x', (1, 13), prefix=' '), - \t\t\tOperator('+', (1, 15), prefix=' '), - \t\t\tName('y', (1, 17), prefix=' '), - \t\t]), - \t]), - \tEndMarker('', (1, 18)), - ])''')), - ] -) -def test_dump_parser_tree(indent, expected_dump): - code = "lambda x, y: x + y" - module = parse(code) - assert module.dump(indent=indent) == expected_dump - - # Check that dumped tree can be eval'd to recover the parser tree and original code. - recovered_code = eval(expected_dump).get_code() - assert recovered_code == code - - -@pytest.mark.parametrize( - 'node,expected_dump,expected_code', [ - ( # Dump intermediate node (not top level module) - parse("def foo(x, y): return x + y").children[0], dedent('''\ - Function([ - Keyword('def', (1, 0)), - Name('foo', (1, 4), prefix=' '), - PythonNode('parameters', [ - Operator('(', (1, 7)), - Param([ - Name('x', (1, 8)), - Operator(',', (1, 9)), - ]), - Param([ - Name('y', (1, 11), prefix=' '), - ]), - Operator(')', (1, 12)), - ]), - Operator(':', (1, 13)), - ReturnStmt([ - Keyword('return', (1, 15), prefix=' '), - PythonNode('arith_expr', [ - Name('x', (1, 22), prefix=' '), - Operator('+', (1, 24), prefix=' '), - Name('y', (1, 26), prefix=' '), - ]), - ]), - ])'''), - "def foo(x, y): return x + y", - ), - ( # Dump leaf - parse("def foo(x, y): return x + y").children[0].children[0], - "Keyword('def', (1, 0))", - 'def', - ), - ( # Dump ErrorLeaf - ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' '), - "ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' ')", - ' error_code', - ), - ( # Dump TypedLeaf - TypedLeaf('type', 'value', (1, 1)), - "TypedLeaf('type', 'value', (1, 1))", - 'value', - ), - ] -) -def test_dump_parser_tree_not_top_level_module(node, expected_dump, expected_code): - dump_result = node.dump() - assert dump_result == expected_dump - - # Check that dumped tree can be eval'd to recover the parser tree and original code. - recovered_code = eval(dump_result).get_code() - assert recovered_code == expected_code - - -def test_dump_parser_tree_invalid_args(): - module = parse("lambda x, y: x + y") - - with pytest.raises(TypeError): - module.dump(indent=1.1) - - -def test_eval_dump_recovers_parent(): - module = parse("lambda x, y: x + y") - module2 = eval(module.dump()) - assert module2.parent is None - lambda_node = module2.children[0] - assert lambda_node.parent is module2 - assert module2.children[1].parent is module2 - assert lambda_node.children[0].parent is lambda_node - param_node = lambda_node.children[1] - assert param_node.parent is lambda_node - assert param_node.children[0].parent is param_node - assert param_node.children[1].parent is param_node - arith_expr_node = lambda_node.children[-1] - assert arith_expr_node.parent is lambda_node - assert arith_expr_node.children[0].parent is arith_expr_node +from textwrap import dedent + +import pytest + +from parso import parse +# Using star import for easier eval testing below. +from parso.python.tree import * # noqa: F403 +from parso.tree import * # noqa: F403 +from parso.tree import ErrorLeaf, TypedLeaf + + +@pytest.mark.parametrize( + 'indent,expected_dump', [ + (None, "Module([" + "Lambda([" + "Keyword('lambda', (1, 0)), " + "Param([" + "Name('x', (1, 7), prefix=' '), " + "Operator(',', (1, 8)), " + "]), " + "Param([" + "Name('y', (1, 10), prefix=' '), " + "]), " + "Operator(':', (1, 11)), " + "PythonNode('arith_expr', [" + "Name('x', (1, 13), prefix=' '), " + "Operator('+', (1, 15), prefix=' '), " + "Name('y', (1, 17), prefix=' '), " + "]), " + "]), " + "EndMarker('', (1, 18)), " + "])"), + (0, dedent('''\ + Module([ + Lambda([ + Keyword('lambda', (1, 0)), + Param([ + Name('x', (1, 7), prefix=' '), + Operator(',', (1, 8)), + ]), + Param([ + Name('y', (1, 10), prefix=' '), + ]), + Operator(':', (1, 11)), + PythonNode('arith_expr', [ + Name('x', (1, 13), prefix=' '), + Operator('+', (1, 15), prefix=' '), + Name('y', (1, 17), prefix=' '), + ]), + ]), + EndMarker('', (1, 18)), + ])''')), + (4, dedent('''\ + Module([ + Lambda([ + Keyword('lambda', (1, 0)), + Param([ + Name('x', (1, 7), prefix=' '), + Operator(',', (1, 8)), + ]), + Param([ + Name('y', (1, 10), prefix=' '), + ]), + Operator(':', (1, 11)), + PythonNode('arith_expr', [ + Name('x', (1, 13), prefix=' '), + Operator('+', (1, 15), prefix=' '), + Name('y', (1, 17), prefix=' '), + ]), + ]), + EndMarker('', (1, 18)), + ])''')), + ('\t', dedent('''\ + Module([ + \tLambda([ + \t\tKeyword('lambda', (1, 0)), + \t\tParam([ + \t\t\tName('x', (1, 7), prefix=' '), + \t\t\tOperator(',', (1, 8)), + \t\t]), + \t\tParam([ + \t\t\tName('y', (1, 10), prefix=' '), + \t\t]), + \t\tOperator(':', (1, 11)), + \t\tPythonNode('arith_expr', [ + \t\t\tName('x', (1, 13), prefix=' '), + \t\t\tOperator('+', (1, 15), prefix=' '), + \t\t\tName('y', (1, 17), prefix=' '), + \t\t]), + \t]), + \tEndMarker('', (1, 18)), + ])''')), + ] +) +def test_dump_parser_tree(indent, expected_dump): + code = "lambda x, y: x + y" + module = parse(code) + assert module.dump(indent=indent) == expected_dump + + # Check that dumped tree can be eval'd to recover the parser tree and original code. + recovered_code = eval(expected_dump).get_code() + assert recovered_code == code + + +@pytest.mark.parametrize( + 'node,expected_dump,expected_code', [ + ( # Dump intermediate node (not top level module) + parse("def foo(x, y): return x + y").children[0], dedent('''\ + Function([ + Keyword('def', (1, 0)), + Name('foo', (1, 4), prefix=' '), + PythonNode('parameters', [ + Operator('(', (1, 7)), + Param([ + Name('x', (1, 8)), + Operator(',', (1, 9)), + ]), + Param([ + Name('y', (1, 11), prefix=' '), + ]), + Operator(')', (1, 12)), + ]), + Operator(':', (1, 13)), + ReturnStmt([ + Keyword('return', (1, 15), prefix=' '), + PythonNode('arith_expr', [ + Name('x', (1, 22), prefix=' '), + Operator('+', (1, 24), prefix=' '), + Name('y', (1, 26), prefix=' '), + ]), + ]), + ])'''), + "def foo(x, y): return x + y", + ), + ( # Dump leaf + parse("def foo(x, y): return x + y").children[0].children[0], + "Keyword('def', (1, 0))", + 'def', + ), + ( # Dump ErrorLeaf + ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' '), + "ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' ')", + ' error_code', + ), + ( # Dump TypedLeaf + TypedLeaf('type', 'value', (1, 1)), + "TypedLeaf('type', 'value', (1, 1))", + 'value', + ), + ] +) +def test_dump_parser_tree_not_top_level_module(node, expected_dump, expected_code): + dump_result = node.dump() + assert dump_result == expected_dump + + # Check that dumped tree can be eval'd to recover the parser tree and original code. + recovered_code = eval(dump_result).get_code() + assert recovered_code == expected_code + + +def test_dump_parser_tree_invalid_args(): + module = parse("lambda x, y: x + y") + + with pytest.raises(TypeError): + module.dump(indent=1.1) + + +def test_eval_dump_recovers_parent(): + module = parse("lambda x, y: x + y") + module2 = eval(module.dump()) + assert module2.parent is None + lambda_node = module2.children[0] + assert lambda_node.parent is module2 + assert module2.children[1].parent is module2 + assert lambda_node.children[0].parent is lambda_node + param_node = lambda_node.children[1] + assert param_node.parent is lambda_node + assert param_node.children[0].parent is param_node + assert param_node.children[1].parent is param_node + arith_expr_node = lambda_node.children[-1] + assert arith_expr_node.parent is lambda_node + assert arith_expr_node.children[0].parent is arith_expr_node diff --git a/contrib/python/parso/py3/tests/test_error_recovery.py b/contrib/python/parso/py3/tests/test_error_recovery.py index 87efd4784a..98abefa0a9 100644 --- a/contrib/python/parso/py3/tests/test_error_recovery.py +++ b/contrib/python/parso/py3/tests/test_error_recovery.py @@ -1,149 +1,149 @@ -from textwrap import dedent - -from parso import parse, load_grammar - - -def test_with_stmt(): - module = parse('with x: f.\na') - assert module.children[0].type == 'with_stmt' - w, with_item, colon, f = module.children[0].children - assert f.type == 'error_node' - assert f.get_code(include_prefix=False) == 'f.' - - assert module.children[2].type == 'name' - - -def test_one_line_function(each_version): - module = parse('def x(): f.', version=each_version) - assert module.children[0].type == 'funcdef' - def_, name, parameters, colon, f = module.children[0].children - assert f.type == 'error_node' - - module = parse('def x(a:', version=each_version) - func = module.children[0] - assert func.type == 'error_node' - if each_version.startswith('2'): - assert func.children[-1].value == 'a' - else: - assert func.children[-1] == ':' - - -def test_if_else(): - module = parse('if x:\n f.\nelse:\n g(') - if_stmt = module.children[0] - if_, test, colon, suite1, else_, colon, suite2 = if_stmt.children - f = suite1.children[1] - assert f.type == 'error_node' - assert f.children[0].value == 'f' - assert f.children[1].value == '.' - g = suite2.children[1] - assert g.children[0].value == 'g' - assert g.children[1].value == '(' - - -def test_if_stmt(): - module = parse('if x: f.\nelse: g(') - if_stmt = module.children[0] - assert if_stmt.type == 'if_stmt' - if_, test, colon, f = if_stmt.children - assert f.type == 'error_node' - assert f.children[0].value == 'f' - assert f.children[1].value == '.' - - assert module.children[1].type == 'newline' - assert module.children[1].value == '\n' - assert module.children[2].type == 'error_leaf' - assert module.children[2].value == 'else' - assert module.children[3].type == 'error_leaf' - assert module.children[3].value == ':' - - in_else_stmt = module.children[4] - assert in_else_stmt.type == 'error_node' - assert in_else_stmt.children[0].value == 'g' - assert in_else_stmt.children[1].value == '(' - - -def test_invalid_token(): - module = parse('a + ? + b') - error_node, q, plus_b, endmarker = module.children - assert error_node.get_code() == 'a +' - assert q.value == '?' - assert q.type == 'error_leaf' - assert plus_b.type == 'factor' - assert plus_b.get_code() == ' + b' - - -def test_invalid_token_in_fstr(): - module = load_grammar(version='3.9').parse('f"{a + ? + b}"') - error_node, q, plus_b, error1, error2, endmarker = module.children - assert error_node.get_code() == 'f"{a +' - assert q.value == '?' - assert q.type == 'error_leaf' - assert plus_b.type == 'error_node' - assert plus_b.get_code() == ' + b' - assert error1.value == '}' - assert error1.type == 'error_leaf' - assert error2.value == '"' - assert error2.type == 'error_leaf' - - -def test_dedent_issues1(): - code = dedent('''\ - class C: - @property - f - g - end - ''') - module = load_grammar(version='3.8').parse(code) - klass, endmarker = module.children - suite = klass.children[-1] - assert suite.children[2].type == 'error_leaf' - assert suite.children[3].get_code(include_prefix=False) == 'f\n' - assert suite.children[5].get_code(include_prefix=False) == 'g\n' - assert suite.type == 'suite' - - -def test_dedent_issues2(): - code = dedent('''\ - class C: - @property - if 1: - g - else: - h - end - ''') - module = load_grammar(version='3.8').parse(code) - klass, endmarker = module.children - suite = klass.children[-1] - assert suite.children[2].type == 'error_leaf' - if_ = suite.children[3] - assert if_.children[0] == 'if' - assert if_.children[3].type == 'suite' - assert if_.children[3].get_code() == '\n g\n' - assert if_.children[4] == 'else' - assert if_.children[6].type == 'suite' - assert if_.children[6].get_code() == '\n h\n' - - assert suite.children[4].get_code(include_prefix=False) == 'end\n' - assert suite.type == 'suite' - - -def test_dedent_issues3(): - code = dedent('''\ - class C: - f - g - ''') - module = load_grammar(version='3.8').parse(code) - klass, endmarker = module.children - suite = klass.children[-1] - assert len(suite.children) == 4 - assert suite.children[1].get_code() == ' f\n' - assert suite.children[1].type == 'simple_stmt' - assert suite.children[2].get_code() == '' - assert suite.children[2].type == 'error_leaf' - assert suite.children[2].token_type == 'ERROR_DEDENT' - assert suite.children[3].get_code() == ' g\n' - assert suite.children[3].type == 'simple_stmt' +from textwrap import dedent + +from parso import parse, load_grammar + + +def test_with_stmt(): + module = parse('with x: f.\na') + assert module.children[0].type == 'with_stmt' + w, with_item, colon, f = module.children[0].children + assert f.type == 'error_node' + assert f.get_code(include_prefix=False) == 'f.' + + assert module.children[2].type == 'name' + + +def test_one_line_function(each_version): + module = parse('def x(): f.', version=each_version) + assert module.children[0].type == 'funcdef' + def_, name, parameters, colon, f = module.children[0].children + assert f.type == 'error_node' + + module = parse('def x(a:', version=each_version) + func = module.children[0] + assert func.type == 'error_node' + if each_version.startswith('2'): + assert func.children[-1].value == 'a' + else: + assert func.children[-1] == ':' + + +def test_if_else(): + module = parse('if x:\n f.\nelse:\n g(') + if_stmt = module.children[0] + if_, test, colon, suite1, else_, colon, suite2 = if_stmt.children + f = suite1.children[1] + assert f.type == 'error_node' + assert f.children[0].value == 'f' + assert f.children[1].value == '.' + g = suite2.children[1] + assert g.children[0].value == 'g' + assert g.children[1].value == '(' + + +def test_if_stmt(): + module = parse('if x: f.\nelse: g(') + if_stmt = module.children[0] + assert if_stmt.type == 'if_stmt' + if_, test, colon, f = if_stmt.children + assert f.type == 'error_node' + assert f.children[0].value == 'f' + assert f.children[1].value == '.' + + assert module.children[1].type == 'newline' + assert module.children[1].value == '\n' + assert module.children[2].type == 'error_leaf' + assert module.children[2].value == 'else' + assert module.children[3].type == 'error_leaf' + assert module.children[3].value == ':' + + in_else_stmt = module.children[4] + assert in_else_stmt.type == 'error_node' + assert in_else_stmt.children[0].value == 'g' + assert in_else_stmt.children[1].value == '(' + + +def test_invalid_token(): + module = parse('a + ? + b') + error_node, q, plus_b, endmarker = module.children + assert error_node.get_code() == 'a +' + assert q.value == '?' + assert q.type == 'error_leaf' + assert plus_b.type == 'factor' + assert plus_b.get_code() == ' + b' + + +def test_invalid_token_in_fstr(): + module = load_grammar(version='3.9').parse('f"{a + ? + b}"') + error_node, q, plus_b, error1, error2, endmarker = module.children + assert error_node.get_code() == 'f"{a +' + assert q.value == '?' + assert q.type == 'error_leaf' + assert plus_b.type == 'error_node' + assert plus_b.get_code() == ' + b' + assert error1.value == '}' + assert error1.type == 'error_leaf' + assert error2.value == '"' + assert error2.type == 'error_leaf' + + +def test_dedent_issues1(): + code = dedent('''\ + class C: + @property + f + g + end + ''') + module = load_grammar(version='3.8').parse(code) + klass, endmarker = module.children + suite = klass.children[-1] + assert suite.children[2].type == 'error_leaf' + assert suite.children[3].get_code(include_prefix=False) == 'f\n' + assert suite.children[5].get_code(include_prefix=False) == 'g\n' + assert suite.type == 'suite' + + +def test_dedent_issues2(): + code = dedent('''\ + class C: + @property + if 1: + g + else: + h + end + ''') + module = load_grammar(version='3.8').parse(code) + klass, endmarker = module.children + suite = klass.children[-1] + assert suite.children[2].type == 'error_leaf' + if_ = suite.children[3] + assert if_.children[0] == 'if' + assert if_.children[3].type == 'suite' + assert if_.children[3].get_code() == '\n g\n' + assert if_.children[4] == 'else' + assert if_.children[6].type == 'suite' + assert if_.children[6].get_code() == '\n h\n' + + assert suite.children[4].get_code(include_prefix=False) == 'end\n' + assert suite.type == 'suite' + + +def test_dedent_issues3(): + code = dedent('''\ + class C: + f + g + ''') + module = load_grammar(version='3.8').parse(code) + klass, endmarker = module.children + suite = klass.children[-1] + assert len(suite.children) == 4 + assert suite.children[1].get_code() == ' f\n' + assert suite.children[1].type == 'simple_stmt' + assert suite.children[2].get_code() == '' + assert suite.children[2].type == 'error_leaf' + assert suite.children[2].token_type == 'ERROR_DEDENT' + assert suite.children[3].get_code() == ' g\n' + assert suite.children[3].type == 'simple_stmt' diff --git a/contrib/python/parso/py3/tests/test_file_python_errors.py b/contrib/python/parso/py3/tests/test_file_python_errors.py index 7083dfeb46..6c9f4307b7 100644 --- a/contrib/python/parso/py3/tests/test_file_python_errors.py +++ b/contrib/python/parso/py3/tests/test_file_python_errors.py @@ -1,23 +1,23 @@ -import os - -import parso - - -def get_python_files(path): - for dir_path, dir_names, file_names in os.walk(path): - for file_name in file_names: - if file_name.endswith('.py'): - yield os.path.join(dir_path, file_name) - - -def test_on_itself(each_version): - """ - There are obviously no syntax erros in the Python code of parso. However - parso should output the same for all versions. - """ - grammar = parso.load_grammar(version=each_version) - path = os.path.dirname(os.path.dirname(__file__)) + '/parso' - for file in get_python_files(path): - tree = grammar.parse(path=file) - errors = list(grammar.iter_errors(tree)) - assert not errors +import os + +import parso + + +def get_python_files(path): + for dir_path, dir_names, file_names in os.walk(path): + for file_name in file_names: + if file_name.endswith('.py'): + yield os.path.join(dir_path, file_name) + + +def test_on_itself(each_version): + """ + There are obviously no syntax erros in the Python code of parso. However + parso should output the same for all versions. + """ + grammar = parso.load_grammar(version=each_version) + path = os.path.dirname(os.path.dirname(__file__)) + '/parso' + for file in get_python_files(path): + tree = grammar.parse(path=file) + errors = list(grammar.iter_errors(tree)) + assert not errors diff --git a/contrib/python/parso/py3/tests/test_fstring.py b/contrib/python/parso/py3/tests/test_fstring.py index c81d027a16..8b6dee62eb 100644 --- a/contrib/python/parso/py3/tests/test_fstring.py +++ b/contrib/python/parso/py3/tests/test_fstring.py @@ -1,164 +1,164 @@ -import pytest -from textwrap import dedent - -from parso import load_grammar, ParserSyntaxError -from parso.python.tokenize import tokenize - - -@pytest.fixture -def grammar(): - return load_grammar(version='3.8') - - -@pytest.mark.parametrize( - 'code', [ - # simple cases - 'f"{1}"', - 'f"""{1}"""', - 'f"{foo} {bar}"', - - # empty string - 'f""', - 'f""""""', - - # empty format specifier is okay - 'f"{1:}"', - - # use of conversion options - 'f"{1!a}"', - 'f"{1!a:1}"', - - # format specifiers - 'f"{1:1}"', - 'f"{1:1.{32}}"', - 'f"{1::>4}"', - 'f"{x:{y}}"', - 'f"{x:{y:}}"', - 'f"{x:{y:1}}"', - - # Escapes - 'f"{{}}"', - 'f"{{{1}}}"', - 'f"{{{1}"', - 'f"1{{2{{3"', - 'f"}}"', - - # New Python 3.8 syntax f'{a=}' - 'f"{a=}"', - 'f"{a()=}"', - - # multiline f-string - 'f"""abc\ndef"""', - 'f"""abc{\n123}def"""', - - # a line continuation inside of an fstring_string - 'f"abc\\\ndef"', - 'f"\\\n{123}\\\n"', - - # a line continuation inside of an fstring_expr - 'f"{\\\n123}"', - - # a line continuation inside of an format spec - 'f"{123:.2\\\nf}"', - - # some unparenthesized syntactic structures - 'f"{*x,}"', - 'f"{*x, *y}"', - 'f"{x, *y}"', - 'f"{*x, y}"', - 'f"{x for x in [1]}"', - - # named unicode characters - 'f"\\N{BULLET}"', - 'f"\\N{FLEUR-DE-LIS}"', - 'f"\\N{NO ENTRY}"', - 'f"Combo {expr} and \\N{NO ENTRY}"', - 'f"\\N{NO ENTRY} and {expr}"', - 'f"\\N{no entry}"', - 'f"\\N{SOYOMBO LETTER -A}"', - 'f"\\N{DOMINO TILE HORIZONTAL-00-00}"', - 'f"""\\N{NO ENTRY}"""', - ] -) -def test_valid(code, grammar): - module = grammar.parse(code, error_recovery=False) - fstring = module.children[0] - assert fstring.type == 'fstring' - assert fstring.get_code() == code - - -@pytest.mark.parametrize( - 'code', [ - # an f-string can't contain unmatched curly braces - 'f"}"', - 'f"{"', - 'f"""}"""', - 'f"""{"""', - - # invalid conversion characters - 'f"{1!{a}}"', - 'f"{1=!{a}}"', - 'f"{!{a}}"', - - # The curly braces must contain an expression - 'f"{}"', - 'f"{:}"', - 'f"{:}}}"', - 'f"{:1}"', - 'f"{!:}"', - 'f"{!}"', - 'f"{!a}"', - - # invalid (empty) format specifiers - 'f"{1:{}}"', - 'f"{1:{:}}"', - - # a newline without a line continuation inside a single-line string - 'f"abc\ndef"', - - # various named unicode escapes that aren't name-shaped - 'f"\\N{ BULLET }"', - 'f"\\N{NO ENTRY}"', - 'f"""\\N{NO\nENTRY}"""', - ] -) -def test_invalid(code, grammar): - with pytest.raises(ParserSyntaxError): - grammar.parse(code, error_recovery=False) - - # It should work with error recovery. - grammar.parse(code, error_recovery=True) - - -@pytest.mark.parametrize( - ('code', 'positions'), [ - # 2 times 2, 5 because python expr and endmarker. - ('f"}{"', [(1, 0), (1, 2), (1, 3), (1, 4), (1, 5)]), - ('f" :{ 1 : } "', [(1, 0), (1, 2), (1, 4), (1, 6), (1, 8), (1, 9), - (1, 10), (1, 11), (1, 12), (1, 13)]), - ('f"""\n {\nfoo\n }"""', [(1, 0), (1, 4), (2, 1), (3, 0), (4, 1), - (4, 2), (4, 5)]), - ('f"\\N{NO ENTRY} and {expr}"', [(1, 0), (1, 2), (1, 19), (1, 20), - (1, 24), (1, 25), (1, 26)]), - ] -) -def test_tokenize_start_pos(code, positions): - tokens = list(tokenize(code, version_info=(3, 6))) - assert positions == [p.start_pos for p in tokens] - - -@pytest.mark.parametrize( - 'code', [ - dedent("""\ - f'''s{ - str.uppe - ''' - """), - 'f"foo', - 'f"""foo', - 'f"abc\ndef"', - ] -) -def test_roundtrip(grammar, code): - tree = grammar.parse(code) - assert tree.get_code() == code +import pytest +from textwrap import dedent + +from parso import load_grammar, ParserSyntaxError +from parso.python.tokenize import tokenize + + +@pytest.fixture +def grammar(): + return load_grammar(version='3.8') + + +@pytest.mark.parametrize( + 'code', [ + # simple cases + 'f"{1}"', + 'f"""{1}"""', + 'f"{foo} {bar}"', + + # empty string + 'f""', + 'f""""""', + + # empty format specifier is okay + 'f"{1:}"', + + # use of conversion options + 'f"{1!a}"', + 'f"{1!a:1}"', + + # format specifiers + 'f"{1:1}"', + 'f"{1:1.{32}}"', + 'f"{1::>4}"', + 'f"{x:{y}}"', + 'f"{x:{y:}}"', + 'f"{x:{y:1}}"', + + # Escapes + 'f"{{}}"', + 'f"{{{1}}}"', + 'f"{{{1}"', + 'f"1{{2{{3"', + 'f"}}"', + + # New Python 3.8 syntax f'{a=}' + 'f"{a=}"', + 'f"{a()=}"', + + # multiline f-string + 'f"""abc\ndef"""', + 'f"""abc{\n123}def"""', + + # a line continuation inside of an fstring_string + 'f"abc\\\ndef"', + 'f"\\\n{123}\\\n"', + + # a line continuation inside of an fstring_expr + 'f"{\\\n123}"', + + # a line continuation inside of an format spec + 'f"{123:.2\\\nf}"', + + # some unparenthesized syntactic structures + 'f"{*x,}"', + 'f"{*x, *y}"', + 'f"{x, *y}"', + 'f"{*x, y}"', + 'f"{x for x in [1]}"', + + # named unicode characters + 'f"\\N{BULLET}"', + 'f"\\N{FLEUR-DE-LIS}"', + 'f"\\N{NO ENTRY}"', + 'f"Combo {expr} and \\N{NO ENTRY}"', + 'f"\\N{NO ENTRY} and {expr}"', + 'f"\\N{no entry}"', + 'f"\\N{SOYOMBO LETTER -A}"', + 'f"\\N{DOMINO TILE HORIZONTAL-00-00}"', + 'f"""\\N{NO ENTRY}"""', + ] +) +def test_valid(code, grammar): + module = grammar.parse(code, error_recovery=False) + fstring = module.children[0] + assert fstring.type == 'fstring' + assert fstring.get_code() == code + + +@pytest.mark.parametrize( + 'code', [ + # an f-string can't contain unmatched curly braces + 'f"}"', + 'f"{"', + 'f"""}"""', + 'f"""{"""', + + # invalid conversion characters + 'f"{1!{a}}"', + 'f"{1=!{a}}"', + 'f"{!{a}}"', + + # The curly braces must contain an expression + 'f"{}"', + 'f"{:}"', + 'f"{:}}}"', + 'f"{:1}"', + 'f"{!:}"', + 'f"{!}"', + 'f"{!a}"', + + # invalid (empty) format specifiers + 'f"{1:{}}"', + 'f"{1:{:}}"', + + # a newline without a line continuation inside a single-line string + 'f"abc\ndef"', + + # various named unicode escapes that aren't name-shaped + 'f"\\N{ BULLET }"', + 'f"\\N{NO ENTRY}"', + 'f"""\\N{NO\nENTRY}"""', + ] +) +def test_invalid(code, grammar): + with pytest.raises(ParserSyntaxError): + grammar.parse(code, error_recovery=False) + + # It should work with error recovery. + grammar.parse(code, error_recovery=True) + + +@pytest.mark.parametrize( + ('code', 'positions'), [ + # 2 times 2, 5 because python expr and endmarker. + ('f"}{"', [(1, 0), (1, 2), (1, 3), (1, 4), (1, 5)]), + ('f" :{ 1 : } "', [(1, 0), (1, 2), (1, 4), (1, 6), (1, 8), (1, 9), + (1, 10), (1, 11), (1, 12), (1, 13)]), + ('f"""\n {\nfoo\n }"""', [(1, 0), (1, 4), (2, 1), (3, 0), (4, 1), + (4, 2), (4, 5)]), + ('f"\\N{NO ENTRY} and {expr}"', [(1, 0), (1, 2), (1, 19), (1, 20), + (1, 24), (1, 25), (1, 26)]), + ] +) +def test_tokenize_start_pos(code, positions): + tokens = list(tokenize(code, version_info=(3, 6))) + assert positions == [p.start_pos for p in tokens] + + +@pytest.mark.parametrize( + 'code', [ + dedent("""\ + f'''s{ + str.uppe + ''' + """), + 'f"foo', + 'f"""foo', + 'f"abc\ndef"', + ] +) +def test_roundtrip(grammar, code): + tree = grammar.parse(code) + assert tree.get_code() == code diff --git a/contrib/python/parso/py3/tests/test_get_code.py b/contrib/python/parso/py3/tests/test_get_code.py index d99d792b93..1ae116b957 100644 --- a/contrib/python/parso/py3/tests/test_get_code.py +++ b/contrib/python/parso/py3/tests/test_get_code.py @@ -1,133 +1,133 @@ -import difflib - -import pytest - -from parso import parse - -code_basic_features = ''' -"""A mod docstring""" - -def a_function(a_argument, a_default = "default"): - """A func docstring""" - - a_result = 3 * a_argument - print(a_result) # a comment - b = """ -from -to""" + "huhu" - - - if a_default == "default": - return str(a_result) - else - return None -''' - - -def diff_code_assert(a, b, n=4): - if a != b: - diff = "\n".join(difflib.unified_diff( - a.splitlines(), - b.splitlines(), - n=n, - lineterm="" - )) - assert False, "Code does not match:\n%s\n\ncreated code:\n%s" % ( - diff, - b - ) - pass - - -def test_basic_parsing(): - """Validate the parsing features""" - - m = parse(code_basic_features) - diff_code_assert( - code_basic_features, - m.get_code() - ) - - -def test_operators(): - src = '5 * 3' - module = parse(src) - diff_code_assert(src, module.get_code()) - - -def test_get_code(): - """Use the same code that the parser also generates, to compare""" - s = '''"""a docstring""" -class SomeClass(object, mixin): - def __init__(self): - self.xy = 3.0 - """statement docstr""" - def some_method(self): - return 1 - def yield_method(self): - while hasattr(self, 'xy'): - yield True - for x in [1, 2]: - yield x - def empty(self): - pass -class Empty: - pass -class WithDocstring: - """class docstr""" - pass -def method_with_docstring(): - """class docstr""" - pass -''' - assert parse(s).get_code() == s - - -def test_end_newlines(): - """ - The Python grammar explicitly needs a newline at the end. Jedi though still - wants to be able, to return the exact same code without the additional new - line the parser needs. - """ - def test(source, end_pos): - module = parse(source) - assert module.get_code() == source - assert module.end_pos == end_pos - - test('a', (1, 1)) - test('a\n', (2, 0)) - test('a\nb', (2, 1)) - test('a\n#comment\n', (3, 0)) - test('a\n#comment', (2, 8)) - test('a#comment', (1, 9)) - test('def a():\n pass', (2, 5)) - - test('def a(', (1, 6)) - - -@pytest.mark.parametrize(('code', 'types'), [ - ('\r', ['endmarker']), - ('\n\r', ['endmarker']) -]) -def test_carriage_return_at_end(code, types): - """ - By adding an artificial newline this created weird side effects for - \r at the end of files. - """ - tree = parse(code) - assert tree.get_code() == code - assert [c.type for c in tree.children] == types - assert tree.end_pos == (len(code) + 1, 0) - - -@pytest.mark.parametrize('code', [ - ' ', - ' F"""', - ' F"""\n', - ' F""" \n', - ' F""" \n3', - ' f"""\n"""', - ' f"""\n"""\n', -]) -def test_full_code_round_trip(code): - assert parse(code).get_code() == code +import difflib + +import pytest + +from parso import parse + +code_basic_features = ''' +"""A mod docstring""" + +def a_function(a_argument, a_default = "default"): + """A func docstring""" + + a_result = 3 * a_argument + print(a_result) # a comment + b = """ +from +to""" + "huhu" + + + if a_default == "default": + return str(a_result) + else + return None +''' + + +def diff_code_assert(a, b, n=4): + if a != b: + diff = "\n".join(difflib.unified_diff( + a.splitlines(), + b.splitlines(), + n=n, + lineterm="" + )) + assert False, "Code does not match:\n%s\n\ncreated code:\n%s" % ( + diff, + b + ) + pass + + +def test_basic_parsing(): + """Validate the parsing features""" + + m = parse(code_basic_features) + diff_code_assert( + code_basic_features, + m.get_code() + ) + + +def test_operators(): + src = '5 * 3' + module = parse(src) + diff_code_assert(src, module.get_code()) + + +def test_get_code(): + """Use the same code that the parser also generates, to compare""" + s = '''"""a docstring""" +class SomeClass(object, mixin): + def __init__(self): + self.xy = 3.0 + """statement docstr""" + def some_method(self): + return 1 + def yield_method(self): + while hasattr(self, 'xy'): + yield True + for x in [1, 2]: + yield x + def empty(self): + pass +class Empty: + pass +class WithDocstring: + """class docstr""" + pass +def method_with_docstring(): + """class docstr""" + pass +''' + assert parse(s).get_code() == s + + +def test_end_newlines(): + """ + The Python grammar explicitly needs a newline at the end. Jedi though still + wants to be able, to return the exact same code without the additional new + line the parser needs. + """ + def test(source, end_pos): + module = parse(source) + assert module.get_code() == source + assert module.end_pos == end_pos + + test('a', (1, 1)) + test('a\n', (2, 0)) + test('a\nb', (2, 1)) + test('a\n#comment\n', (3, 0)) + test('a\n#comment', (2, 8)) + test('a#comment', (1, 9)) + test('def a():\n pass', (2, 5)) + + test('def a(', (1, 6)) + + +@pytest.mark.parametrize(('code', 'types'), [ + ('\r', ['endmarker']), + ('\n\r', ['endmarker']) +]) +def test_carriage_return_at_end(code, types): + """ + By adding an artificial newline this created weird side effects for + \r at the end of files. + """ + tree = parse(code) + assert tree.get_code() == code + assert [c.type for c in tree.children] == types + assert tree.end_pos == (len(code) + 1, 0) + + +@pytest.mark.parametrize('code', [ + ' ', + ' F"""', + ' F"""\n', + ' F""" \n', + ' F""" \n3', + ' f"""\n"""', + ' f"""\n"""\n', +]) +def test_full_code_round_trip(code): + assert parse(code).get_code() == code diff --git a/contrib/python/parso/py3/tests/test_grammar.py b/contrib/python/parso/py3/tests/test_grammar.py index 60a249b8f1..a18265229c 100644 --- a/contrib/python/parso/py3/tests/test_grammar.py +++ b/contrib/python/parso/py3/tests/test_grammar.py @@ -1,8 +1,8 @@ -import parso - -import pytest - - -def test_non_unicode(): - with pytest.raises(UnicodeDecodeError): - parso.parse(b'\xe4') +import parso + +import pytest + + +def test_non_unicode(): + with pytest.raises(UnicodeDecodeError): + parso.parse(b'\xe4') diff --git a/contrib/python/parso/py3/tests/test_load_grammar.py b/contrib/python/parso/py3/tests/test_load_grammar.py index 0ea648eb3e..a4b02f2973 100644 --- a/contrib/python/parso/py3/tests/test_load_grammar.py +++ b/contrib/python/parso/py3/tests/test_load_grammar.py @@ -1,31 +1,31 @@ -import pytest -from parso.grammar import load_grammar -from parso import utils - - -def test_load_inexisting_grammar(): - # This version shouldn't be out for a while, but if we ever do, wow! - with pytest.raises(NotImplementedError): - load_grammar(version='15.8') - # The same is true for very old grammars (even though this is probably not - # going to be an issue. - with pytest.raises(NotImplementedError): - load_grammar(version='1.5') - - -@pytest.mark.parametrize(('string', 'result'), [ - ('2', (2, 7)), ('3', (3, 6)), ('1.1', (1, 1)), ('1.1.1', (1, 1)), ('300.1.31', (300, 1)) -]) -def test_parse_version(string, result): - assert utils._parse_version(string) == result - - -@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5']) -def test_invalid_grammar_version(string): - with pytest.raises(ValueError): - load_grammar(version=string) - - -def test_grammar_int_version(): - with pytest.raises(TypeError): - load_grammar(version=3.8) +import pytest +from parso.grammar import load_grammar +from parso import utils + + +def test_load_inexisting_grammar(): + # This version shouldn't be out for a while, but if we ever do, wow! + with pytest.raises(NotImplementedError): + load_grammar(version='15.8') + # The same is true for very old grammars (even though this is probably not + # going to be an issue. + with pytest.raises(NotImplementedError): + load_grammar(version='1.5') + + +@pytest.mark.parametrize(('string', 'result'), [ + ('2', (2, 7)), ('3', (3, 6)), ('1.1', (1, 1)), ('1.1.1', (1, 1)), ('300.1.31', (300, 1)) +]) +def test_parse_version(string, result): + assert utils._parse_version(string) == result + + +@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5']) +def test_invalid_grammar_version(string): + with pytest.raises(ValueError): + load_grammar(version=string) + + +def test_grammar_int_version(): + with pytest.raises(TypeError): + load_grammar(version=3.8) diff --git a/contrib/python/parso/py3/tests/test_normalizer_issues_files.py b/contrib/python/parso/py3/tests/test_normalizer_issues_files.py index c6a23497e5..ad38a68646 100644 --- a/contrib/python/parso/py3/tests/test_normalizer_issues_files.py +++ b/contrib/python/parso/py3/tests/test_normalizer_issues_files.py @@ -1,71 +1,71 @@ -""" -To easily verify if our normalizer raises the right error codes, just use the -tests of pydocstyle. -""" - -import difflib -import re -from functools import total_ordering -from typing import Iterator, Tuple - -import parso -from parso.utils import python_bytes_to_unicode - - -@total_ordering -class WantedIssue: - def __init__(self, code: str, line: int, column: int) -> None: - self.code = code - self._line = line - self._column = column - - def __eq__(self, other): - return self.code == other.code and self.start_pos == other.start_pos - - def __lt__(self, other: 'WantedIssue') -> bool: - return self.start_pos < other.start_pos or self.code < other.code - - def __hash__(self) -> int: - return hash(str(self.code) + str(self._line) + str(self._column)) - - @property - def start_pos(self) -> Tuple[int, int]: - return self._line, self._column - - -def collect_errors(code: str) -> Iterator[WantedIssue]: - for line_nr, line in enumerate(code.splitlines(), 1): - match = re.match(r'(\s*)#: (.*)$', line) - if match is not None: - codes = match.group(2) - for code in codes.split(): - code, _, add_indent = code.partition(':') - column = int(add_indent or len(match.group(1))) - - code, _, add_line = code.partition('+') - ln = line_nr + 1 + int(add_line or 0) - - yield WantedIssue(code[1:], ln, column) - - -def test_normalizer_issue(normalizer_issue_case): - def sort(issues): - issues = sorted(issues, key=lambda i: (i.start_pos, i.code)) - return ["(%s, %s): %s" % (i.start_pos[0], i.start_pos[1], i.code) - for i in issues] - - with open(normalizer_issue_case.path, 'rb') as f: - code = python_bytes_to_unicode(f.read()) - - desired = sort(collect_errors(code)) - - grammar = parso.load_grammar(version=normalizer_issue_case.python_version) - module = grammar.parse(code) - issues = grammar._get_normalizer_issues(module) - actual = sort(issues) - - diff = '\n'.join(difflib.ndiff(desired, actual)) - # To make the pytest -v diff a bit prettier, stop pytest to rewrite assert - # statements by executing the comparison earlier. - _bool = desired == actual - assert _bool, '\n' + diff +""" +To easily verify if our normalizer raises the right error codes, just use the +tests of pydocstyle. +""" + +import difflib +import re +from functools import total_ordering +from typing import Iterator, Tuple + +import parso +from parso.utils import python_bytes_to_unicode + + +@total_ordering +class WantedIssue: + def __init__(self, code: str, line: int, column: int) -> None: + self.code = code + self._line = line + self._column = column + + def __eq__(self, other): + return self.code == other.code and self.start_pos == other.start_pos + + def __lt__(self, other: 'WantedIssue') -> bool: + return self.start_pos < other.start_pos or self.code < other.code + + def __hash__(self) -> int: + return hash(str(self.code) + str(self._line) + str(self._column)) + + @property + def start_pos(self) -> Tuple[int, int]: + return self._line, self._column + + +def collect_errors(code: str) -> Iterator[WantedIssue]: + for line_nr, line in enumerate(code.splitlines(), 1): + match = re.match(r'(\s*)#: (.*)$', line) + if match is not None: + codes = match.group(2) + for code in codes.split(): + code, _, add_indent = code.partition(':') + column = int(add_indent or len(match.group(1))) + + code, _, add_line = code.partition('+') + ln = line_nr + 1 + int(add_line or 0) + + yield WantedIssue(code[1:], ln, column) + + +def test_normalizer_issue(normalizer_issue_case): + def sort(issues): + issues = sorted(issues, key=lambda i: (i.start_pos, i.code)) + return ["(%s, %s): %s" % (i.start_pos[0], i.start_pos[1], i.code) + for i in issues] + + with open(normalizer_issue_case.path, 'rb') as f: + code = python_bytes_to_unicode(f.read()) + + desired = sort(collect_errors(code)) + + grammar = parso.load_grammar(version=normalizer_issue_case.python_version) + module = grammar.parse(code) + issues = grammar._get_normalizer_issues(module) + actual = sort(issues) + + diff = '\n'.join(difflib.ndiff(desired, actual)) + # To make the pytest -v diff a bit prettier, stop pytest to rewrite assert + # statements by executing the comparison earlier. + _bool = desired == actual + assert _bool, '\n' + diff diff --git a/contrib/python/parso/py3/tests/test_old_fast_parser.py b/contrib/python/parso/py3/tests/test_old_fast_parser.py index 6f332cfc54..788b978cbb 100644 --- a/contrib/python/parso/py3/tests/test_old_fast_parser.py +++ b/contrib/python/parso/py3/tests/test_old_fast_parser.py @@ -1,209 +1,209 @@ -""" -These tests test the cases that the old fast parser tested with the normal -parser. - -The old fast parser doesn't exist anymore and was replaced with a diff parser. -However the tests might still be relevant for the parser. -""" - -from textwrap import dedent - -from parso import parse - - -def test_carriage_return_splitting(): - source = dedent(''' - - - - "string" - - class Foo(): - pass - ''') - source = source.replace('\n', '\r\n') - module = parse(source) - assert [n.value for lst in module.get_used_names().values() for n in lst] == ['Foo'] - - -def check_p(src, number_parsers_used, number_of_splits=None, number_of_misses=0): - if number_of_splits is None: - number_of_splits = number_parsers_used - - module_node = parse(src) - - assert src == module_node.get_code() - return module_node - - -def test_for(): - src = dedent("""\ - for a in [1,2]: - a - - for a1 in 1,"": - a1 - """) - check_p(src, 1) - - -def test_class_with_class_var(): - src = dedent("""\ - class SuperClass: - class_super = 3 - def __init__(self): - self.foo = 4 - pass - """) - check_p(src, 3) - - -def test_func_with_if(): - src = dedent("""\ - def recursion(a): - if foo: - return recursion(a) - else: - if bar: - return inexistent - else: - return a - """) - check_p(src, 1) - - -def test_decorator(): - src = dedent("""\ - class Decorator(): - @memoize - def dec(self, a): - return a - """) - check_p(src, 2) - - -def test_nested_funcs(): - src = dedent("""\ - def memoize(func): - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - return wrapper - """) - check_p(src, 3) - - -def test_multi_line_params(): - src = dedent("""\ - def x(a, - b): - pass - - foo = 1 - """) - check_p(src, 2) - - -def test_class_func_if(): - src = dedent("""\ - class Class: - def func(self): - if 1: - a - else: - b - - pass - """) - check_p(src, 3) - - -def test_multi_line_for(): - src = dedent("""\ - for x in [1, - 2]: - pass - - pass - """) - check_p(src, 1) - - -def test_wrong_indentation(): - src = dedent("""\ - def func(): - a - b - a - """) - check_p(src, 1) - - src = dedent("""\ - def complex(): - def nested(): - a - b - a - - def other(): - pass - """) - check_p(src, 3) - - -def test_strange_parentheses(): - src = dedent(""" - class X(): - a = (1 - if 1 else 2) - def x(): - pass - """) - check_p(src, 2) - - -def test_fake_parentheses(): - """ - The fast parser splitting counts parentheses, but not as correct tokens. - Therefore parentheses in string tokens are included as well. This needs to - be accounted for. - """ - src = dedent(r""" - def x(): - a = (')' - if 1 else 2) - def y(): - pass - def z(): - pass - """) - check_p(src, 3, 2, 1) - - -def test_additional_indent(): - source = dedent('''\ - int( - def x(): - pass - ''') - - check_p(source, 2) - - -def test_round_trip(): - code = dedent(''' - def x(): - """hahaha""" - func''') - - assert parse(code).get_code() == code - - -def test_parentheses_in_string(): - code = dedent(''' - def x(): - '(' - - import abc - - abc.''') - check_p(code, 2, 1, 1) +""" +These tests test the cases that the old fast parser tested with the normal +parser. + +The old fast parser doesn't exist anymore and was replaced with a diff parser. +However the tests might still be relevant for the parser. +""" + +from textwrap import dedent + +from parso import parse + + +def test_carriage_return_splitting(): + source = dedent(''' + + + + "string" + + class Foo(): + pass + ''') + source = source.replace('\n', '\r\n') + module = parse(source) + assert [n.value for lst in module.get_used_names().values() for n in lst] == ['Foo'] + + +def check_p(src, number_parsers_used, number_of_splits=None, number_of_misses=0): + if number_of_splits is None: + number_of_splits = number_parsers_used + + module_node = parse(src) + + assert src == module_node.get_code() + return module_node + + +def test_for(): + src = dedent("""\ + for a in [1,2]: + a + + for a1 in 1,"": + a1 + """) + check_p(src, 1) + + +def test_class_with_class_var(): + src = dedent("""\ + class SuperClass: + class_super = 3 + def __init__(self): + self.foo = 4 + pass + """) + check_p(src, 3) + + +def test_func_with_if(): + src = dedent("""\ + def recursion(a): + if foo: + return recursion(a) + else: + if bar: + return inexistent + else: + return a + """) + check_p(src, 1) + + +def test_decorator(): + src = dedent("""\ + class Decorator(): + @memoize + def dec(self, a): + return a + """) + check_p(src, 2) + + +def test_nested_funcs(): + src = dedent("""\ + def memoize(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + """) + check_p(src, 3) + + +def test_multi_line_params(): + src = dedent("""\ + def x(a, + b): + pass + + foo = 1 + """) + check_p(src, 2) + + +def test_class_func_if(): + src = dedent("""\ + class Class: + def func(self): + if 1: + a + else: + b + + pass + """) + check_p(src, 3) + + +def test_multi_line_for(): + src = dedent("""\ + for x in [1, + 2]: + pass + + pass + """) + check_p(src, 1) + + +def test_wrong_indentation(): + src = dedent("""\ + def func(): + a + b + a + """) + check_p(src, 1) + + src = dedent("""\ + def complex(): + def nested(): + a + b + a + + def other(): + pass + """) + check_p(src, 3) + + +def test_strange_parentheses(): + src = dedent(""" + class X(): + a = (1 + if 1 else 2) + def x(): + pass + """) + check_p(src, 2) + + +def test_fake_parentheses(): + """ + The fast parser splitting counts parentheses, but not as correct tokens. + Therefore parentheses in string tokens are included as well. This needs to + be accounted for. + """ + src = dedent(r""" + def x(): + a = (')' + if 1 else 2) + def y(): + pass + def z(): + pass + """) + check_p(src, 3, 2, 1) + + +def test_additional_indent(): + source = dedent('''\ + int( + def x(): + pass + ''') + + check_p(source, 2) + + +def test_round_trip(): + code = dedent(''' + def x(): + """hahaha""" + func''') + + assert parse(code).get_code() == code + + +def test_parentheses_in_string(): + code = dedent(''' + def x(): + '(' + + import abc + + abc.''') + check_p(code, 2, 1, 1) diff --git a/contrib/python/parso/py3/tests/test_param_splitting.py b/contrib/python/parso/py3/tests/test_param_splitting.py index 3ea5f1653b..aa4878100e 100644 --- a/contrib/python/parso/py3/tests/test_param_splitting.py +++ b/contrib/python/parso/py3/tests/test_param_splitting.py @@ -1,47 +1,47 @@ -''' -To make the life of any analysis easier, we are generating Param objects -instead of simple parser objects. -''' - -from textwrap import dedent - -from parso import parse - - -def assert_params(param_string, **wanted_dct): - source = dedent(''' - def x(%s): - pass - ''') % param_string - - module = parse(source) - funcdef = next(module.iter_funcdefs()) - dct = dict((p.name.value, p.default and p.default.get_code()) - for p in funcdef.get_params()) - assert dct == wanted_dct - assert module.get_code() == source - - -def test_split_params_with_separation_star(): - assert_params('x, y=1, *, z=3', x=None, y='1', z='3') - assert_params('*, x', x=None) - assert_params('*') - - -def test_split_params_with_stars(): - assert_params('x, *args', x=None, args=None) - assert_params('**kwargs', kwargs=None) - assert_params('*args, **kwargs', args=None, kwargs=None) - - -def test_kw_only_no_kw(works_in_py): - """ - Parsing this should be working. In CPython the parser also parses this and - in a later step the AST complains. - """ - module = works_in_py.parse('def test(arg, *):\n pass') - if module is not None: - func = module.children[0] - open_, p1, asterisk, close = func._get_param_nodes() - assert p1.get_code('arg,') - assert asterisk.value == '*' +''' +To make the life of any analysis easier, we are generating Param objects +instead of simple parser objects. +''' + +from textwrap import dedent + +from parso import parse + + +def assert_params(param_string, **wanted_dct): + source = dedent(''' + def x(%s): + pass + ''') % param_string + + module = parse(source) + funcdef = next(module.iter_funcdefs()) + dct = dict((p.name.value, p.default and p.default.get_code()) + for p in funcdef.get_params()) + assert dct == wanted_dct + assert module.get_code() == source + + +def test_split_params_with_separation_star(): + assert_params('x, y=1, *, z=3', x=None, y='1', z='3') + assert_params('*, x', x=None) + assert_params('*') + + +def test_split_params_with_stars(): + assert_params('x, *args', x=None, args=None) + assert_params('**kwargs', kwargs=None) + assert_params('*args, **kwargs', args=None, kwargs=None) + + +def test_kw_only_no_kw(works_in_py): + """ + Parsing this should be working. In CPython the parser also parses this and + in a later step the AST complains. + """ + module = works_in_py.parse('def test(arg, *):\n pass') + if module is not None: + func = module.children[0] + open_, p1, asterisk, close = func._get_param_nodes() + assert p1.get_code('arg,') + assert asterisk.value == '*' diff --git a/contrib/python/parso/py3/tests/test_parser.py b/contrib/python/parso/py3/tests/test_parser.py index e087b0d554..b1f41bbac9 100644 --- a/contrib/python/parso/py3/tests/test_parser.py +++ b/contrib/python/parso/py3/tests/test_parser.py @@ -1,208 +1,208 @@ -# -*- coding: utf-8 -*- -from textwrap import dedent - -import pytest - -from parso import parse -from parso.python import tree -from parso.utils import split_lines - - -def test_basic_parsing(each_version): - def compare(string): - """Generates the AST object and then regenerates the code.""" - assert parse(string, version=each_version).get_code() == string - - compare('\na #pass\n') - compare('wblabla* 1\t\n') - compare('def x(a, b:3): pass\n') - compare('assert foo\n') - - -def test_subscope_names(each_version): - def get_sub(source): - return parse(source, version=each_version).children[0] - - name = get_sub('class Foo: pass').name - assert name.start_pos == (1, len('class ')) - assert name.end_pos == (1, len('class Foo')) - assert name.value == 'Foo' - - name = get_sub('def foo(): pass').name - assert name.start_pos == (1, len('def ')) - assert name.end_pos == (1, len('def foo')) - assert name.value == 'foo' - - -def test_import_names(each_version): - def get_import(source): - return next(parse(source, version=each_version).iter_imports()) - - imp = get_import('import math\n') - names = imp.get_defined_names() - assert len(names) == 1 - assert names[0].value == 'math' - assert names[0].start_pos == (1, len('import ')) - assert names[0].end_pos == (1, len('import math')) - - assert imp.start_pos == (1, 0) - assert imp.end_pos == (1, len('import math')) - - -def test_end_pos(each_version): - s = dedent(''' - x = ['a', 'b', 'c'] - def func(): - y = None - ''') - parser = parse(s, version=each_version) - scope = next(parser.iter_funcdefs()) - assert scope.start_pos == (3, 0) - assert scope.end_pos == (5, 0) - - -def test_carriage_return_statements(each_version): - source = dedent(''' - foo = 'ns1!' - - # this is a namespace package - ''') - source = source.replace('\n', '\r\n') - stmt = parse(source, version=each_version).children[0] - assert '#' not in stmt.get_code() - - -def test_incomplete_list_comprehension(each_version): - """ Shouldn't raise an error, same bug as #418. """ - # With the old parser this actually returned a statement. With the new - # parser only valid statements generate one. - children = parse('(1 for def', version=each_version).children - assert [c.type for c in children] == \ - ['error_node', 'error_node', 'endmarker'] - - -def test_newline_positions(each_version): - endmarker = parse('a\n', version=each_version).children[-1] - assert endmarker.end_pos == (2, 0) - new_line = endmarker.get_previous_leaf() - assert new_line.start_pos == (1, 1) - assert new_line.end_pos == (2, 0) - - -def test_end_pos_error_correction(each_version): - """ - Source code without ending newline are given one, because the Python - grammar needs it. However, they are removed again. We still want the right - end_pos, even if something breaks in the parser (error correction). - """ - s = 'def x():\n .' - m = parse(s, version=each_version) - func = m.children[0] - assert func.type == 'funcdef' - assert func.end_pos == (2, 2) - assert m.end_pos == (2, 2) - - -def test_param_splitting(each_version): - """ - Jedi splits parameters into params, this is not what the grammar does, - but Jedi does this to simplify argument parsing. - """ - def check(src, result): - m = parse(src, version=each_version) - assert not list(m.iter_funcdefs()) - - check('def x(a, (b, c)):\n pass', ['a']) - check('def x((b, c)):\n pass', []) - - -def test_unicode_string(): - s = tree.String(None, 'bö', (0, 0)) - assert repr(s) # Should not raise an Error! - - -def test_backslash_dos_style(each_version): - assert parse('\\\r\n', version=each_version) - - -def test_started_lambda_stmt(each_version): - m = parse('lambda a, b: a i', version=each_version) - assert m.children[0].type == 'error_node' - - -@pytest.mark.parametrize('code', ['foo "', 'foo """\n', 'foo """\nbar']) -def test_open_string_literal(each_version, code): - """ - Testing mostly if removing the last newline works. - """ - lines = split_lines(code, keepends=True) - end_pos = (len(lines), len(lines[-1])) - module = parse(code, version=each_version) - assert module.get_code() == code - assert module.end_pos == end_pos == module.children[1].end_pos - - -def test_too_many_params(): - with pytest.raises(TypeError): - parse('asdf', hello=3) - - -def test_dedent_at_end(each_version): - code = dedent(''' - for foobar in [1]: - foobar''') - module = parse(code, version=each_version) - assert module.get_code() == code - suite = module.children[0].children[-1] - foobar = suite.children[-1] - assert foobar.type == 'name' - - -def test_no_error_nodes(each_version): - def check(node): - assert node.type not in ('error_leaf', 'error_node') - - try: - children = node.children - except AttributeError: - pass - else: - for child in children: - check(child) - - check(parse("if foo:\n bar", version=each_version)) - - -def test_named_expression(works_ge_py38): - works_ge_py38.parse("(a := 1, a + 1)") - - -def test_extended_rhs_annassign(works_ge_py38): - works_ge_py38.parse("x: y = z,") - works_ge_py38.parse("x: Tuple[int, ...] = z, *q, w") - - -@pytest.mark.parametrize( - 'param_code', [ - 'a=1, /', - 'a, /', - 'a=1, /, b=3', - 'a, /, b', - 'a, /, b', - 'a, /, *, b', - 'a, /, **kwargs', - ] -) -def test_positional_only_arguments(works_ge_py38, param_code): - works_ge_py38.parse("def x(%s): pass" % param_code) - - -@pytest.mark.parametrize( - 'expression', [ - 'a + a', - 'lambda x: x', - 'a := lambda x: x' - ] -) -def test_decorator_expression(works_ge_py39, expression): - works_ge_py39.parse("@%s\ndef x(): pass" % expression) +# -*- coding: utf-8 -*- +from textwrap import dedent + +import pytest + +from parso import parse +from parso.python import tree +from parso.utils import split_lines + + +def test_basic_parsing(each_version): + def compare(string): + """Generates the AST object and then regenerates the code.""" + assert parse(string, version=each_version).get_code() == string + + compare('\na #pass\n') + compare('wblabla* 1\t\n') + compare('def x(a, b:3): pass\n') + compare('assert foo\n') + + +def test_subscope_names(each_version): + def get_sub(source): + return parse(source, version=each_version).children[0] + + name = get_sub('class Foo: pass').name + assert name.start_pos == (1, len('class ')) + assert name.end_pos == (1, len('class Foo')) + assert name.value == 'Foo' + + name = get_sub('def foo(): pass').name + assert name.start_pos == (1, len('def ')) + assert name.end_pos == (1, len('def foo')) + assert name.value == 'foo' + + +def test_import_names(each_version): + def get_import(source): + return next(parse(source, version=each_version).iter_imports()) + + imp = get_import('import math\n') + names = imp.get_defined_names() + assert len(names) == 1 + assert names[0].value == 'math' + assert names[0].start_pos == (1, len('import ')) + assert names[0].end_pos == (1, len('import math')) + + assert imp.start_pos == (1, 0) + assert imp.end_pos == (1, len('import math')) + + +def test_end_pos(each_version): + s = dedent(''' + x = ['a', 'b', 'c'] + def func(): + y = None + ''') + parser = parse(s, version=each_version) + scope = next(parser.iter_funcdefs()) + assert scope.start_pos == (3, 0) + assert scope.end_pos == (5, 0) + + +def test_carriage_return_statements(each_version): + source = dedent(''' + foo = 'ns1!' + + # this is a namespace package + ''') + source = source.replace('\n', '\r\n') + stmt = parse(source, version=each_version).children[0] + assert '#' not in stmt.get_code() + + +def test_incomplete_list_comprehension(each_version): + """ Shouldn't raise an error, same bug as #418. """ + # With the old parser this actually returned a statement. With the new + # parser only valid statements generate one. + children = parse('(1 for def', version=each_version).children + assert [c.type for c in children] == \ + ['error_node', 'error_node', 'endmarker'] + + +def test_newline_positions(each_version): + endmarker = parse('a\n', version=each_version).children[-1] + assert endmarker.end_pos == (2, 0) + new_line = endmarker.get_previous_leaf() + assert new_line.start_pos == (1, 1) + assert new_line.end_pos == (2, 0) + + +def test_end_pos_error_correction(each_version): + """ + Source code without ending newline are given one, because the Python + grammar needs it. However, they are removed again. We still want the right + end_pos, even if something breaks in the parser (error correction). + """ + s = 'def x():\n .' + m = parse(s, version=each_version) + func = m.children[0] + assert func.type == 'funcdef' + assert func.end_pos == (2, 2) + assert m.end_pos == (2, 2) + + +def test_param_splitting(each_version): + """ + Jedi splits parameters into params, this is not what the grammar does, + but Jedi does this to simplify argument parsing. + """ + def check(src, result): + m = parse(src, version=each_version) + assert not list(m.iter_funcdefs()) + + check('def x(a, (b, c)):\n pass', ['a']) + check('def x((b, c)):\n pass', []) + + +def test_unicode_string(): + s = tree.String(None, 'bö', (0, 0)) + assert repr(s) # Should not raise an Error! + + +def test_backslash_dos_style(each_version): + assert parse('\\\r\n', version=each_version) + + +def test_started_lambda_stmt(each_version): + m = parse('lambda a, b: a i', version=each_version) + assert m.children[0].type == 'error_node' + + +@pytest.mark.parametrize('code', ['foo "', 'foo """\n', 'foo """\nbar']) +def test_open_string_literal(each_version, code): + """ + Testing mostly if removing the last newline works. + """ + lines = split_lines(code, keepends=True) + end_pos = (len(lines), len(lines[-1])) + module = parse(code, version=each_version) + assert module.get_code() == code + assert module.end_pos == end_pos == module.children[1].end_pos + + +def test_too_many_params(): + with pytest.raises(TypeError): + parse('asdf', hello=3) + + +def test_dedent_at_end(each_version): + code = dedent(''' + for foobar in [1]: + foobar''') + module = parse(code, version=each_version) + assert module.get_code() == code + suite = module.children[0].children[-1] + foobar = suite.children[-1] + assert foobar.type == 'name' + + +def test_no_error_nodes(each_version): + def check(node): + assert node.type not in ('error_leaf', 'error_node') + + try: + children = node.children + except AttributeError: + pass + else: + for child in children: + check(child) + + check(parse("if foo:\n bar", version=each_version)) + + +def test_named_expression(works_ge_py38): + works_ge_py38.parse("(a := 1, a + 1)") + + +def test_extended_rhs_annassign(works_ge_py38): + works_ge_py38.parse("x: y = z,") + works_ge_py38.parse("x: Tuple[int, ...] = z, *q, w") + + +@pytest.mark.parametrize( + 'param_code', [ + 'a=1, /', + 'a, /', + 'a=1, /, b=3', + 'a, /, b', + 'a, /, b', + 'a, /, *, b', + 'a, /, **kwargs', + ] +) +def test_positional_only_arguments(works_ge_py38, param_code): + works_ge_py38.parse("def x(%s): pass" % param_code) + + +@pytest.mark.parametrize( + 'expression', [ + 'a + a', + 'lambda x: x', + 'a := lambda x: x' + ] +) +def test_decorator_expression(works_ge_py39, expression): + works_ge_py39.parse("@%s\ndef x(): pass" % expression) diff --git a/contrib/python/parso/py3/tests/test_parser_tree.py b/contrib/python/parso/py3/tests/test_parser_tree.py index b994b9bbb8..5a0a86272b 100644 --- a/contrib/python/parso/py3/tests/test_parser_tree.py +++ b/contrib/python/parso/py3/tests/test_parser_tree.py @@ -1,266 +1,266 @@ -# -*- coding: utf-8 # This file contains Unicode characters. - -from textwrap import dedent - -import pytest - -from parso import parse -from parso.python import tree -from parso.tree import search_ancestor - - -class TestsFunctionAndLambdaParsing: - - FIXTURES = [ - ('def my_function(x, y, z) -> str:\n return x + y * z\n', { - 'name': 'my_function', - 'call_sig': 'my_function(x, y, z)', - 'params': ['x', 'y', 'z'], - 'annotation': "str", - }), - ('lambda x, y, z: x + y * z\n', { - 'name': '<lambda>', - 'call_sig': '<lambda>(x, y, z)', - 'params': ['x', 'y', 'z'], - }), - ] - - @pytest.fixture(params=FIXTURES) - def node(self, request): - parsed = parse(dedent(request.param[0]), version='3.10') - request.keywords['expected'] = request.param[1] - child = parsed.children[0] - if child.type == 'simple_stmt': - child = child.children[0] - return child - - @pytest.fixture() - def expected(self, request, node): - return request.keywords['expected'] - - def test_name(self, node, expected): - if node.type != 'lambdef': - assert isinstance(node.name, tree.Name) - assert node.name.value == expected['name'] - - def test_params(self, node, expected): - assert isinstance(node.get_params(), list) - assert all(isinstance(x, tree.Param) for x in node.get_params()) - assert [str(x.name.value) for x in node.get_params()] == [x for x in expected['params']] - - def test_is_generator(self, node, expected): - assert node.is_generator() is expected.get('is_generator', False) - - def test_yields(self, node, expected): - assert node.is_generator() == expected.get('yields', False) - - def test_annotation(self, node, expected): - expected_annotation = expected.get('annotation', None) - if expected_annotation is None: - assert node.annotation is None - else: - assert node.annotation.value == expected_annotation - - -def test_end_pos_line(each_version): - # jedi issue #150 - s = "x()\nx( )\nx( )\nx ( )\n" - - module = parse(s, version=each_version) - for i, simple_stmt in enumerate(module.children[:-1]): - expr_stmt = simple_stmt.children[0] - assert expr_stmt.end_pos == (i + 1, i + 3) - - -def test_default_param(each_version): - func = parse('def x(foo=42): pass', version=each_version).children[0] - param, = func.get_params() - assert param.default.value == '42' - assert param.annotation is None - assert not param.star_count - - -def test_annotation_param(each_version): - func = parse('def x(foo: 3): pass', version=each_version).children[0] - param, = func.get_params() - assert param.default is None - assert param.annotation.value == '3' - assert not param.star_count - - -def test_annotation_params(each_version): - func = parse('def x(foo: 3, bar: 4): pass', version=each_version).children[0] - param1, param2 = func.get_params() - - assert param1.default is None - assert param1.annotation.value == '3' - assert not param1.star_count - - assert param2.default is None - assert param2.annotation.value == '4' - assert not param2.star_count - - -def test_default_and_annotation_param(each_version): - func = parse('def x(foo:3=42): pass', version=each_version).children[0] - param, = func.get_params() - assert param.default.value == '42' - assert param.annotation.value == '3' - assert not param.star_count - - -def get_yield_exprs(code, version): - return list(parse(code, version=version).children[0].iter_yield_exprs()) - - -def get_return_stmts(code): - return list(parse(code).children[0].iter_return_stmts()) - - -def get_raise_stmts(code, child): - return list(parse(code).children[child].iter_raise_stmts()) - - -def test_yields(each_version): - y, = get_yield_exprs('def x(): yield', each_version) - assert y.value == 'yield' - assert y.type == 'keyword' - - y, = get_yield_exprs('def x(): (yield 1)', each_version) - assert y.type == 'yield_expr' - - y, = get_yield_exprs('def x(): [1, (yield)]', each_version) - assert y.type == 'keyword' - - -def test_yield_from(): - y, = get_yield_exprs('def x(): (yield from 1)', '3.8') - assert y.type == 'yield_expr' - - -def test_returns(): - r, = get_return_stmts('def x(): return') - assert r.value == 'return' - assert r.type == 'keyword' - - r, = get_return_stmts('def x(): return 1') - assert r.type == 'return_stmt' - - -def test_raises(): - code = """ -def single_function(): - raise Exception -def top_function(): - def inner_function(): - raise NotImplementedError() - inner_function() - raise Exception -def top_function_three(): - try: - raise NotImplementedError() - except NotImplementedError: - pass - raise Exception - """ - - r = get_raise_stmts(code, 0) # Lists in a simple Function - assert len(list(r)) == 1 - - r = get_raise_stmts(code, 1) # Doesn't Exceptions list in closures - assert len(list(r)) == 1 - - r = get_raise_stmts(code, 2) # Lists inside try-catch - assert len(list(r)) == 2 - - -@pytest.mark.parametrize( - 'code, name_index, is_definition, include_setitem', [ - ('x = 3', 0, True, False), - ('x.y = 3', 0, False, False), - ('x.y = 3', 1, True, False), - ('x.y = u.v = z', 0, False, False), - ('x.y = u.v = z', 1, True, False), - ('x.y = u.v = z', 2, False, False), - ('x.y = u.v, w = z', 3, True, False), - ('x.y = u.v, w = z', 4, True, False), - ('x.y = u.v, w = z', 5, False, False), - - ('x, y = z', 0, True, False), - ('x, y = z', 1, True, False), - ('x, y = z', 2, False, False), - ('x, y = z', 2, False, False), - ('x[0], y = z', 2, False, False), - ('x[0] = z', 0, False, False), - ('x[0], y = z', 0, False, False), - ('x[0], y = z', 2, False, True), - ('x[0] = z', 0, True, True), - ('x[0], y = z', 0, True, True), - ('x: int = z', 0, True, False), - ('x: int = z', 1, False, False), - ('x: int = z', 2, False, False), - ('x: int', 0, True, False), - ('x: int', 1, False, False), - ] -) -def test_is_definition(code, name_index, is_definition, include_setitem): - module = parse(code, version='3.8') - name = module.get_first_leaf() - while True: - if name.type == 'name': - if name_index == 0: - break - name_index -= 1 - name = name.get_next_leaf() - - assert name.is_definition(include_setitem=include_setitem) == is_definition - - -def test_iter_funcdefs(): - code = dedent(''' - def normal(): ... - async def asyn(): ... - @dec - def dec_normal(): ... - @dec1 - @dec2 - async def dec_async(): ... - def broken - ''') - module = parse(code, version='3.8') - func_names = [f.name.value for f in module.iter_funcdefs()] - assert func_names == ['normal', 'asyn', 'dec_normal', 'dec_async'] - - -def test_with_stmt_get_test_node_from_name(): - code = "with A as X.Y, B as (Z), C as Q[0], D as Q['foo']: pass" - with_stmt = parse(code, version='3').children[0] - tests = [ - with_stmt.get_test_node_from_name(name).value - for name in with_stmt.get_defined_names(include_setitem=True) - ] - assert tests == ["A", "B", "C", "D"] - - -sample_module = parse('x + y') -sample_node = sample_module.children[0] -sample_leaf = sample_node.children[0] - - -@pytest.mark.parametrize( - 'node,node_types,expected_ancestor', [ - (sample_module, ('file_input',), None), - (sample_node, ('arith_expr',), None), - (sample_node, ('file_input', 'eval_input'), sample_module), - (sample_leaf, ('name',), None), - (sample_leaf, ('arith_expr',), sample_node), - (sample_leaf, ('file_input',), sample_module), - (sample_leaf, ('file_input', 'arith_expr'), sample_node), - (sample_leaf, ('shift_expr',), None), - (sample_leaf, ('name', 'shift_expr',), None), - (sample_leaf, (), None), - ] -) -def test_search_ancestor(node, node_types, expected_ancestor): - assert node.search_ancestor(*node_types) is expected_ancestor - assert search_ancestor(node, *node_types) is expected_ancestor # deprecated +# -*- coding: utf-8 # This file contains Unicode characters. + +from textwrap import dedent + +import pytest + +from parso import parse +from parso.python import tree +from parso.tree import search_ancestor + + +class TestsFunctionAndLambdaParsing: + + FIXTURES = [ + ('def my_function(x, y, z) -> str:\n return x + y * z\n', { + 'name': 'my_function', + 'call_sig': 'my_function(x, y, z)', + 'params': ['x', 'y', 'z'], + 'annotation': "str", + }), + ('lambda x, y, z: x + y * z\n', { + 'name': '<lambda>', + 'call_sig': '<lambda>(x, y, z)', + 'params': ['x', 'y', 'z'], + }), + ] + + @pytest.fixture(params=FIXTURES) + def node(self, request): + parsed = parse(dedent(request.param[0]), version='3.10') + request.keywords['expected'] = request.param[1] + child = parsed.children[0] + if child.type == 'simple_stmt': + child = child.children[0] + return child + + @pytest.fixture() + def expected(self, request, node): + return request.keywords['expected'] + + def test_name(self, node, expected): + if node.type != 'lambdef': + assert isinstance(node.name, tree.Name) + assert node.name.value == expected['name'] + + def test_params(self, node, expected): + assert isinstance(node.get_params(), list) + assert all(isinstance(x, tree.Param) for x in node.get_params()) + assert [str(x.name.value) for x in node.get_params()] == [x for x in expected['params']] + + def test_is_generator(self, node, expected): + assert node.is_generator() is expected.get('is_generator', False) + + def test_yields(self, node, expected): + assert node.is_generator() == expected.get('yields', False) + + def test_annotation(self, node, expected): + expected_annotation = expected.get('annotation', None) + if expected_annotation is None: + assert node.annotation is None + else: + assert node.annotation.value == expected_annotation + + +def test_end_pos_line(each_version): + # jedi issue #150 + s = "x()\nx( )\nx( )\nx ( )\n" + + module = parse(s, version=each_version) + for i, simple_stmt in enumerate(module.children[:-1]): + expr_stmt = simple_stmt.children[0] + assert expr_stmt.end_pos == (i + 1, i + 3) + + +def test_default_param(each_version): + func = parse('def x(foo=42): pass', version=each_version).children[0] + param, = func.get_params() + assert param.default.value == '42' + assert param.annotation is None + assert not param.star_count + + +def test_annotation_param(each_version): + func = parse('def x(foo: 3): pass', version=each_version).children[0] + param, = func.get_params() + assert param.default is None + assert param.annotation.value == '3' + assert not param.star_count + + +def test_annotation_params(each_version): + func = parse('def x(foo: 3, bar: 4): pass', version=each_version).children[0] + param1, param2 = func.get_params() + + assert param1.default is None + assert param1.annotation.value == '3' + assert not param1.star_count + + assert param2.default is None + assert param2.annotation.value == '4' + assert not param2.star_count + + +def test_default_and_annotation_param(each_version): + func = parse('def x(foo:3=42): pass', version=each_version).children[0] + param, = func.get_params() + assert param.default.value == '42' + assert param.annotation.value == '3' + assert not param.star_count + + +def get_yield_exprs(code, version): + return list(parse(code, version=version).children[0].iter_yield_exprs()) + + +def get_return_stmts(code): + return list(parse(code).children[0].iter_return_stmts()) + + +def get_raise_stmts(code, child): + return list(parse(code).children[child].iter_raise_stmts()) + + +def test_yields(each_version): + y, = get_yield_exprs('def x(): yield', each_version) + assert y.value == 'yield' + assert y.type == 'keyword' + + y, = get_yield_exprs('def x(): (yield 1)', each_version) + assert y.type == 'yield_expr' + + y, = get_yield_exprs('def x(): [1, (yield)]', each_version) + assert y.type == 'keyword' + + +def test_yield_from(): + y, = get_yield_exprs('def x(): (yield from 1)', '3.8') + assert y.type == 'yield_expr' + + +def test_returns(): + r, = get_return_stmts('def x(): return') + assert r.value == 'return' + assert r.type == 'keyword' + + r, = get_return_stmts('def x(): return 1') + assert r.type == 'return_stmt' + + +def test_raises(): + code = """ +def single_function(): + raise Exception +def top_function(): + def inner_function(): + raise NotImplementedError() + inner_function() + raise Exception +def top_function_three(): + try: + raise NotImplementedError() + except NotImplementedError: + pass + raise Exception + """ + + r = get_raise_stmts(code, 0) # Lists in a simple Function + assert len(list(r)) == 1 + + r = get_raise_stmts(code, 1) # Doesn't Exceptions list in closures + assert len(list(r)) == 1 + + r = get_raise_stmts(code, 2) # Lists inside try-catch + assert len(list(r)) == 2 + + +@pytest.mark.parametrize( + 'code, name_index, is_definition, include_setitem', [ + ('x = 3', 0, True, False), + ('x.y = 3', 0, False, False), + ('x.y = 3', 1, True, False), + ('x.y = u.v = z', 0, False, False), + ('x.y = u.v = z', 1, True, False), + ('x.y = u.v = z', 2, False, False), + ('x.y = u.v, w = z', 3, True, False), + ('x.y = u.v, w = z', 4, True, False), + ('x.y = u.v, w = z', 5, False, False), + + ('x, y = z', 0, True, False), + ('x, y = z', 1, True, False), + ('x, y = z', 2, False, False), + ('x, y = z', 2, False, False), + ('x[0], y = z', 2, False, False), + ('x[0] = z', 0, False, False), + ('x[0], y = z', 0, False, False), + ('x[0], y = z', 2, False, True), + ('x[0] = z', 0, True, True), + ('x[0], y = z', 0, True, True), + ('x: int = z', 0, True, False), + ('x: int = z', 1, False, False), + ('x: int = z', 2, False, False), + ('x: int', 0, True, False), + ('x: int', 1, False, False), + ] +) +def test_is_definition(code, name_index, is_definition, include_setitem): + module = parse(code, version='3.8') + name = module.get_first_leaf() + while True: + if name.type == 'name': + if name_index == 0: + break + name_index -= 1 + name = name.get_next_leaf() + + assert name.is_definition(include_setitem=include_setitem) == is_definition + + +def test_iter_funcdefs(): + code = dedent(''' + def normal(): ... + async def asyn(): ... + @dec + def dec_normal(): ... + @dec1 + @dec2 + async def dec_async(): ... + def broken + ''') + module = parse(code, version='3.8') + func_names = [f.name.value for f in module.iter_funcdefs()] + assert func_names == ['normal', 'asyn', 'dec_normal', 'dec_async'] + + +def test_with_stmt_get_test_node_from_name(): + code = "with A as X.Y, B as (Z), C as Q[0], D as Q['foo']: pass" + with_stmt = parse(code, version='3').children[0] + tests = [ + with_stmt.get_test_node_from_name(name).value + for name in with_stmt.get_defined_names(include_setitem=True) + ] + assert tests == ["A", "B", "C", "D"] + + +sample_module = parse('x + y') +sample_node = sample_module.children[0] +sample_leaf = sample_node.children[0] + + +@pytest.mark.parametrize( + 'node,node_types,expected_ancestor', [ + (sample_module, ('file_input',), None), + (sample_node, ('arith_expr',), None), + (sample_node, ('file_input', 'eval_input'), sample_module), + (sample_leaf, ('name',), None), + (sample_leaf, ('arith_expr',), sample_node), + (sample_leaf, ('file_input',), sample_module), + (sample_leaf, ('file_input', 'arith_expr'), sample_node), + (sample_leaf, ('shift_expr',), None), + (sample_leaf, ('name', 'shift_expr',), None), + (sample_leaf, (), None), + ] +) +def test_search_ancestor(node, node_types, expected_ancestor): + assert node.search_ancestor(*node_types) is expected_ancestor + assert search_ancestor(node, *node_types) is expected_ancestor # deprecated diff --git a/contrib/python/parso/py3/tests/test_pep8.py b/contrib/python/parso/py3/tests/test_pep8.py index 06cffb4af9..a65ccd6fce 100644 --- a/contrib/python/parso/py3/tests/test_pep8.py +++ b/contrib/python/parso/py3/tests/test_pep8.py @@ -1,42 +1,42 @@ -import parso - - -def issues(code): - grammar = parso.load_grammar() - module = parso.parse(code) - return grammar._get_normalizer_issues(module) - - -def test_eof_newline(): - def assert_issue(code): - found = issues(code) - assert len(found) == 1 - issue, = found - assert issue.code == 292 - - assert not issues('asdf = 1\n') - assert not issues('asdf = 1\r\n') - assert not issues('asdf = 1\r') - assert_issue('asdf = 1') - assert_issue('asdf = 1\n# foo') - assert_issue('# foobar') - assert_issue('') - assert_issue('foo = 1 # comment') - - -def test_eof_blankline(): - def assert_issue(code): - found = issues(code) - assert len(found) == 1 - issue, = found - assert issue.code == 391 - - assert_issue('asdf = 1\n\n') - assert_issue('# foobar\n\n') - assert_issue('\n\n') - - -def test_shebang(): - assert not issues('#!\n') - assert not issues('#!/foo\n') - assert not issues('#! python\n') +import parso + + +def issues(code): + grammar = parso.load_grammar() + module = parso.parse(code) + return grammar._get_normalizer_issues(module) + + +def test_eof_newline(): + def assert_issue(code): + found = issues(code) + assert len(found) == 1 + issue, = found + assert issue.code == 292 + + assert not issues('asdf = 1\n') + assert not issues('asdf = 1\r\n') + assert not issues('asdf = 1\r') + assert_issue('asdf = 1') + assert_issue('asdf = 1\n# foo') + assert_issue('# foobar') + assert_issue('') + assert_issue('foo = 1 # comment') + + +def test_eof_blankline(): + def assert_issue(code): + found = issues(code) + assert len(found) == 1 + issue, = found + assert issue.code == 391 + + assert_issue('asdf = 1\n\n') + assert_issue('# foobar\n\n') + assert_issue('\n\n') + + +def test_shebang(): + assert not issues('#!\n') + assert not issues('#!/foo\n') + assert not issues('#! python\n') diff --git a/contrib/python/parso/py3/tests/test_pgen2.py b/contrib/python/parso/py3/tests/test_pgen2.py index 85ccacfb47..04ee7a329a 100644 --- a/contrib/python/parso/py3/tests/test_pgen2.py +++ b/contrib/python/parso/py3/tests/test_pgen2.py @@ -1,357 +1,357 @@ -from textwrap import dedent - -import pytest - -from parso import load_grammar -from parso import ParserSyntaxError -from parso.pgen2 import generate_grammar -from parso.python import tokenize - - -def _parse(code, version=None): - code = dedent(code) + "\n\n" - grammar = load_grammar(version=version) - return grammar.parse(code, error_recovery=False) - - -def _invalid_syntax(code, version=None, **kwargs): - with pytest.raises(ParserSyntaxError): - module = _parse(code, version=version, **kwargs) - # For debugging - print(module.children) - - -def test_formfeed(each_version): - s = "foo\n\x0c\nfoo\n" - t = _parse(s, each_version) - assert t.children[0].children[0].type == 'name' - assert t.children[1].children[0].type == 'name' - s = "1\n\x0c\x0c\n2\n" - t = _parse(s, each_version) - - with pytest.raises(ParserSyntaxError): - s = "\n\x0c2\n" - _parse(s, each_version) - - -def test_matrix_multiplication_operator(works_in_py): - works_in_py.parse("a @ b") - works_in_py.parse("a @= b") - - -def test_yield_from(works_in_py, each_version): - works_in_py.parse("yield from x") - works_in_py.parse("(yield from x) + y") - _invalid_syntax("yield from", each_version) - - -def test_await_expr(works_in_py): - works_in_py.parse("""async def foo(): - await x - """) - - works_in_py.parse("""async def foo(): - - def foo(): pass - - def foo(): pass - - await x - """) - - works_in_py.parse("""async def foo(): return await a""") - - works_in_py.parse("""def foo(): - def foo(): pass - async def foo(): await x - """) - - -@pytest.mark.parametrize( - 'code', [ - "async = 1", - "await = 1", - "def async(): pass", - ] -) -def test_async_var(works_not_in_py, code): - works_not_in_py.parse(code) - - -def test_async_for(works_in_py): - works_in_py.parse("async def foo():\n async for a in b: pass") - - -@pytest.mark.parametrize("body", [ - """[1 async for a in b - ]""", - """[1 async - for a in b - ]""", - """[ - 1 - async for a in b - ]""", - """[ - 1 - async for a - in b - ]""", - """[ - 1 - async - for - a - in - b - ]""", - """ [ - 1 async for a in b - ]""", -]) -def test_async_for_comprehension_newline(works_in_py, body): - # Issue #139 - works_in_py.parse("""async def foo(): - {}""".format(body)) - - -def test_async_with(works_in_py): - works_in_py.parse("async def foo():\n async with a: pass") - - -def test_async_with_invalid(works_in_py): - works_in_py.parse("""def foo():\n async with a: pass""") - - -def test_raise_3x_style_1(each_version): - _parse("raise", each_version) - - -def test_raise_2x_style_2(works_not_in_py): - works_not_in_py.parse("raise E, V") - - -def test_raise_2x_style_3(works_not_in_py): - works_not_in_py.parse("raise E, V, T") - - -def test_raise_2x_style_invalid_1(each_version): - _invalid_syntax("raise E, V, T, Z", version=each_version) - - -def test_raise_3x_style(works_in_py): - works_in_py.parse("raise E1 from E2") - - -def test_raise_3x_style_invalid_1(each_version): - _invalid_syntax("raise E, V from E1", each_version) - - -def test_raise_3x_style_invalid_2(each_version): - _invalid_syntax("raise E from E1, E2", each_version) - - -def test_raise_3x_style_invalid_3(each_version): - _invalid_syntax("raise from E1, E2", each_version) - - -def test_raise_3x_style_invalid_4(each_version): - _invalid_syntax("raise E from", each_version) - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef -def test_annotation_1(works_in_py): - works_in_py.parse("""def f(x) -> list: pass""") - - -def test_annotation_2(works_in_py): - works_in_py.parse("""def f(x:int): pass""") - - -def test_annotation_3(works_in_py): - works_in_py.parse("""def f(*x:str): pass""") - - -def test_annotation_4(works_in_py): - works_in_py.parse("""def f(**x:float): pass""") - - -def test_annotation_5(works_in_py): - works_in_py.parse("""def f(x, y:1+2): pass""") - - -def test_annotation_6(each_version): - _invalid_syntax("""def f(a, (b:1, c:2, d)): pass""", each_version) - - -def test_annotation_7(each_version): - _invalid_syntax("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""", each_version) - - -def test_annotation_8(each_version): - s = """def f(a, (b:1, c:2, d), e:3=4, f=5, - *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" - _invalid_syntax(s, each_version) - - -def test_except_new(each_version): - s = dedent(""" - try: - x - except E as N: - y""") - _parse(s, each_version) - - -def test_except_old(works_not_in_py): - s = dedent(""" - try: - x - except E, N: - y""") - works_not_in_py.parse(s) - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms -def test_set_literal_1(works_in_py): - works_in_py.parse("""x = {'one'}""") - - -def test_set_literal_2(works_in_py): - works_in_py.parse("""x = {'one', 1,}""") - - -def test_set_literal_3(works_in_py): - works_in_py.parse("""x = {'one', 'two', 'three'}""") - - -def test_set_literal_4(works_in_py): - works_in_py.parse("""x = {2, 3, 4,}""") - - -def test_new_octal_notation(each_version): - _parse("""0o7777777777777""", each_version) - _invalid_syntax("""0o7324528887""", each_version) - - -def test_old_octal_notation(works_not_in_py): - works_not_in_py.parse("07") - - -def test_long_notation(works_not_in_py): - works_not_in_py.parse("0xFl") - works_not_in_py.parse("0xFL") - works_not_in_py.parse("0b1l") - works_not_in_py.parse("0B1L") - works_not_in_py.parse("0o7l") - works_not_in_py.parse("0O7L") - works_not_in_py.parse("0l") - works_not_in_py.parse("0L") - works_not_in_py.parse("10l") - works_not_in_py.parse("10L") - - -def test_new_binary_notation(each_version): - _parse("""0b101010""", each_version) - _invalid_syntax("""0b0101021""", each_version) - - -def test_class_new_syntax(works_in_py): - works_in_py.parse("class B(t=7): pass") - works_in_py.parse("class B(t, *args): pass") - works_in_py.parse("class B(t, **kwargs): pass") - works_in_py.parse("class B(t, *args, **kwargs): pass") - works_in_py.parse("class B(t, y=9, *args, **kwargs): pass") - - -def test_parser_idempotency_extended_unpacking(works_in_py): - """A cut-down version of pytree_idempotency.py.""" - works_in_py.parse("a, *b, c = x\n") - works_in_py.parse("[*a, b] = x\n") - works_in_py.parse("(z, *y, w) = m\n") - works_in_py.parse("for *z, m in d: pass\n") - - -def test_multiline_bytes_literals(each_version): - s = """ - md5test(b"\xaa" * 80, - (b"Test Using Larger Than Block-Size Key " - b"and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - _parse(s, each_version) - - -def test_multiline_bytes_tripquote_literals(each_version): - s = ''' - b""" - <?xml version="1.0" encoding="UTF-8"?> - <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN"> - """ - ''' - _parse(s, each_version) - - -def test_ellipsis(works_in_py, each_version): - works_in_py.parse("...") - _parse("[0][...]", version=each_version) - - -def test_dict_unpacking(works_in_py): - works_in_py.parse("{**dict(a=3), foo:2}") - - -def test_multiline_str_literals(each_version): - s = """ - md5test("\xaa" * 80, - ("Test Using Larger Than Block-Size Key " - "and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - _parse(s, each_version) - - -def test_py2_backticks(works_not_in_py): - works_not_in_py.parse("`1`") - - -def test_py2_string_prefixes(works_not_in_py): - works_not_in_py.parse("ur'1'") - works_not_in_py.parse("Ur'1'") - works_not_in_py.parse("UR'1'") - _invalid_syntax("ru'1'", works_not_in_py.version) - - -def py_br(each_version): - _parse('br""', each_version) - - -def test_py3_rb(works_in_py): - works_in_py.parse("rb'1'") - works_in_py.parse("RB'1'") - - -def test_left_recursion(): - with pytest.raises(ValueError, match='left recursion'): - generate_grammar('foo: foo NAME\n', tokenize.PythonTokenTypes) - - -@pytest.mark.parametrize( - 'grammar, error_match', [ - ['foo: bar | baz\nbar: NAME\nbaz: NAME\n', - r"foo is ambiguous.*given a (PythonTokenTypes\.)?NAME.*bar or baz"], - ['''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''', - r"foo is ambiguous.*given a ReservedString\(x\).*bar or baz"], - ['''foo: bar | 'x'\nbar: 'x'\n''', - r"foo is ambiguous.*given a ReservedString\(x\).*bar or foo"], - # An ambiguity with the second (not the first) child of a production - ['outer: "a" [inner] "b" "c"\ninner: "b" "c" [inner]\n', - r"outer is ambiguous.*given a ReservedString\(b\).*inner or outer"], - # An ambiguity hidden by a level of indirection (middle) - ['outer: "a" [middle] "b" "c"\nmiddle: inner\ninner: "b" "c" [inner]\n', - r"outer is ambiguous.*given a ReservedString\(b\).*middle or outer"], - ] -) -def test_ambiguities(grammar, error_match): - with pytest.raises(ValueError, match=error_match): - generate_grammar(grammar, tokenize.PythonTokenTypes) +from textwrap import dedent + +import pytest + +from parso import load_grammar +from parso import ParserSyntaxError +from parso.pgen2 import generate_grammar +from parso.python import tokenize + + +def _parse(code, version=None): + code = dedent(code) + "\n\n" + grammar = load_grammar(version=version) + return grammar.parse(code, error_recovery=False) + + +def _invalid_syntax(code, version=None, **kwargs): + with pytest.raises(ParserSyntaxError): + module = _parse(code, version=version, **kwargs) + # For debugging + print(module.children) + + +def test_formfeed(each_version): + s = "foo\n\x0c\nfoo\n" + t = _parse(s, each_version) + assert t.children[0].children[0].type == 'name' + assert t.children[1].children[0].type == 'name' + s = "1\n\x0c\x0c\n2\n" + t = _parse(s, each_version) + + with pytest.raises(ParserSyntaxError): + s = "\n\x0c2\n" + _parse(s, each_version) + + +def test_matrix_multiplication_operator(works_in_py): + works_in_py.parse("a @ b") + works_in_py.parse("a @= b") + + +def test_yield_from(works_in_py, each_version): + works_in_py.parse("yield from x") + works_in_py.parse("(yield from x) + y") + _invalid_syntax("yield from", each_version) + + +def test_await_expr(works_in_py): + works_in_py.parse("""async def foo(): + await x + """) + + works_in_py.parse("""async def foo(): + + def foo(): pass + + def foo(): pass + + await x + """) + + works_in_py.parse("""async def foo(): return await a""") + + works_in_py.parse("""def foo(): + def foo(): pass + async def foo(): await x + """) + + +@pytest.mark.parametrize( + 'code', [ + "async = 1", + "await = 1", + "def async(): pass", + ] +) +def test_async_var(works_not_in_py, code): + works_not_in_py.parse(code) + + +def test_async_for(works_in_py): + works_in_py.parse("async def foo():\n async for a in b: pass") + + +@pytest.mark.parametrize("body", [ + """[1 async for a in b + ]""", + """[1 async + for a in b + ]""", + """[ + 1 + async for a in b + ]""", + """[ + 1 + async for a + in b + ]""", + """[ + 1 + async + for + a + in + b + ]""", + """ [ + 1 async for a in b + ]""", +]) +def test_async_for_comprehension_newline(works_in_py, body): + # Issue #139 + works_in_py.parse("""async def foo(): + {}""".format(body)) + + +def test_async_with(works_in_py): + works_in_py.parse("async def foo():\n async with a: pass") + + +def test_async_with_invalid(works_in_py): + works_in_py.parse("""def foo():\n async with a: pass""") + + +def test_raise_3x_style_1(each_version): + _parse("raise", each_version) + + +def test_raise_2x_style_2(works_not_in_py): + works_not_in_py.parse("raise E, V") + + +def test_raise_2x_style_3(works_not_in_py): + works_not_in_py.parse("raise E, V, T") + + +def test_raise_2x_style_invalid_1(each_version): + _invalid_syntax("raise E, V, T, Z", version=each_version) + + +def test_raise_3x_style(works_in_py): + works_in_py.parse("raise E1 from E2") + + +def test_raise_3x_style_invalid_1(each_version): + _invalid_syntax("raise E, V from E1", each_version) + + +def test_raise_3x_style_invalid_2(each_version): + _invalid_syntax("raise E from E1, E2", each_version) + + +def test_raise_3x_style_invalid_3(each_version): + _invalid_syntax("raise from E1, E2", each_version) + + +def test_raise_3x_style_invalid_4(each_version): + _invalid_syntax("raise E from", each_version) + + +# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef +def test_annotation_1(works_in_py): + works_in_py.parse("""def f(x) -> list: pass""") + + +def test_annotation_2(works_in_py): + works_in_py.parse("""def f(x:int): pass""") + + +def test_annotation_3(works_in_py): + works_in_py.parse("""def f(*x:str): pass""") + + +def test_annotation_4(works_in_py): + works_in_py.parse("""def f(**x:float): pass""") + + +def test_annotation_5(works_in_py): + works_in_py.parse("""def f(x, y:1+2): pass""") + + +def test_annotation_6(each_version): + _invalid_syntax("""def f(a, (b:1, c:2, d)): pass""", each_version) + + +def test_annotation_7(each_version): + _invalid_syntax("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""", each_version) + + +def test_annotation_8(each_version): + s = """def f(a, (b:1, c:2, d), e:3=4, f=5, + *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" + _invalid_syntax(s, each_version) + + +def test_except_new(each_version): + s = dedent(""" + try: + x + except E as N: + y""") + _parse(s, each_version) + + +def test_except_old(works_not_in_py): + s = dedent(""" + try: + x + except E, N: + y""") + works_not_in_py.parse(s) + + +# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms +def test_set_literal_1(works_in_py): + works_in_py.parse("""x = {'one'}""") + + +def test_set_literal_2(works_in_py): + works_in_py.parse("""x = {'one', 1,}""") + + +def test_set_literal_3(works_in_py): + works_in_py.parse("""x = {'one', 'two', 'three'}""") + + +def test_set_literal_4(works_in_py): + works_in_py.parse("""x = {2, 3, 4,}""") + + +def test_new_octal_notation(each_version): + _parse("""0o7777777777777""", each_version) + _invalid_syntax("""0o7324528887""", each_version) + + +def test_old_octal_notation(works_not_in_py): + works_not_in_py.parse("07") + + +def test_long_notation(works_not_in_py): + works_not_in_py.parse("0xFl") + works_not_in_py.parse("0xFL") + works_not_in_py.parse("0b1l") + works_not_in_py.parse("0B1L") + works_not_in_py.parse("0o7l") + works_not_in_py.parse("0O7L") + works_not_in_py.parse("0l") + works_not_in_py.parse("0L") + works_not_in_py.parse("10l") + works_not_in_py.parse("10L") + + +def test_new_binary_notation(each_version): + _parse("""0b101010""", each_version) + _invalid_syntax("""0b0101021""", each_version) + + +def test_class_new_syntax(works_in_py): + works_in_py.parse("class B(t=7): pass") + works_in_py.parse("class B(t, *args): pass") + works_in_py.parse("class B(t, **kwargs): pass") + works_in_py.parse("class B(t, *args, **kwargs): pass") + works_in_py.parse("class B(t, y=9, *args, **kwargs): pass") + + +def test_parser_idempotency_extended_unpacking(works_in_py): + """A cut-down version of pytree_idempotency.py.""" + works_in_py.parse("a, *b, c = x\n") + works_in_py.parse("[*a, b] = x\n") + works_in_py.parse("(z, *y, w) = m\n") + works_in_py.parse("for *z, m in d: pass\n") + + +def test_multiline_bytes_literals(each_version): + s = """ + md5test(b"\xaa" * 80, + (b"Test Using Larger Than Block-Size Key " + b"and Larger Than One Block-Size Data"), + "6f630fad67cda0ee1fb1f562db3aa53e") + """ + _parse(s, each_version) + + +def test_multiline_bytes_tripquote_literals(each_version): + s = ''' + b""" + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN"> + """ + ''' + _parse(s, each_version) + + +def test_ellipsis(works_in_py, each_version): + works_in_py.parse("...") + _parse("[0][...]", version=each_version) + + +def test_dict_unpacking(works_in_py): + works_in_py.parse("{**dict(a=3), foo:2}") + + +def test_multiline_str_literals(each_version): + s = """ + md5test("\xaa" * 80, + ("Test Using Larger Than Block-Size Key " + "and Larger Than One Block-Size Data"), + "6f630fad67cda0ee1fb1f562db3aa53e") + """ + _parse(s, each_version) + + +def test_py2_backticks(works_not_in_py): + works_not_in_py.parse("`1`") + + +def test_py2_string_prefixes(works_not_in_py): + works_not_in_py.parse("ur'1'") + works_not_in_py.parse("Ur'1'") + works_not_in_py.parse("UR'1'") + _invalid_syntax("ru'1'", works_not_in_py.version) + + +def py_br(each_version): + _parse('br""', each_version) + + +def test_py3_rb(works_in_py): + works_in_py.parse("rb'1'") + works_in_py.parse("RB'1'") + + +def test_left_recursion(): + with pytest.raises(ValueError, match='left recursion'): + generate_grammar('foo: foo NAME\n', tokenize.PythonTokenTypes) + + +@pytest.mark.parametrize( + 'grammar, error_match', [ + ['foo: bar | baz\nbar: NAME\nbaz: NAME\n', + r"foo is ambiguous.*given a (PythonTokenTypes\.)?NAME.*bar or baz"], + ['''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''', + r"foo is ambiguous.*given a ReservedString\(x\).*bar or baz"], + ['''foo: bar | 'x'\nbar: 'x'\n''', + r"foo is ambiguous.*given a ReservedString\(x\).*bar or foo"], + # An ambiguity with the second (not the first) child of a production + ['outer: "a" [inner] "b" "c"\ninner: "b" "c" [inner]\n', + r"outer is ambiguous.*given a ReservedString\(b\).*inner or outer"], + # An ambiguity hidden by a level of indirection (middle) + ['outer: "a" [middle] "b" "c"\nmiddle: inner\ninner: "b" "c" [inner]\n', + r"outer is ambiguous.*given a ReservedString\(b\).*middle or outer"], + ] +) +def test_ambiguities(grammar, error_match): + with pytest.raises(ValueError, match=error_match): + generate_grammar(grammar, tokenize.PythonTokenTypes) diff --git a/contrib/python/parso/py3/tests/test_prefix.py b/contrib/python/parso/py3/tests/test_prefix.py index 58c1dcf98f..e2b9b511bc 100644 --- a/contrib/python/parso/py3/tests/test_prefix.py +++ b/contrib/python/parso/py3/tests/test_prefix.py @@ -1,75 +1,75 @@ -from itertools import zip_longest -from codecs import BOM_UTF8 - -import pytest - -import parso - -unicode_bom = BOM_UTF8.decode('utf-8') - - -@pytest.mark.parametrize(('string', 'tokens'), [ - ('', ['']), - ('#', ['#', '']), - (' # ', ['# ', '']), - (' # \n', ['# ', '\n', '']), - (' # \f\n', ['# ', '\f', '\n', '']), - (' \n', ['\n', '']), - (' \n ', ['\n', ' ']), - (' \f ', ['\f', ' ']), - (' \f ', ['\f', ' ']), - (' \r\n', ['\r\n', '']), - (' \r', ['\r', '']), - ('\\\n', ['\\\n', '']), - ('\\\r\n', ['\\\r\n', '']), - ('\t\t\n\t', ['\n', '\t']), -]) -def test_simple_prefix_splitting(string, tokens): - tree = parso.parse(string) - leaf = tree.children[0] - assert leaf.type == 'endmarker' - - parsed_tokens = list(leaf._split_prefix()) - start_pos = (1, 0) - for pt, expected in zip_longest(parsed_tokens, tokens): - assert pt.value == expected - - # Calculate the estimated end_pos - if expected.endswith('\n') or expected.endswith('\r'): - end_pos = start_pos[0] + 1, 0 - else: - end_pos = start_pos[0], start_pos[1] + len(expected) + len(pt.spacing) - - # assert start_pos == pt.start_pos - assert end_pos == pt.end_pos - start_pos = end_pos - - -@pytest.mark.parametrize(('string', 'types'), [ - ('# ', ['comment', 'spacing']), - ('\r\n', ['newline', 'spacing']), - ('\f', ['formfeed', 'spacing']), - ('\\\n', ['backslash', 'spacing']), - (' \t', ['spacing']), - (' \t ', ['spacing']), - (unicode_bom + ' # ', ['bom', 'comment', 'spacing']), -]) -def test_prefix_splitting_types(string, types): - tree = parso.parse(string) - leaf = tree.children[0] - assert leaf.type == 'endmarker' - parsed_tokens = list(leaf._split_prefix()) - assert [t.type for t in parsed_tokens] == types - - -def test_utf8_bom(): - tree = parso.parse(unicode_bom + 'a = 1') - expr_stmt = tree.children[0] - assert expr_stmt.start_pos == (1, 0) - - tree = parso.parse(unicode_bom + '\n') - endmarker = tree.children[0] - parts = list(endmarker._split_prefix()) - assert [p.type for p in parts] == ['bom', 'newline', 'spacing'] - assert [p.start_pos for p in parts] == [(1, 0), (1, 0), (2, 0)] - assert [p.end_pos for p in parts] == [(1, 0), (2, 0), (2, 0)] +from itertools import zip_longest +from codecs import BOM_UTF8 + +import pytest + +import parso + +unicode_bom = BOM_UTF8.decode('utf-8') + + +@pytest.mark.parametrize(('string', 'tokens'), [ + ('', ['']), + ('#', ['#', '']), + (' # ', ['# ', '']), + (' # \n', ['# ', '\n', '']), + (' # \f\n', ['# ', '\f', '\n', '']), + (' \n', ['\n', '']), + (' \n ', ['\n', ' ']), + (' \f ', ['\f', ' ']), + (' \f ', ['\f', ' ']), + (' \r\n', ['\r\n', '']), + (' \r', ['\r', '']), + ('\\\n', ['\\\n', '']), + ('\\\r\n', ['\\\r\n', '']), + ('\t\t\n\t', ['\n', '\t']), +]) +def test_simple_prefix_splitting(string, tokens): + tree = parso.parse(string) + leaf = tree.children[0] + assert leaf.type == 'endmarker' + + parsed_tokens = list(leaf._split_prefix()) + start_pos = (1, 0) + for pt, expected in zip_longest(parsed_tokens, tokens): + assert pt.value == expected + + # Calculate the estimated end_pos + if expected.endswith('\n') or expected.endswith('\r'): + end_pos = start_pos[0] + 1, 0 + else: + end_pos = start_pos[0], start_pos[1] + len(expected) + len(pt.spacing) + + # assert start_pos == pt.start_pos + assert end_pos == pt.end_pos + start_pos = end_pos + + +@pytest.mark.parametrize(('string', 'types'), [ + ('# ', ['comment', 'spacing']), + ('\r\n', ['newline', 'spacing']), + ('\f', ['formfeed', 'spacing']), + ('\\\n', ['backslash', 'spacing']), + (' \t', ['spacing']), + (' \t ', ['spacing']), + (unicode_bom + ' # ', ['bom', 'comment', 'spacing']), +]) +def test_prefix_splitting_types(string, types): + tree = parso.parse(string) + leaf = tree.children[0] + assert leaf.type == 'endmarker' + parsed_tokens = list(leaf._split_prefix()) + assert [t.type for t in parsed_tokens] == types + + +def test_utf8_bom(): + tree = parso.parse(unicode_bom + 'a = 1') + expr_stmt = tree.children[0] + assert expr_stmt.start_pos == (1, 0) + + tree = parso.parse(unicode_bom + '\n') + endmarker = tree.children[0] + parts = list(endmarker._split_prefix()) + assert [p.type for p in parts] == ['bom', 'newline', 'spacing'] + assert [p.start_pos for p in parts] == [(1, 0), (1, 0), (2, 0)] + assert [p.end_pos for p in parts] == [(1, 0), (2, 0), (2, 0)] diff --git a/contrib/python/parso/py3/tests/test_python_errors.py b/contrib/python/parso/py3/tests/test_python_errors.py index adf5f06931..c16e9b8663 100644 --- a/contrib/python/parso/py3/tests/test_python_errors.py +++ b/contrib/python/parso/py3/tests/test_python_errors.py @@ -1,510 +1,510 @@ -""" -Testing if parso finds syntax errors and indentation errors. -""" -import sys -import warnings - -import pytest - -import parso - -from textwrap import dedent -from parso._compatibility import is_pypy -from .failing_examples import FAILING_EXAMPLES, indent, build_nested - - -if is_pypy: - # The errors in PyPy might be different. Just skip the module for now. - pytestmark = pytest.mark.skip() - - -def _get_error_list(code, version=None): - grammar = parso.load_grammar(version=version) - tree = grammar.parse(code) - return list(grammar.iter_errors(tree)) - - -def assert_comparison(code, error_code, positions): - errors = [(error.start_pos, error.code) for error in _get_error_list(code)] - assert [(pos, error_code) for pos in positions] == errors - - -@pytest.mark.parametrize('code', FAILING_EXAMPLES) -def test_python_exception_matches(code): - wanted, line_nr = _get_actual_exception(code) - - errors = _get_error_list(code) - actual = None - if errors: - error, = errors - actual = error.message - assert actual in wanted - # Somehow in Python2.7 the SyntaxError().lineno is sometimes None - assert line_nr is None or line_nr == error.start_pos[0] - - -def test_non_async_in_async(): - """ - This example doesn't work with FAILING_EXAMPLES, because the line numbers - are not always the same / incorrect in Python 3.8. - """ - # Raises multiple errors in previous versions. - code = 'async def foo():\n def nofoo():[x async for x in []]' - wanted, line_nr = _get_actual_exception(code) - - errors = _get_error_list(code) - if errors: - error, = errors - actual = error.message - assert actual in wanted - if sys.version_info[:2] not in ((3, 8), (3, 9)): - assert line_nr == error.start_pos[0] - else: - assert line_nr == 0 # For whatever reason this is zero in Python 3.8/3.9 - - -@pytest.mark.parametrize( - ('code', 'positions'), [ - ('1 +', [(1, 3)]), - ('1 +\n', [(1, 3)]), - ('1 +\n2 +', [(1, 3), (2, 3)]), - ('x + 2', []), - ('[\n', [(2, 0)]), - ('[\ndef x(): pass', [(2, 0)]), - ('[\nif 1: pass', [(2, 0)]), - ('1+?', [(1, 2)]), - ('?', [(1, 0)]), - ('??', [(1, 0)]), - ('? ?', [(1, 0)]), - ('?\n?', [(1, 0), (2, 0)]), - ('? * ?', [(1, 0)]), - ('1 + * * 2', [(1, 4)]), - ('?\n1\n?', [(1, 0), (3, 0)]), - ] -) -def test_syntax_errors(code, positions): - assert_comparison(code, 901, positions) - - -@pytest.mark.parametrize( - ('code', 'positions'), [ - (' 1', [(1, 0)]), - ('def x():\n 1\n 2', [(3, 0)]), - ('def x():\n 1\n 2', [(3, 0)]), - ('def x():\n1', [(2, 0)]), - ] -) -def test_indentation_errors(code, positions): - assert_comparison(code, 903, positions) - - -def _get_actual_exception(code): - with warnings.catch_warnings(): - # We don't care about warnings where locals/globals misbehave here. - # It's as simple as either an error or not. - warnings.filterwarnings('ignore', category=SyntaxWarning) - try: - compile(code, '<unknown>', 'exec') - except (SyntaxError, IndentationError) as e: - wanted = e.__class__.__name__ + ': ' + e.msg - line_nr = e.lineno - except ValueError as e: - # The ValueError comes from byte literals in Python 2 like '\x' - # that are oddly enough not SyntaxErrors. - wanted = 'SyntaxError: (value error) ' + str(e) - line_nr = None - else: - assert False, "The piece of code should raise an exception." - - # SyntaxError - if wanted == 'SyntaxError: assignment to keyword': - return [wanted, "SyntaxError: can't assign to keyword", - 'SyntaxError: cannot assign to __debug__'], line_nr - elif wanted == 'SyntaxError: f-string: unterminated string': - wanted = 'SyntaxError: EOL while scanning string literal' - elif wanted == 'SyntaxError: f-string expression part cannot include a backslash': - return [ - wanted, - "SyntaxError: EOL while scanning string literal", - "SyntaxError: unexpected character after line continuation character", - ], line_nr - elif wanted == "SyntaxError: f-string: expecting '}'": - wanted = 'SyntaxError: EOL while scanning string literal' - elif wanted == 'SyntaxError: f-string: empty expression not allowed': - wanted = 'SyntaxError: invalid syntax' - elif wanted == "SyntaxError: f-string expression part cannot include '#'": - wanted = 'SyntaxError: invalid syntax' - elif wanted == "SyntaxError: f-string: single '}' is not allowed": - wanted = 'SyntaxError: invalid syntax' - return [wanted], line_nr - - -def test_default_except_error_postition(): - # For this error the position seemed to be one line off in Python < 3.10, - # but that doesn't really matter. - code = 'try: pass\nexcept: pass\nexcept X: pass' - wanted, line_nr = _get_actual_exception(code) - error, = _get_error_list(code) - assert error.message in wanted - if sys.version_info[:2] >= (3, 10): - assert line_nr == error.start_pos[0] - else: - assert line_nr != error.start_pos[0] - # I think this is the better position. - assert error.start_pos[0] == 2 - - -def test_statically_nested_blocks(): - def build(code, depth): - if depth == 0: - return code - - new_code = 'if 1:\n' + indent(code) - return build(new_code, depth - 1) - - def get_error(depth, add_func=False): - code = build('foo', depth) - if add_func: - code = 'def bar():\n' + indent(code) - errors = _get_error_list(code) - if errors: - assert errors[0].message == 'SyntaxError: too many statically nested blocks' - return errors[0] - return None - - assert get_error(19) is None - assert get_error(19, add_func=True) is None - - assert get_error(20) - assert get_error(20, add_func=True) - - -def test_future_import_first(): - def is_issue(code, *args, **kwargs): - code = code % args - return bool(_get_error_list(code, **kwargs)) - - i1 = 'from __future__ import division' - i2 = 'from __future__ import absolute_import' - i3 = 'from __future__ import annotations' - assert not is_issue(i1) - assert not is_issue(i1 + ';' + i2) - assert not is_issue(i1 + '\n' + i2) - assert not is_issue('"";' + i1) - assert not is_issue('"";' + i1) - assert not is_issue('""\n' + i1) - assert not is_issue('""\n%s\n%s', i1, i2) - assert not is_issue('""\n%s;%s', i1, i2) - assert not is_issue('"";%s;%s ', i1, i2) - assert not is_issue('"";%s\n%s ', i1, i2) - assert not is_issue(i3, version="3.7") - assert is_issue(i3, version="3.6") - assert is_issue('1;' + i1) - assert is_issue('1\n' + i1) - assert is_issue('"";1\n' + i1) - assert is_issue('""\n%s\nfrom x import a\n%s', i1, i2) - assert is_issue('%s\n""\n%s', i1, i2) - - -def test_named_argument_issues(works_not_in_py): - message = works_not_in_py.get_error_message('def foo(*, **dict): pass') - message = works_not_in_py.get_error_message('def foo(*): pass') - if works_not_in_py.version.startswith('2'): - assert message == 'SyntaxError: invalid syntax' - else: - assert message == 'SyntaxError: named arguments must follow bare *' - - works_not_in_py.assert_no_error_in_passing('def foo(*, name): pass') - works_not_in_py.assert_no_error_in_passing('def foo(bar, *, name=1): pass') - works_not_in_py.assert_no_error_in_passing('def foo(bar, *, name=1, **dct): pass') - - -def test_escape_decode_literals(each_version): - """ - We are using internal functions to assure that unicode/bytes escaping is - without syntax errors. Here we make a bit of quality assurance that this - works through versions, because the internal function might change over - time. - """ - def get_msg(end, to=1): - base = "SyntaxError: (unicode error) 'unicodeescape' " \ - "codec can't decode bytes in position 0-%s: " % to - return base + end - - def get_msgs(escape): - return (get_msg('end of string in escape sequence'), - get_msg(r"truncated %s escape" % escape)) - - error, = _get_error_list(r'u"\x"', version=each_version) - assert error.message in get_msgs(r'\xXX') - - error, = _get_error_list(r'u"\u"', version=each_version) - assert error.message in get_msgs(r'\uXXXX') - - error, = _get_error_list(r'u"\U"', version=each_version) - assert error.message in get_msgs(r'\UXXXXXXXX') - - error, = _get_error_list(r'u"\N{}"', version=each_version) - assert error.message == get_msg(r'malformed \N character escape', to=2) - - error, = _get_error_list(r'u"\N{foo}"', version=each_version) - assert error.message == get_msg(r'unknown Unicode character name', to=6) - - # Finally bytes. - error, = _get_error_list(r'b"\x"', version=each_version) - wanted = r'SyntaxError: (value error) invalid \x escape at position 0' - assert error.message == wanted - - -def test_too_many_levels_of_indentation(): - assert not _get_error_list(build_nested('pass', 99)) - assert _get_error_list(build_nested('pass', 100)) - base = 'def x():\n if x:\n' - assert not _get_error_list(build_nested('pass', 49, base=base)) - assert _get_error_list(build_nested('pass', 50, base=base)) - - -def test_paren_kwarg(): - assert _get_error_list("print((sep)=seperator)", version="3.8") - assert not _get_error_list("print((sep)=seperator)", version="3.7") - - -@pytest.mark.parametrize( - 'code', [ - "f'{*args,}'", - r'f"\""', - r'f"\\\""', - r'fr"\""', - r'fr"\\\""', - r"print(f'Some {x:.2f} and some {y}')", - # Unparenthesized yield expression - 'def foo(): return f"{yield 1}"', - ] -) -def test_valid_fstrings(code): - assert not _get_error_list(code, version='3.6') - - -@pytest.mark.parametrize( - 'code', [ - 'a = (b := 1)', - '[x4 := x ** 5 for x in range(7)]', - '[total := total + v for v in range(10)]', - 'while chunk := file.read(2):\n pass', - 'numbers = [y := math.factorial(x), y**2, y**3]', - '{(a:="a"): (b:=1)}', - '{(y:=1): 2 for x in range(5)}', - 'a[(b:=0)]', - 'a[(b:=0, c:=0)]', - 'a[(b:=0):1:2]', - ] -) -def test_valid_namedexpr(code): - assert not _get_error_list(code, version='3.8') - - -@pytest.mark.parametrize( - 'code', [ - '{x := 1, 2, 3}', - '{x4 := x ** 5 for x in range(7)}', - ] -) -def test_valid_namedexpr_set(code): - assert not _get_error_list(code, version='3.9') - - -@pytest.mark.parametrize( - 'code', [ - 'a[b:=0]', - 'a[b:=0, c:=0]', - ] -) -def test_valid_namedexpr_index(code): - assert not _get_error_list(code, version='3.10') - - -@pytest.mark.parametrize( - ('code', 'message'), [ - ("f'{1+}'", ('invalid syntax')), - (r'f"\"', ('invalid syntax')), - (r'fr"\"', ('invalid syntax')), - ] -) -def test_invalid_fstrings(code, message): - """ - Some fstring errors are handled differntly in 3.6 and other versions. - Therefore check specifically for these errors here. - """ - error, = _get_error_list(code, version='3.6') - assert message in error.message - - -@pytest.mark.parametrize( - 'code', [ - "from foo import (\nbar,\n rab,\n)", - "from foo import (bar, rab, )", - ] -) -def test_trailing_comma(code): - errors = _get_error_list(code) - assert not errors - - -def test_continue_in_finally(): - code = dedent('''\ - for a in [1]: - try: - pass - finally: - continue - ''') - assert not _get_error_list(code, version="3.8") - assert _get_error_list(code, version="3.7") - - -@pytest.mark.parametrize( - 'template', [ - "a, b, {target}, c = d", - "a, b, *{target}, c = d", - "(a, *{target}), c = d", - "for x, {target} in y: pass", - "for x, q, {target} in y: pass", - "for x, q, *{target} in y: pass", - "for (x, *{target}), q in y: pass", - ] -) -@pytest.mark.parametrize( - 'target', [ - "True", - "False", - "None", - "__debug__" - ] -) -def test_forbidden_name(template, target): - assert _get_error_list(template.format(target=target), version="3") - - -def test_repeated_kwarg(): - # python 3.9+ shows which argument is repeated - assert ( - _get_error_list("f(q=1, q=2)", version="3.8")[0].message - == "SyntaxError: keyword argument repeated" - ) - assert ( - _get_error_list("f(q=1, q=2)", version="3.9")[0].message - == "SyntaxError: keyword argument repeated: q" - ) - - -@pytest.mark.parametrize( - ('source', 'no_errors'), [ - ('a(a for a in b,)', False), - ('a(a for a in b, a)', False), - ('a(a, a for a in b)', False), - ('a(a, b, a for a in b, c, d)', False), - ('a(a for a in b)', True), - ('a((a for a in b), c)', True), - ('a(c, (a for a in b))', True), - ('a(a, b, (a for a in b), c, d)', True), - ] -) -def test_unparenthesized_genexp(source, no_errors): - assert bool(_get_error_list(source)) ^ no_errors - - -@pytest.mark.parametrize( - ('source', 'no_errors'), [ - ('*x = 2', False), - ('(*y) = 1', False), - ('((*z)) = 1', False), - ('*a,', True), - ('*a, = 1', True), - ('(*a,)', True), - ('(*a,) = 1', True), - ('[*a]', True), - ('[*a] = 1', True), - ('a, *b', True), - ('a, *b = 1', True), - ('a, *b, c', True), - ('a, *b, c = 1', True), - ('a, (*b, c), d', True), - ('a, (*b, c), d = 1', True), - ('*a.b,', True), - ('*a.b, = 1', True), - ('*a[b],', True), - ('*a[b], = 1', True), - ('*a[b::], c', True), - ('*a[b::], c = 1', True), - ('(a, *[b, c])', True), - ('(a, *[b, c]) = 1', True), - ('[a, *(b, [*c])]', True), - ('[a, *(b, [*c])] = 1', True), - ('[*(1,2,3)]', True), - ('{*(1,2,3)}', True), - ('[*(1,2,3),]', True), - ('[*(1,2,3), *(4,5,6)]', True), - ('[0, *(1,2,3)]', True), - ('{*(1,2,3),}', True), - ('{*(1,2,3), *(4,5,6)}', True), - ('{0, *(4,5,6)}', True) - ] -) -def test_starred_expr(source, no_errors): - assert bool(_get_error_list(source, version="3")) ^ no_errors - - -@pytest.mark.parametrize( - 'code', [ - 'a, (*b), c', - 'a, (*b), c = 1', - 'a, ((*b)), c', - 'a, ((*b)), c = 1', - ] -) -def test_parenthesized_single_starred_expr(code): - assert not _get_error_list(code, version='3.8') - assert _get_error_list(code, version='3.9') - - -@pytest.mark.parametrize( - 'code', [ - '() = ()', - '() = []', - '[] = ()', - '[] = []', - ] -) -def test_valid_empty_assignment(code): - assert not _get_error_list(code) - - -@pytest.mark.parametrize( - 'code', [ - 'del ()', - 'del []', - 'del x', - 'del x,', - 'del x, y', - 'del (x, y)', - 'del [x, y]', - 'del (x, [y, z])', - 'del x.y, x[y]', - 'del f(x)[y::]', - 'del x[[*y]]', - 'del x[[*y]::]', - ] -) -def test_valid_del(code): - assert not _get_error_list(code) - - -@pytest.mark.parametrize( - ('source', 'version', 'no_errors'), [ - ('[x for x in range(10) if lambda: 1]', '3.8', True), - ('[x for x in range(10) if lambda: 1]', '3.9', False), - ('[x for x in range(10) if (lambda: 1)]', '3.9', True), - ] -) -def test_lambda_in_comp_if(source, version, no_errors): - assert bool(_get_error_list(source, version=version)) ^ no_errors +""" +Testing if parso finds syntax errors and indentation errors. +""" +import sys +import warnings + +import pytest + +import parso + +from textwrap import dedent +from parso._compatibility import is_pypy +from .failing_examples import FAILING_EXAMPLES, indent, build_nested + + +if is_pypy: + # The errors in PyPy might be different. Just skip the module for now. + pytestmark = pytest.mark.skip() + + +def _get_error_list(code, version=None): + grammar = parso.load_grammar(version=version) + tree = grammar.parse(code) + return list(grammar.iter_errors(tree)) + + +def assert_comparison(code, error_code, positions): + errors = [(error.start_pos, error.code) for error in _get_error_list(code)] + assert [(pos, error_code) for pos in positions] == errors + + +@pytest.mark.parametrize('code', FAILING_EXAMPLES) +def test_python_exception_matches(code): + wanted, line_nr = _get_actual_exception(code) + + errors = _get_error_list(code) + actual = None + if errors: + error, = errors + actual = error.message + assert actual in wanted + # Somehow in Python2.7 the SyntaxError().lineno is sometimes None + assert line_nr is None or line_nr == error.start_pos[0] + + +def test_non_async_in_async(): + """ + This example doesn't work with FAILING_EXAMPLES, because the line numbers + are not always the same / incorrect in Python 3.8. + """ + # Raises multiple errors in previous versions. + code = 'async def foo():\n def nofoo():[x async for x in []]' + wanted, line_nr = _get_actual_exception(code) + + errors = _get_error_list(code) + if errors: + error, = errors + actual = error.message + assert actual in wanted + if sys.version_info[:2] not in ((3, 8), (3, 9)): + assert line_nr == error.start_pos[0] + else: + assert line_nr == 0 # For whatever reason this is zero in Python 3.8/3.9 + + +@pytest.mark.parametrize( + ('code', 'positions'), [ + ('1 +', [(1, 3)]), + ('1 +\n', [(1, 3)]), + ('1 +\n2 +', [(1, 3), (2, 3)]), + ('x + 2', []), + ('[\n', [(2, 0)]), + ('[\ndef x(): pass', [(2, 0)]), + ('[\nif 1: pass', [(2, 0)]), + ('1+?', [(1, 2)]), + ('?', [(1, 0)]), + ('??', [(1, 0)]), + ('? ?', [(1, 0)]), + ('?\n?', [(1, 0), (2, 0)]), + ('? * ?', [(1, 0)]), + ('1 + * * 2', [(1, 4)]), + ('?\n1\n?', [(1, 0), (3, 0)]), + ] +) +def test_syntax_errors(code, positions): + assert_comparison(code, 901, positions) + + +@pytest.mark.parametrize( + ('code', 'positions'), [ + (' 1', [(1, 0)]), + ('def x():\n 1\n 2', [(3, 0)]), + ('def x():\n 1\n 2', [(3, 0)]), + ('def x():\n1', [(2, 0)]), + ] +) +def test_indentation_errors(code, positions): + assert_comparison(code, 903, positions) + + +def _get_actual_exception(code): + with warnings.catch_warnings(): + # We don't care about warnings where locals/globals misbehave here. + # It's as simple as either an error or not. + warnings.filterwarnings('ignore', category=SyntaxWarning) + try: + compile(code, '<unknown>', 'exec') + except (SyntaxError, IndentationError) as e: + wanted = e.__class__.__name__ + ': ' + e.msg + line_nr = e.lineno + except ValueError as e: + # The ValueError comes from byte literals in Python 2 like '\x' + # that are oddly enough not SyntaxErrors. + wanted = 'SyntaxError: (value error) ' + str(e) + line_nr = None + else: + assert False, "The piece of code should raise an exception." + + # SyntaxError + if wanted == 'SyntaxError: assignment to keyword': + return [wanted, "SyntaxError: can't assign to keyword", + 'SyntaxError: cannot assign to __debug__'], line_nr + elif wanted == 'SyntaxError: f-string: unterminated string': + wanted = 'SyntaxError: EOL while scanning string literal' + elif wanted == 'SyntaxError: f-string expression part cannot include a backslash': + return [ + wanted, + "SyntaxError: EOL while scanning string literal", + "SyntaxError: unexpected character after line continuation character", + ], line_nr + elif wanted == "SyntaxError: f-string: expecting '}'": + wanted = 'SyntaxError: EOL while scanning string literal' + elif wanted == 'SyntaxError: f-string: empty expression not allowed': + wanted = 'SyntaxError: invalid syntax' + elif wanted == "SyntaxError: f-string expression part cannot include '#'": + wanted = 'SyntaxError: invalid syntax' + elif wanted == "SyntaxError: f-string: single '}' is not allowed": + wanted = 'SyntaxError: invalid syntax' + return [wanted], line_nr + + +def test_default_except_error_postition(): + # For this error the position seemed to be one line off in Python < 3.10, + # but that doesn't really matter. + code = 'try: pass\nexcept: pass\nexcept X: pass' + wanted, line_nr = _get_actual_exception(code) + error, = _get_error_list(code) + assert error.message in wanted + if sys.version_info[:2] >= (3, 10): + assert line_nr == error.start_pos[0] + else: + assert line_nr != error.start_pos[0] + # I think this is the better position. + assert error.start_pos[0] == 2 + + +def test_statically_nested_blocks(): + def build(code, depth): + if depth == 0: + return code + + new_code = 'if 1:\n' + indent(code) + return build(new_code, depth - 1) + + def get_error(depth, add_func=False): + code = build('foo', depth) + if add_func: + code = 'def bar():\n' + indent(code) + errors = _get_error_list(code) + if errors: + assert errors[0].message == 'SyntaxError: too many statically nested blocks' + return errors[0] + return None + + assert get_error(19) is None + assert get_error(19, add_func=True) is None + + assert get_error(20) + assert get_error(20, add_func=True) + + +def test_future_import_first(): + def is_issue(code, *args, **kwargs): + code = code % args + return bool(_get_error_list(code, **kwargs)) + + i1 = 'from __future__ import division' + i2 = 'from __future__ import absolute_import' + i3 = 'from __future__ import annotations' + assert not is_issue(i1) + assert not is_issue(i1 + ';' + i2) + assert not is_issue(i1 + '\n' + i2) + assert not is_issue('"";' + i1) + assert not is_issue('"";' + i1) + assert not is_issue('""\n' + i1) + assert not is_issue('""\n%s\n%s', i1, i2) + assert not is_issue('""\n%s;%s', i1, i2) + assert not is_issue('"";%s;%s ', i1, i2) + assert not is_issue('"";%s\n%s ', i1, i2) + assert not is_issue(i3, version="3.7") + assert is_issue(i3, version="3.6") + assert is_issue('1;' + i1) + assert is_issue('1\n' + i1) + assert is_issue('"";1\n' + i1) + assert is_issue('""\n%s\nfrom x import a\n%s', i1, i2) + assert is_issue('%s\n""\n%s', i1, i2) + + +def test_named_argument_issues(works_not_in_py): + message = works_not_in_py.get_error_message('def foo(*, **dict): pass') + message = works_not_in_py.get_error_message('def foo(*): pass') + if works_not_in_py.version.startswith('2'): + assert message == 'SyntaxError: invalid syntax' + else: + assert message == 'SyntaxError: named arguments must follow bare *' + + works_not_in_py.assert_no_error_in_passing('def foo(*, name): pass') + works_not_in_py.assert_no_error_in_passing('def foo(bar, *, name=1): pass') + works_not_in_py.assert_no_error_in_passing('def foo(bar, *, name=1, **dct): pass') + + +def test_escape_decode_literals(each_version): + """ + We are using internal functions to assure that unicode/bytes escaping is + without syntax errors. Here we make a bit of quality assurance that this + works through versions, because the internal function might change over + time. + """ + def get_msg(end, to=1): + base = "SyntaxError: (unicode error) 'unicodeescape' " \ + "codec can't decode bytes in position 0-%s: " % to + return base + end + + def get_msgs(escape): + return (get_msg('end of string in escape sequence'), + get_msg(r"truncated %s escape" % escape)) + + error, = _get_error_list(r'u"\x"', version=each_version) + assert error.message in get_msgs(r'\xXX') + + error, = _get_error_list(r'u"\u"', version=each_version) + assert error.message in get_msgs(r'\uXXXX') + + error, = _get_error_list(r'u"\U"', version=each_version) + assert error.message in get_msgs(r'\UXXXXXXXX') + + error, = _get_error_list(r'u"\N{}"', version=each_version) + assert error.message == get_msg(r'malformed \N character escape', to=2) + + error, = _get_error_list(r'u"\N{foo}"', version=each_version) + assert error.message == get_msg(r'unknown Unicode character name', to=6) + + # Finally bytes. + error, = _get_error_list(r'b"\x"', version=each_version) + wanted = r'SyntaxError: (value error) invalid \x escape at position 0' + assert error.message == wanted + + +def test_too_many_levels_of_indentation(): + assert not _get_error_list(build_nested('pass', 99)) + assert _get_error_list(build_nested('pass', 100)) + base = 'def x():\n if x:\n' + assert not _get_error_list(build_nested('pass', 49, base=base)) + assert _get_error_list(build_nested('pass', 50, base=base)) + + +def test_paren_kwarg(): + assert _get_error_list("print((sep)=seperator)", version="3.8") + assert not _get_error_list("print((sep)=seperator)", version="3.7") + + +@pytest.mark.parametrize( + 'code', [ + "f'{*args,}'", + r'f"\""', + r'f"\\\""', + r'fr"\""', + r'fr"\\\""', + r"print(f'Some {x:.2f} and some {y}')", + # Unparenthesized yield expression + 'def foo(): return f"{yield 1}"', + ] +) +def test_valid_fstrings(code): + assert not _get_error_list(code, version='3.6') + + +@pytest.mark.parametrize( + 'code', [ + 'a = (b := 1)', + '[x4 := x ** 5 for x in range(7)]', + '[total := total + v for v in range(10)]', + 'while chunk := file.read(2):\n pass', + 'numbers = [y := math.factorial(x), y**2, y**3]', + '{(a:="a"): (b:=1)}', + '{(y:=1): 2 for x in range(5)}', + 'a[(b:=0)]', + 'a[(b:=0, c:=0)]', + 'a[(b:=0):1:2]', + ] +) +def test_valid_namedexpr(code): + assert not _get_error_list(code, version='3.8') + + +@pytest.mark.parametrize( + 'code', [ + '{x := 1, 2, 3}', + '{x4 := x ** 5 for x in range(7)}', + ] +) +def test_valid_namedexpr_set(code): + assert not _get_error_list(code, version='3.9') + + +@pytest.mark.parametrize( + 'code', [ + 'a[b:=0]', + 'a[b:=0, c:=0]', + ] +) +def test_valid_namedexpr_index(code): + assert not _get_error_list(code, version='3.10') + + +@pytest.mark.parametrize( + ('code', 'message'), [ + ("f'{1+}'", ('invalid syntax')), + (r'f"\"', ('invalid syntax')), + (r'fr"\"', ('invalid syntax')), + ] +) +def test_invalid_fstrings(code, message): + """ + Some fstring errors are handled differntly in 3.6 and other versions. + Therefore check specifically for these errors here. + """ + error, = _get_error_list(code, version='3.6') + assert message in error.message + + +@pytest.mark.parametrize( + 'code', [ + "from foo import (\nbar,\n rab,\n)", + "from foo import (bar, rab, )", + ] +) +def test_trailing_comma(code): + errors = _get_error_list(code) + assert not errors + + +def test_continue_in_finally(): + code = dedent('''\ + for a in [1]: + try: + pass + finally: + continue + ''') + assert not _get_error_list(code, version="3.8") + assert _get_error_list(code, version="3.7") + + +@pytest.mark.parametrize( + 'template', [ + "a, b, {target}, c = d", + "a, b, *{target}, c = d", + "(a, *{target}), c = d", + "for x, {target} in y: pass", + "for x, q, {target} in y: pass", + "for x, q, *{target} in y: pass", + "for (x, *{target}), q in y: pass", + ] +) +@pytest.mark.parametrize( + 'target', [ + "True", + "False", + "None", + "__debug__" + ] +) +def test_forbidden_name(template, target): + assert _get_error_list(template.format(target=target), version="3") + + +def test_repeated_kwarg(): + # python 3.9+ shows which argument is repeated + assert ( + _get_error_list("f(q=1, q=2)", version="3.8")[0].message + == "SyntaxError: keyword argument repeated" + ) + assert ( + _get_error_list("f(q=1, q=2)", version="3.9")[0].message + == "SyntaxError: keyword argument repeated: q" + ) + + +@pytest.mark.parametrize( + ('source', 'no_errors'), [ + ('a(a for a in b,)', False), + ('a(a for a in b, a)', False), + ('a(a, a for a in b)', False), + ('a(a, b, a for a in b, c, d)', False), + ('a(a for a in b)', True), + ('a((a for a in b), c)', True), + ('a(c, (a for a in b))', True), + ('a(a, b, (a for a in b), c, d)', True), + ] +) +def test_unparenthesized_genexp(source, no_errors): + assert bool(_get_error_list(source)) ^ no_errors + + +@pytest.mark.parametrize( + ('source', 'no_errors'), [ + ('*x = 2', False), + ('(*y) = 1', False), + ('((*z)) = 1', False), + ('*a,', True), + ('*a, = 1', True), + ('(*a,)', True), + ('(*a,) = 1', True), + ('[*a]', True), + ('[*a] = 1', True), + ('a, *b', True), + ('a, *b = 1', True), + ('a, *b, c', True), + ('a, *b, c = 1', True), + ('a, (*b, c), d', True), + ('a, (*b, c), d = 1', True), + ('*a.b,', True), + ('*a.b, = 1', True), + ('*a[b],', True), + ('*a[b], = 1', True), + ('*a[b::], c', True), + ('*a[b::], c = 1', True), + ('(a, *[b, c])', True), + ('(a, *[b, c]) = 1', True), + ('[a, *(b, [*c])]', True), + ('[a, *(b, [*c])] = 1', True), + ('[*(1,2,3)]', True), + ('{*(1,2,3)}', True), + ('[*(1,2,3),]', True), + ('[*(1,2,3), *(4,5,6)]', True), + ('[0, *(1,2,3)]', True), + ('{*(1,2,3),}', True), + ('{*(1,2,3), *(4,5,6)}', True), + ('{0, *(4,5,6)}', True) + ] +) +def test_starred_expr(source, no_errors): + assert bool(_get_error_list(source, version="3")) ^ no_errors + + +@pytest.mark.parametrize( + 'code', [ + 'a, (*b), c', + 'a, (*b), c = 1', + 'a, ((*b)), c', + 'a, ((*b)), c = 1', + ] +) +def test_parenthesized_single_starred_expr(code): + assert not _get_error_list(code, version='3.8') + assert _get_error_list(code, version='3.9') + + +@pytest.mark.parametrize( + 'code', [ + '() = ()', + '() = []', + '[] = ()', + '[] = []', + ] +) +def test_valid_empty_assignment(code): + assert not _get_error_list(code) + + +@pytest.mark.parametrize( + 'code', [ + 'del ()', + 'del []', + 'del x', + 'del x,', + 'del x, y', + 'del (x, y)', + 'del [x, y]', + 'del (x, [y, z])', + 'del x.y, x[y]', + 'del f(x)[y::]', + 'del x[[*y]]', + 'del x[[*y]::]', + ] +) +def test_valid_del(code): + assert not _get_error_list(code) + + +@pytest.mark.parametrize( + ('source', 'version', 'no_errors'), [ + ('[x for x in range(10) if lambda: 1]', '3.8', True), + ('[x for x in range(10) if lambda: 1]', '3.9', False), + ('[x for x in range(10) if (lambda: 1)]', '3.9', True), + ] +) +def test_lambda_in_comp_if(source, version, no_errors): + assert bool(_get_error_list(source, version=version)) ^ no_errors diff --git a/contrib/python/parso/py3/tests/test_tokenize.py b/contrib/python/parso/py3/tests/test_tokenize.py index 0029fc8a51..53b41f6cea 100644 --- a/contrib/python/parso/py3/tests/test_tokenize.py +++ b/contrib/python/parso/py3/tests/test_tokenize.py @@ -1,429 +1,429 @@ -# -*- coding: utf-8 # This file contains Unicode characters. - -from textwrap import dedent - -import pytest - -from parso.utils import split_lines, parse_version_string -from parso.python.token import PythonTokenTypes -from parso.python import tokenize -from parso import parse -from parso.python.tokenize import PythonToken - - -# To make it easier to access some of the token types, just put them here. -NAME = PythonTokenTypes.NAME -NEWLINE = PythonTokenTypes.NEWLINE -STRING = PythonTokenTypes.STRING -NUMBER = PythonTokenTypes.NUMBER -INDENT = PythonTokenTypes.INDENT -DEDENT = PythonTokenTypes.DEDENT -ERRORTOKEN = PythonTokenTypes.ERRORTOKEN -OP = PythonTokenTypes.OP -ENDMARKER = PythonTokenTypes.ENDMARKER -ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT -FSTRING_START = PythonTokenTypes.FSTRING_START -FSTRING_STRING = PythonTokenTypes.FSTRING_STRING -FSTRING_END = PythonTokenTypes.FSTRING_END - - -def _get_token_list(string, version=None): - # Load the current version. - version_info = parse_version_string(version) - return list(tokenize.tokenize(string, version_info=version_info)) - - -def test_end_pos_one_line(): - parsed = parse(dedent(''' - def testit(): - a = "huhu" - ''')) - simple_stmt = next(parsed.iter_funcdefs()).get_suite().children[-1] - string = simple_stmt.children[0].get_rhs() - assert string.end_pos == (3, 14) - - -def test_end_pos_multi_line(): - parsed = parse(dedent(''' - def testit(): - a = """huhu - asdfasdf""" + "h" - ''')) - expr_stmt = next(parsed.iter_funcdefs()).get_suite().children[1].children[0] - string_leaf = expr_stmt.get_rhs().children[0] - assert string_leaf.end_pos == (4, 11) - - -def test_simple_no_whitespace(): - # Test a simple one line string, no preceding whitespace - simple_docstring = '"""simple one line docstring"""' - token_list = _get_token_list(simple_docstring) - _, value, _, prefix = token_list[0] - assert prefix == '' - assert value == '"""simple one line docstring"""' - - -def test_simple_with_whitespace(): - # Test a simple one line string with preceding whitespace and newline - simple_docstring = ' """simple one line docstring""" \r\n' - token_list = _get_token_list(simple_docstring) - assert token_list[0][0] == INDENT - typ, value, start_pos, prefix = token_list[1] - assert prefix == ' ' - assert value == '"""simple one line docstring"""' - assert typ == STRING - typ, value, start_pos, prefix = token_list[2] - assert prefix == ' ' - assert typ == NEWLINE - - -def test_function_whitespace(): - # Test function definition whitespace identification - fundef = dedent(''' - def test_whitespace(*args, **kwargs): - x = 1 - if x > 0: - print(True) - ''') - token_list = _get_token_list(fundef) - for _, value, _, prefix in token_list: - if value == 'test_whitespace': - assert prefix == ' ' - if value == '(': - assert prefix == '' - if value == '*': - assert prefix == '' - if value == '**': - assert prefix == ' ' - if value == 'print': - assert prefix == ' ' - if value == 'if': - assert prefix == ' ' - - -def test_tokenize_multiline_I(): - # Make sure multiline string having newlines have the end marker on the - # next line - fundef = '''""""\n''' - token_list = _get_token_list(fundef) - assert token_list == [PythonToken(ERRORTOKEN, '""""\n', (1, 0), ''), - PythonToken(ENDMARKER, '', (2, 0), '')] - - -def test_tokenize_multiline_II(): - # Make sure multiline string having no newlines have the end marker on - # same line - fundef = '''""""''' - token_list = _get_token_list(fundef) - assert token_list == [PythonToken(ERRORTOKEN, '""""', (1, 0), ''), - PythonToken(ENDMARKER, '', (1, 4), '')] - - -def test_tokenize_multiline_III(): - # Make sure multiline string having newlines have the end marker on the - # next line even if several newline - fundef = '''""""\n\n''' - token_list = _get_token_list(fundef) - assert token_list == [PythonToken(ERRORTOKEN, '""""\n\n', (1, 0), ''), - PythonToken(ENDMARKER, '', (3, 0), '')] - - -def test_identifier_contains_unicode(): - fundef = dedent(''' - def 我あφ(): - pass - ''') - token_list = _get_token_list(fundef) - unicode_token = token_list[1] - assert unicode_token[0] == NAME - - -def test_quoted_strings(): - string_tokens = [ - 'u"test"', - 'u"""test"""', - 'U"""test"""', - "u'''test'''", - "U'''test'''", - ] - - for s in string_tokens: - module = parse('''a = %s\n''' % s) - simple_stmt = module.children[0] - expr_stmt = simple_stmt.children[0] - assert len(expr_stmt.children) == 3 - string_tok = expr_stmt.children[2] - assert string_tok.type == 'string' - assert string_tok.value == s - - -def test_ur_literals(): - """ - Decided to parse `u''` literals regardless of Python version. This makes - probably sense: - - - Python 3+ doesn't support it, but it doesn't hurt - not be. While this is incorrect, it's just incorrect for one "old" and in - the future not very important version. - - All the other Python versions work very well with it. - """ - def check(literal, is_literal=True): - token_list = _get_token_list(literal) - typ, result_literal, _, _ = token_list[0] - if is_literal: - if typ != FSTRING_START: - assert typ == STRING - assert result_literal == literal - else: - assert typ == NAME - - check('u""') - check('ur""', is_literal=False) - check('Ur""', is_literal=False) - check('UR""', is_literal=False) - check('bR""') - check('Rb""') - - check('fr""') - check('rF""') - check('f""') - check('F""') - - -def test_error_literal(): - error_token, newline, endmarker = _get_token_list('"\n') - assert error_token.type == ERRORTOKEN - assert error_token.string == '"' - assert newline.type == NEWLINE - assert endmarker.type == ENDMARKER - assert endmarker.prefix == '' - - bracket, error_token, endmarker = _get_token_list('( """') - assert error_token.type == ERRORTOKEN - assert error_token.prefix == ' ' - assert error_token.string == '"""' - assert endmarker.type == ENDMARKER - assert endmarker.prefix == '' - - -def test_endmarker_end_pos(): - def check(code): - tokens = _get_token_list(code) - lines = split_lines(code) - assert tokens[-1].end_pos == (len(lines), len(lines[-1])) - - check('#c') - check('#c\n') - check('a\n') - check('a') - check(r'a\\n') - check('a\\') - - -@pytest.mark.parametrize( - ('code', 'types'), [ - # Indentation - (' foo', [INDENT, NAME, DEDENT]), - (' foo\n bar', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, DEDENT]), - (' foo\n bar \n baz', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, - NEWLINE, NAME, DEDENT]), - (' foo\nbar', [INDENT, NAME, NEWLINE, DEDENT, NAME]), - - # Name stuff - ('1foo1', [NUMBER, NAME]), - ('மெல்லினம்', [NAME]), - ('²', [ERRORTOKEN]), - ('ä²ö', [NAME, ERRORTOKEN, NAME]), - ('ää²¹öö', [NAME, ERRORTOKEN, NAME]), - (' \x00a', [INDENT, ERRORTOKEN, NAME, DEDENT]), - (dedent('''\ - class BaseCache: - a - def - b - def - c - '''), [NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE, - ERROR_DEDENT, NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, - NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, DEDENT]), - (' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]), - ('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP, - NEWLINE, DEDENT, NAME, DEDENT]), - (' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]), - ] -) -def test_token_types(code, types): - actual_types = [t.type for t in _get_token_list(code)] - assert actual_types == types + [ENDMARKER] - - -def test_error_string(): - indent, t1, newline, token, endmarker = _get_token_list(' "\n') - assert t1.type == ERRORTOKEN - assert t1.prefix == ' ' - assert t1.string == '"' - assert newline.type == NEWLINE - assert endmarker.prefix == '' - assert endmarker.string == '' - - -def test_indent_error_recovery(): - code = dedent("""\ - str( - from x import a - def - """) - lst = _get_token_list(code) - expected = [ - # `str(` - INDENT, NAME, OP, - # `from parso` - NAME, NAME, - # `import a` on same line as the previous from parso - NAME, NAME, NEWLINE, - # Dedent happens, because there's an import now and the import - # statement "breaks" out of the opening paren on the first line. - DEDENT, - # `b` - NAME, NEWLINE, ENDMARKER] - assert [t.type for t in lst] == expected - - -def test_error_token_after_dedent(): - code = dedent("""\ - class C: - pass - $foo - """) - lst = _get_token_list(code) - expected = [ - NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, - # $foo\n - ERRORTOKEN, NAME, NEWLINE, ENDMARKER - ] - assert [t.type for t in lst] == expected - - -def test_brackets_no_indentation(): - """ - There used to be an issue that the parentheses counting would go below - zero. This should not happen. - """ - code = dedent("""\ - } - { - } - """) - lst = _get_token_list(code) - assert [t.type for t in lst] == [OP, NEWLINE, OP, OP, NEWLINE, ENDMARKER] - - -def test_form_feed(): - indent, error_token, dedent_, endmarker = _get_token_list(dedent('''\ - \f"""''')) - assert error_token.prefix == '\f' - assert error_token.string == '"""' - assert endmarker.prefix == '' - assert indent.type == INDENT - assert dedent_.type == DEDENT - - -def test_carriage_return(): - lst = _get_token_list(' =\\\rclass') - assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER] - - -def test_backslash(): - code = '\\\n# 1 \n' - endmarker, = _get_token_list(code) - assert endmarker.prefix == code - - -@pytest.mark.parametrize( - ('code', 'types'), [ - # f-strings - ('f"', [FSTRING_START]), - ('f""', [FSTRING_START, FSTRING_END]), - ('f" {}"', [FSTRING_START, FSTRING_STRING, OP, OP, FSTRING_END]), - ('f" "{}', [FSTRING_START, FSTRING_STRING, FSTRING_END, OP, OP]), - (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), - (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), - - # format spec - (r'f"Some {x:.2f}{y}"', [FSTRING_START, FSTRING_STRING, OP, NAME, OP, - FSTRING_STRING, OP, OP, NAME, OP, FSTRING_END]), - - # multiline f-string - ('f"""abc\ndef"""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), - ('f"""abc{\n123}def"""', [ - FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING, - FSTRING_END - ]), - - # a line continuation inside of an fstring_string - ('f"abc\\\ndef"', [ - FSTRING_START, FSTRING_STRING, FSTRING_END - ]), - ('f"\\\n{123}\\\n"', [ - FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING, - FSTRING_END - ]), - - # a line continuation inside of an fstring_expr - ('f"{\\\n123}"', [FSTRING_START, OP, NUMBER, OP, FSTRING_END]), - - # a line continuation inside of an format spec - ('f"{123:.2\\\nf}"', [ - FSTRING_START, OP, NUMBER, OP, FSTRING_STRING, OP, FSTRING_END - ]), - - # a newline without a line continuation inside a single-line string is - # wrong, and will generate an ERRORTOKEN - ('f"abc\ndef"', [ - FSTRING_START, FSTRING_STRING, NEWLINE, NAME, ERRORTOKEN - ]), - - # a more complex example - (r'print(f"Some {x:.2f}a{y}")', [ - NAME, OP, FSTRING_START, FSTRING_STRING, OP, NAME, OP, - FSTRING_STRING, OP, FSTRING_STRING, OP, NAME, OP, FSTRING_END, OP - ]), - # issue #86, a string-like in an f-string expression - ('f"{ ""}"', [ - FSTRING_START, OP, FSTRING_END, STRING - ]), - ('f"{ f""}"', [ - FSTRING_START, OP, NAME, FSTRING_END, STRING - ]), - ] -) -def test_fstring_token_types(code, types, each_version): - actual_types = [t.type for t in _get_token_list(code, each_version)] - assert types + [ENDMARKER] == actual_types - - -@pytest.mark.parametrize( - ('code', 'types'), [ - # issue #87, `:=` in the outest paratheses should be tokenized - # as a format spec marker and part of the format - ('f"{x:=10}"', [ - FSTRING_START, OP, NAME, OP, FSTRING_STRING, OP, FSTRING_END - ]), - ('f"{(x:=10)}"', [ - FSTRING_START, OP, OP, NAME, OP, NUMBER, OP, OP, FSTRING_END - ]), - ] -) -def test_fstring_assignment_expression(code, types, version_ge_py38): - actual_types = [t.type for t in _get_token_list(code, version_ge_py38)] - assert types + [ENDMARKER] == actual_types - - -def test_fstring_end_error_pos(version_ge_py38): - f_start, f_string, bracket, f_end, endmarker = \ - _get_token_list('f" { "', version_ge_py38) - assert f_start.start_pos == (1, 0) - assert f_string.start_pos == (1, 2) - assert bracket.start_pos == (1, 3) - assert f_end.start_pos == (1, 5) - assert endmarker.start_pos == (1, 6) +# -*- coding: utf-8 # This file contains Unicode characters. + +from textwrap import dedent + +import pytest + +from parso.utils import split_lines, parse_version_string +from parso.python.token import PythonTokenTypes +from parso.python import tokenize +from parso import parse +from parso.python.tokenize import PythonToken + + +# To make it easier to access some of the token types, just put them here. +NAME = PythonTokenTypes.NAME +NEWLINE = PythonTokenTypes.NEWLINE +STRING = PythonTokenTypes.STRING +NUMBER = PythonTokenTypes.NUMBER +INDENT = PythonTokenTypes.INDENT +DEDENT = PythonTokenTypes.DEDENT +ERRORTOKEN = PythonTokenTypes.ERRORTOKEN +OP = PythonTokenTypes.OP +ENDMARKER = PythonTokenTypes.ENDMARKER +ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT +FSTRING_START = PythonTokenTypes.FSTRING_START +FSTRING_STRING = PythonTokenTypes.FSTRING_STRING +FSTRING_END = PythonTokenTypes.FSTRING_END + + +def _get_token_list(string, version=None): + # Load the current version. + version_info = parse_version_string(version) + return list(tokenize.tokenize(string, version_info=version_info)) + + +def test_end_pos_one_line(): + parsed = parse(dedent(''' + def testit(): + a = "huhu" + ''')) + simple_stmt = next(parsed.iter_funcdefs()).get_suite().children[-1] + string = simple_stmt.children[0].get_rhs() + assert string.end_pos == (3, 14) + + +def test_end_pos_multi_line(): + parsed = parse(dedent(''' + def testit(): + a = """huhu + asdfasdf""" + "h" + ''')) + expr_stmt = next(parsed.iter_funcdefs()).get_suite().children[1].children[0] + string_leaf = expr_stmt.get_rhs().children[0] + assert string_leaf.end_pos == (4, 11) + + +def test_simple_no_whitespace(): + # Test a simple one line string, no preceding whitespace + simple_docstring = '"""simple one line docstring"""' + token_list = _get_token_list(simple_docstring) + _, value, _, prefix = token_list[0] + assert prefix == '' + assert value == '"""simple one line docstring"""' + + +def test_simple_with_whitespace(): + # Test a simple one line string with preceding whitespace and newline + simple_docstring = ' """simple one line docstring""" \r\n' + token_list = _get_token_list(simple_docstring) + assert token_list[0][0] == INDENT + typ, value, start_pos, prefix = token_list[1] + assert prefix == ' ' + assert value == '"""simple one line docstring"""' + assert typ == STRING + typ, value, start_pos, prefix = token_list[2] + assert prefix == ' ' + assert typ == NEWLINE + + +def test_function_whitespace(): + # Test function definition whitespace identification + fundef = dedent(''' + def test_whitespace(*args, **kwargs): + x = 1 + if x > 0: + print(True) + ''') + token_list = _get_token_list(fundef) + for _, value, _, prefix in token_list: + if value == 'test_whitespace': + assert prefix == ' ' + if value == '(': + assert prefix == '' + if value == '*': + assert prefix == '' + if value == '**': + assert prefix == ' ' + if value == 'print': + assert prefix == ' ' + if value == 'if': + assert prefix == ' ' + + +def test_tokenize_multiline_I(): + # Make sure multiline string having newlines have the end marker on the + # next line + fundef = '''""""\n''' + token_list = _get_token_list(fundef) + assert token_list == [PythonToken(ERRORTOKEN, '""""\n', (1, 0), ''), + PythonToken(ENDMARKER, '', (2, 0), '')] + + +def test_tokenize_multiline_II(): + # Make sure multiline string having no newlines have the end marker on + # same line + fundef = '''""""''' + token_list = _get_token_list(fundef) + assert token_list == [PythonToken(ERRORTOKEN, '""""', (1, 0), ''), + PythonToken(ENDMARKER, '', (1, 4), '')] + + +def test_tokenize_multiline_III(): + # Make sure multiline string having newlines have the end marker on the + # next line even if several newline + fundef = '''""""\n\n''' + token_list = _get_token_list(fundef) + assert token_list == [PythonToken(ERRORTOKEN, '""""\n\n', (1, 0), ''), + PythonToken(ENDMARKER, '', (3, 0), '')] + + +def test_identifier_contains_unicode(): + fundef = dedent(''' + def 我あφ(): + pass + ''') + token_list = _get_token_list(fundef) + unicode_token = token_list[1] + assert unicode_token[0] == NAME + + +def test_quoted_strings(): + string_tokens = [ + 'u"test"', + 'u"""test"""', + 'U"""test"""', + "u'''test'''", + "U'''test'''", + ] + + for s in string_tokens: + module = parse('''a = %s\n''' % s) + simple_stmt = module.children[0] + expr_stmt = simple_stmt.children[0] + assert len(expr_stmt.children) == 3 + string_tok = expr_stmt.children[2] + assert string_tok.type == 'string' + assert string_tok.value == s + + +def test_ur_literals(): + """ + Decided to parse `u''` literals regardless of Python version. This makes + probably sense: + + - Python 3+ doesn't support it, but it doesn't hurt + not be. While this is incorrect, it's just incorrect for one "old" and in + the future not very important version. + - All the other Python versions work very well with it. + """ + def check(literal, is_literal=True): + token_list = _get_token_list(literal) + typ, result_literal, _, _ = token_list[0] + if is_literal: + if typ != FSTRING_START: + assert typ == STRING + assert result_literal == literal + else: + assert typ == NAME + + check('u""') + check('ur""', is_literal=False) + check('Ur""', is_literal=False) + check('UR""', is_literal=False) + check('bR""') + check('Rb""') + + check('fr""') + check('rF""') + check('f""') + check('F""') + + +def test_error_literal(): + error_token, newline, endmarker = _get_token_list('"\n') + assert error_token.type == ERRORTOKEN + assert error_token.string == '"' + assert newline.type == NEWLINE + assert endmarker.type == ENDMARKER + assert endmarker.prefix == '' + + bracket, error_token, endmarker = _get_token_list('( """') + assert error_token.type == ERRORTOKEN + assert error_token.prefix == ' ' + assert error_token.string == '"""' + assert endmarker.type == ENDMARKER + assert endmarker.prefix == '' + + +def test_endmarker_end_pos(): + def check(code): + tokens = _get_token_list(code) + lines = split_lines(code) + assert tokens[-1].end_pos == (len(lines), len(lines[-1])) + + check('#c') + check('#c\n') + check('a\n') + check('a') + check(r'a\\n') + check('a\\') + + +@pytest.mark.parametrize( + ('code', 'types'), [ + # Indentation + (' foo', [INDENT, NAME, DEDENT]), + (' foo\n bar', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, DEDENT]), + (' foo\n bar \n baz', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, + NEWLINE, NAME, DEDENT]), + (' foo\nbar', [INDENT, NAME, NEWLINE, DEDENT, NAME]), + + # Name stuff + ('1foo1', [NUMBER, NAME]), + ('மெல்லினம்', [NAME]), + ('²', [ERRORTOKEN]), + ('ä²ö', [NAME, ERRORTOKEN, NAME]), + ('ää²¹öö', [NAME, ERRORTOKEN, NAME]), + (' \x00a', [INDENT, ERRORTOKEN, NAME, DEDENT]), + (dedent('''\ + class BaseCache: + a + def + b + def + c + '''), [NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE, + ERROR_DEDENT, NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, + NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, DEDENT]), + (' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]), + ('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP, + NEWLINE, DEDENT, NAME, DEDENT]), + (' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]), + ] +) +def test_token_types(code, types): + actual_types = [t.type for t in _get_token_list(code)] + assert actual_types == types + [ENDMARKER] + + +def test_error_string(): + indent, t1, newline, token, endmarker = _get_token_list(' "\n') + assert t1.type == ERRORTOKEN + assert t1.prefix == ' ' + assert t1.string == '"' + assert newline.type == NEWLINE + assert endmarker.prefix == '' + assert endmarker.string == '' + + +def test_indent_error_recovery(): + code = dedent("""\ + str( + from x import a + def + """) + lst = _get_token_list(code) + expected = [ + # `str(` + INDENT, NAME, OP, + # `from parso` + NAME, NAME, + # `import a` on same line as the previous from parso + NAME, NAME, NEWLINE, + # Dedent happens, because there's an import now and the import + # statement "breaks" out of the opening paren on the first line. + DEDENT, + # `b` + NAME, NEWLINE, ENDMARKER] + assert [t.type for t in lst] == expected + + +def test_error_token_after_dedent(): + code = dedent("""\ + class C: + pass + $foo + """) + lst = _get_token_list(code) + expected = [ + NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, + # $foo\n + ERRORTOKEN, NAME, NEWLINE, ENDMARKER + ] + assert [t.type for t in lst] == expected + + +def test_brackets_no_indentation(): + """ + There used to be an issue that the parentheses counting would go below + zero. This should not happen. + """ + code = dedent("""\ + } + { + } + """) + lst = _get_token_list(code) + assert [t.type for t in lst] == [OP, NEWLINE, OP, OP, NEWLINE, ENDMARKER] + + +def test_form_feed(): + indent, error_token, dedent_, endmarker = _get_token_list(dedent('''\ + \f"""''')) + assert error_token.prefix == '\f' + assert error_token.string == '"""' + assert endmarker.prefix == '' + assert indent.type == INDENT + assert dedent_.type == DEDENT + + +def test_carriage_return(): + lst = _get_token_list(' =\\\rclass') + assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER] + + +def test_backslash(): + code = '\\\n# 1 \n' + endmarker, = _get_token_list(code) + assert endmarker.prefix == code + + +@pytest.mark.parametrize( + ('code', 'types'), [ + # f-strings + ('f"', [FSTRING_START]), + ('f""', [FSTRING_START, FSTRING_END]), + ('f" {}"', [FSTRING_START, FSTRING_STRING, OP, OP, FSTRING_END]), + ('f" "{}', [FSTRING_START, FSTRING_STRING, FSTRING_END, OP, OP]), + (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), + (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), + + # format spec + (r'f"Some {x:.2f}{y}"', [FSTRING_START, FSTRING_STRING, OP, NAME, OP, + FSTRING_STRING, OP, OP, NAME, OP, FSTRING_END]), + + # multiline f-string + ('f"""abc\ndef"""', [FSTRING_START, FSTRING_STRING, FSTRING_END]), + ('f"""abc{\n123}def"""', [ + FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING, + FSTRING_END + ]), + + # a line continuation inside of an fstring_string + ('f"abc\\\ndef"', [ + FSTRING_START, FSTRING_STRING, FSTRING_END + ]), + ('f"\\\n{123}\\\n"', [ + FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING, + FSTRING_END + ]), + + # a line continuation inside of an fstring_expr + ('f"{\\\n123}"', [FSTRING_START, OP, NUMBER, OP, FSTRING_END]), + + # a line continuation inside of an format spec + ('f"{123:.2\\\nf}"', [ + FSTRING_START, OP, NUMBER, OP, FSTRING_STRING, OP, FSTRING_END + ]), + + # a newline without a line continuation inside a single-line string is + # wrong, and will generate an ERRORTOKEN + ('f"abc\ndef"', [ + FSTRING_START, FSTRING_STRING, NEWLINE, NAME, ERRORTOKEN + ]), + + # a more complex example + (r'print(f"Some {x:.2f}a{y}")', [ + NAME, OP, FSTRING_START, FSTRING_STRING, OP, NAME, OP, + FSTRING_STRING, OP, FSTRING_STRING, OP, NAME, OP, FSTRING_END, OP + ]), + # issue #86, a string-like in an f-string expression + ('f"{ ""}"', [ + FSTRING_START, OP, FSTRING_END, STRING + ]), + ('f"{ f""}"', [ + FSTRING_START, OP, NAME, FSTRING_END, STRING + ]), + ] +) +def test_fstring_token_types(code, types, each_version): + actual_types = [t.type for t in _get_token_list(code, each_version)] + assert types + [ENDMARKER] == actual_types + + +@pytest.mark.parametrize( + ('code', 'types'), [ + # issue #87, `:=` in the outest paratheses should be tokenized + # as a format spec marker and part of the format + ('f"{x:=10}"', [ + FSTRING_START, OP, NAME, OP, FSTRING_STRING, OP, FSTRING_END + ]), + ('f"{(x:=10)}"', [ + FSTRING_START, OP, OP, NAME, OP, NUMBER, OP, OP, FSTRING_END + ]), + ] +) +def test_fstring_assignment_expression(code, types, version_ge_py38): + actual_types = [t.type for t in _get_token_list(code, version_ge_py38)] + assert types + [ENDMARKER] == actual_types + + +def test_fstring_end_error_pos(version_ge_py38): + f_start, f_string, bracket, f_end, endmarker = \ + _get_token_list('f" { "', version_ge_py38) + assert f_start.start_pos == (1, 0) + assert f_string.start_pos == (1, 2) + assert bracket.start_pos == (1, 3) + assert f_end.start_pos == (1, 5) + assert endmarker.start_pos == (1, 6) diff --git a/contrib/python/parso/py3/tests/test_utils.py b/contrib/python/parso/py3/tests/test_utils.py index 300a54ebc2..90b6f875a5 100644 --- a/contrib/python/parso/py3/tests/test_utils.py +++ b/contrib/python/parso/py3/tests/test_utils.py @@ -1,107 +1,107 @@ -from codecs import BOM_UTF8 - -from parso.utils import ( - split_lines, - parse_version_string, - python_bytes_to_unicode, -) - -import parso - -import pytest - - -@pytest.mark.parametrize( - ('string', 'expected_result', 'keepends'), [ - ('asd\r\n', ['asd', ''], False), - ('asd\r\n', ['asd\r\n', ''], True), - ('asd\r', ['asd', ''], False), - ('asd\r', ['asd\r', ''], True), - ('asd\n', ['asd', ''], False), - ('asd\n', ['asd\n', ''], True), - - ('asd\r\n\f', ['asd', '\f'], False), - ('asd\r\n\f', ['asd\r\n', '\f'], True), - - ('\fasd\r\n', ['\fasd', ''], False), - ('\fasd\r\n', ['\fasd\r\n', ''], True), - - ('', [''], False), - ('', [''], True), - - ('\n', ['', ''], False), - ('\n', ['\n', ''], True), - - ('\r', ['', ''], False), - ('\r', ['\r', ''], True), - - # Invalid line breaks - ('a\vb', ['a\vb'], False), - ('a\vb', ['a\vb'], True), - ('\x1C', ['\x1C'], False), - ('\x1C', ['\x1C'], True), - ] -) -def test_split_lines(string, expected_result, keepends): - assert split_lines(string, keepends=keepends) == expected_result - - -def test_python_bytes_to_unicode_unicode_text(): - source = ( - b"# vim: fileencoding=utf-8\n" - b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n" - ) - actual = python_bytes_to_unicode(source) - expected = source.decode('utf-8') - assert actual == expected - - -def test_utf8_bom(): - unicode_bom = BOM_UTF8.decode('utf-8') - - module = parso.parse(unicode_bom) - endmarker = module.children[0] - assert endmarker.type == 'endmarker' - assert unicode_bom == endmarker.prefix - - module = parso.parse(unicode_bom + 'foo = 1') - expr_stmt = module.children[0] - assert expr_stmt.type == 'expr_stmt' - assert unicode_bom == expr_stmt.get_first_leaf().prefix - - -@pytest.mark.parametrize( - ('code', 'errors'), [ - (b'# coding: wtf-12\nfoo', 'strict'), - (b'# coding: wtf-12\nfoo', 'replace'), - (b'# coding: wtf-12\r\nfoo', 'strict'), - (b'# coding: wtf-12\r\nfoo', 'replace'), - (b'# coding: wtf-12\rfoo', 'strict'), - (b'# coding: wtf-12\rfoo', 'replace'), - ] -) -def test_bytes_to_unicode_failing_encoding(code, errors): - if errors == 'strict': - with pytest.raises(LookupError): - python_bytes_to_unicode(code, errors=errors) - else: - python_bytes_to_unicode(code, errors=errors) - - -@pytest.mark.parametrize( - ('version_str', 'version'), [ - ('3', (3,)), - ('3.6', (3, 6)), - ('3.6.10', (3, 6)), - ('3.10', (3, 10)), - ('3.10a9', (3, 10)), - ('3.10b9', (3, 10)), - ('3.10rc9', (3, 10)), - ] -) -def test_parse_version_string(version_str, version): - parsed_version = parse_version_string(version_str) - if len(version) == 1: - assert parsed_version[0] == version[0] - else: - assert parsed_version == version +from codecs import BOM_UTF8 + +from parso.utils import ( + split_lines, + parse_version_string, + python_bytes_to_unicode, +) + +import parso + +import pytest + + +@pytest.mark.parametrize( + ('string', 'expected_result', 'keepends'), [ + ('asd\r\n', ['asd', ''], False), + ('asd\r\n', ['asd\r\n', ''], True), + ('asd\r', ['asd', ''], False), + ('asd\r', ['asd\r', ''], True), + ('asd\n', ['asd', ''], False), + ('asd\n', ['asd\n', ''], True), + + ('asd\r\n\f', ['asd', '\f'], False), + ('asd\r\n\f', ['asd\r\n', '\f'], True), + + ('\fasd\r\n', ['\fasd', ''], False), + ('\fasd\r\n', ['\fasd\r\n', ''], True), + + ('', [''], False), + ('', [''], True), + + ('\n', ['', ''], False), + ('\n', ['\n', ''], True), + + ('\r', ['', ''], False), + ('\r', ['\r', ''], True), + + # Invalid line breaks + ('a\vb', ['a\vb'], False), + ('a\vb', ['a\vb'], True), + ('\x1C', ['\x1C'], False), + ('\x1C', ['\x1C'], True), + ] +) +def test_split_lines(string, expected_result, keepends): + assert split_lines(string, keepends=keepends) == expected_result + + +def test_python_bytes_to_unicode_unicode_text(): + source = ( + b"# vim: fileencoding=utf-8\n" + b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n" + ) + actual = python_bytes_to_unicode(source) + expected = source.decode('utf-8') + assert actual == expected + + +def test_utf8_bom(): + unicode_bom = BOM_UTF8.decode('utf-8') + + module = parso.parse(unicode_bom) + endmarker = module.children[0] + assert endmarker.type == 'endmarker' + assert unicode_bom == endmarker.prefix + + module = parso.parse(unicode_bom + 'foo = 1') + expr_stmt = module.children[0] + assert expr_stmt.type == 'expr_stmt' + assert unicode_bom == expr_stmt.get_first_leaf().prefix + + +@pytest.mark.parametrize( + ('code', 'errors'), [ + (b'# coding: wtf-12\nfoo', 'strict'), + (b'# coding: wtf-12\nfoo', 'replace'), + (b'# coding: wtf-12\r\nfoo', 'strict'), + (b'# coding: wtf-12\r\nfoo', 'replace'), + (b'# coding: wtf-12\rfoo', 'strict'), + (b'# coding: wtf-12\rfoo', 'replace'), + ] +) +def test_bytes_to_unicode_failing_encoding(code, errors): + if errors == 'strict': + with pytest.raises(LookupError): + python_bytes_to_unicode(code, errors=errors) + else: + python_bytes_to_unicode(code, errors=errors) + + +@pytest.mark.parametrize( + ('version_str', 'version'), [ + ('3', (3,)), + ('3.6', (3, 6)), + ('3.6.10', (3, 6)), + ('3.10', (3, 10)), + ('3.10a9', (3, 10)), + ('3.10b9', (3, 10)), + ('3.10rc9', (3, 10)), + ] +) +def test_parse_version_string(version_str, version): + parsed_version = parse_version_string(version_str) + if len(version) == 1: + assert parsed_version[0] == version[0] + else: + assert parsed_version == version diff --git a/contrib/python/parso/py3/tests/ya.make b/contrib/python/parso/py3/tests/ya.make index 72957e757d..3642d47d2a 100644 --- a/contrib/python/parso/py3/tests/ya.make +++ b/contrib/python/parso/py3/tests/ya.make @@ -1,41 +1,41 @@ -PY3TEST() - -OWNER(g:python-contrib) - -PEERDIR( - contrib/python/parso -) - -DATA( - arcadia/contrib/python/parso/py3/tests -) - -TEST_SRCS( - __init__.py - conftest.py - failing_examples.py - test_cache.py - test_diff_parser.py - test_dump_tree.py - test_error_recovery.py - test_file_python_errors.py - test_fstring.py - test_get_code.py - test_grammar.py - test_load_grammar.py - test_normalizer_issues_files.py - test_old_fast_parser.py - test_param_splitting.py - test_parser.py - test_parser_tree.py - test_pep8.py - test_pgen2.py - test_prefix.py - test_python_errors.py - test_tokenize.py - test_utils.py -) - -NO_LINT() - -END() +PY3TEST() + +OWNER(g:python-contrib) + +PEERDIR( + contrib/python/parso +) + +DATA( + arcadia/contrib/python/parso/py3/tests +) + +TEST_SRCS( + __init__.py + conftest.py + failing_examples.py + test_cache.py + test_diff_parser.py + test_dump_tree.py + test_error_recovery.py + test_file_python_errors.py + test_fstring.py + test_get_code.py + test_grammar.py + test_load_grammar.py + test_normalizer_issues_files.py + test_old_fast_parser.py + test_param_splitting.py + test_parser.py + test_parser_tree.py + test_pep8.py + test_pgen2.py + test_prefix.py + test_python_errors.py + test_tokenize.py + test_utils.py +) + +NO_LINT() + +END() |