aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Compiler/Parsing.py
diff options
context:
space:
mode:
authororivej <orivej@yandex-team.ru>2022-02-10 16:45:01 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:01 +0300
commit2d37894b1b037cf24231090eda8589bbb44fb6fc (patch)
treebe835aa92c6248212e705f25388ebafcf84bc7a1 /contrib/tools/cython/Cython/Compiler/Parsing.py
parent718c552901d703c502ccbefdfc3c9028d608b947 (diff)
downloadydb-2d37894b1b037cf24231090eda8589bbb44fb6fc.tar.gz
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Compiler/Parsing.py')
-rw-r--r--contrib/tools/cython/Cython/Compiler/Parsing.py1402
1 files changed, 701 insertions, 701 deletions
diff --git a/contrib/tools/cython/Cython/Compiler/Parsing.py b/contrib/tools/cython/Cython/Compiler/Parsing.py
index 14110fcc06..4d2f12a24a 100644
--- a/contrib/tools/cython/Cython/Compiler/Parsing.py
+++ b/contrib/tools/cython/Cython/Compiler/Parsing.py
@@ -8,37 +8,37 @@ from __future__ import absolute_import
# This should be done automatically
import cython
cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
- bytes_literal=object, StringEncoding=object,
+ bytes_literal=object, StringEncoding=object,
FileSourceDescriptor=object, lookup_unicodechar=object, unicode_category=object,
Future=object, Options=object, error=object, warning=object,
Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object,
re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object,
- partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint,
- _CDEF_MODIFIERS=tuple)
+ partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint,
+ _CDEF_MODIFIERS=tuple)
-from io import StringIO
+from io import StringIO
import re
-import sys
+import sys
from unicodedata import lookup as lookup_unicodechar, category as unicode_category
-from functools import partial, reduce
+from functools import partial, reduce
-from .Scanning import PyrexScanner, FileSourceDescriptor, StringSourceDescriptor
+from .Scanning import PyrexScanner, FileSourceDescriptor, StringSourceDescriptor
from . import Nodes
from . import ExprNodes
from . import Builtin
from . import StringEncoding
-from .StringEncoding import EncodedString, bytes_literal, _unicode, _bytes
+from .StringEncoding import EncodedString, bytes_literal, _unicode, _bytes
from .ModuleNode import ModuleNode
from .Errors import error, warning
from .. import Utils
from . import Future
from . import Options
-_IS_PY3 = sys.version_info[0] >= 3
+_IS_PY3 = sys.version_info[0] >= 3
_IS_2BYTE_UNICODE = sys.maxunicode == 0xffff
-_CDEF_MODIFIERS = ('inline', 'nogil', 'api')
+_CDEF_MODIFIERS = ('inline', 'nogil', 'api')
+
-
class Ctx(object):
# Parsing context
level = 'other'
@@ -62,8 +62,8 @@ class Ctx(object):
d.update(kwds)
return ctx
-
-def p_ident(s, message="Expected an identifier"):
+
+def p_ident(s, message="Expected an identifier"):
if s.sy == 'IDENT':
name = s.systring
s.next()
@@ -218,7 +218,7 @@ def p_starred_expr(s):
starred = False
expr = p_bit_expr(s)
if starred:
- expr = ExprNodes.StarredUnpackingNode(pos, expr)
+ expr = ExprNodes.StarredUnpackingNode(pos, expr)
return expr
def p_cascaded_cmp(s):
@@ -358,7 +358,7 @@ def p_sizeof(s):
s.expect(')')
return node
-
+
def p_yield_expression(s):
# s.sy == "yield"
pos = s.position()
@@ -368,8 +368,8 @@ def p_yield_expression(s):
is_yield_from = True
s.next()
if s.sy != ')' and s.sy not in statement_terminators:
- # "yield from" does not support implicit tuples, but "yield" does ("yield 1,2")
- arg = p_test(s) if is_yield_from else p_testlist(s)
+ # "yield from" does not support implicit tuples, but "yield" does ("yield 1,2")
+ arg = p_test(s) if is_yield_from else p_testlist(s)
else:
if is_yield_from:
s.error("'yield from' requires a source argument",
@@ -380,47 +380,47 @@ def p_yield_expression(s):
else:
return ExprNodes.YieldExprNode(pos, arg=arg)
-
+
def p_yield_statement(s):
# s.sy == "yield"
yield_expr = p_yield_expression(s)
return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
-def p_async_statement(s, ctx, decorators):
- # s.sy >> 'async' ...
- if s.sy == 'def':
- # 'async def' statements aren't allowed in pxd files
- if 'pxd' in ctx.level:
- s.error('def statement not allowed here')
- s.level = ctx.level
- return p_def_statement(s, decorators, is_async_def=True)
- elif decorators:
- s.error("Decorators can only be followed by functions or classes")
- elif s.sy == 'for':
- return p_for_statement(s, is_async=True)
- elif s.sy == 'with':
- s.next()
- return p_with_items(s, is_async=True)
- else:
- s.error("expected one of 'def', 'for', 'with' after 'async'")
-
-
-#power: atom_expr ('**' factor)*
-#atom_expr: ['await'] atom trailer*
-
+def p_async_statement(s, ctx, decorators):
+ # s.sy >> 'async' ...
+ if s.sy == 'def':
+ # 'async def' statements aren't allowed in pxd files
+ if 'pxd' in ctx.level:
+ s.error('def statement not allowed here')
+ s.level = ctx.level
+ return p_def_statement(s, decorators, is_async_def=True)
+ elif decorators:
+ s.error("Decorators can only be followed by functions or classes")
+ elif s.sy == 'for':
+ return p_for_statement(s, is_async=True)
+ elif s.sy == 'with':
+ s.next()
+ return p_with_items(s, is_async=True)
+ else:
+ s.error("expected one of 'def', 'for', 'with' after 'async'")
+
+
+#power: atom_expr ('**' factor)*
+#atom_expr: ['await'] atom trailer*
+
def p_power(s):
if s.systring == 'new' and s.peek()[0] == 'IDENT':
return p_new_expr(s)
- await_pos = None
- if s.sy == 'await':
- await_pos = s.position()
- s.next()
+ await_pos = None
+ if s.sy == 'await':
+ await_pos = s.position()
+ s.next()
n1 = p_atom(s)
while s.sy in ('(', '[', '.'):
n1 = p_trailer(s, n1)
- if await_pos:
- n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1)
+ if await_pos:
+ n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1)
if s.sy == '**':
pos = s.position()
s.next()
@@ -428,7 +428,7 @@ def p_power(s):
n1 = ExprNodes.binop_node(pos, '**', n1, n2)
return n1
-
+
def p_new_expr(s):
# s.systring == 'new'.
pos = s.position()
@@ -446,39 +446,39 @@ def p_trailer(s, node1):
return p_index(s, node1)
else: # s.sy == '.'
s.next()
- name = p_ident(s)
+ name = p_ident(s)
return ExprNodes.AttributeNode(pos,
- obj=node1, attribute=name)
+ obj=node1, attribute=name)
+
-
# arglist: argument (',' argument)* [',']
# argument: [test '='] test # Really [keyword '='] test
-# since PEP 448:
-# argument: ( test [comp_for] |
-# test '=' test |
-# '**' expr |
-# star_expr )
-
-def p_call_parse_args(s, allow_genexp=True):
+# since PEP 448:
+# argument: ( test [comp_for] |
+# test '=' test |
+# '**' expr |
+# star_expr )
+
+def p_call_parse_args(s, allow_genexp=True):
# s.sy == '('
pos = s.position()
s.next()
positional_args = []
keyword_args = []
- starstar_seen = False
- last_was_tuple_unpack = False
- while s.sy != ')':
+ starstar_seen = False
+ last_was_tuple_unpack = False
+ while s.sy != ')':
if s.sy == '*':
- if starstar_seen:
- s.error("Non-keyword arg following keyword arg", pos=s.position())
+ if starstar_seen:
+ s.error("Non-keyword arg following keyword arg", pos=s.position())
+ s.next()
+ positional_args.append(p_test(s))
+ last_was_tuple_unpack = True
+ elif s.sy == '**':
s.next()
- positional_args.append(p_test(s))
- last_was_tuple_unpack = True
- elif s.sy == '**':
- s.next()
- keyword_args.append(p_test(s))
- starstar_seen = True
+ keyword_args.append(p_test(s))
+ starstar_seen = True
else:
arg = p_test(s)
if s.sy == '=':
@@ -486,86 +486,86 @@ def p_call_parse_args(s, allow_genexp=True):
if not arg.is_name:
s.error("Expected an identifier before '='",
pos=arg.pos)
- encoded_name = s.context.intern_ustring(arg.name)
+ encoded_name = s.context.intern_ustring(arg.name)
keyword = ExprNodes.IdentifierStringNode(
arg.pos, value=encoded_name)
arg = p_test(s)
keyword_args.append((keyword, arg))
else:
if keyword_args:
- s.error("Non-keyword arg following keyword arg", pos=arg.pos)
- if positional_args and not last_was_tuple_unpack:
- positional_args[-1].append(arg)
- else:
- positional_args.append([arg])
- last_was_tuple_unpack = False
+ s.error("Non-keyword arg following keyword arg", pos=arg.pos)
+ if positional_args and not last_was_tuple_unpack:
+ positional_args[-1].append(arg)
+ else:
+ positional_args.append([arg])
+ last_was_tuple_unpack = False
if s.sy != ',':
break
s.next()
if s.sy in ('for', 'async'):
- if not keyword_args and not last_was_tuple_unpack:
- if len(positional_args) == 1 and len(positional_args[0]) == 1:
- positional_args = [[p_genexp(s, positional_args[0][0])]]
+ if not keyword_args and not last_was_tuple_unpack:
+ if len(positional_args) == 1 and len(positional_args[0]) == 1:
+ positional_args = [[p_genexp(s, positional_args[0][0])]]
s.expect(')')
- return positional_args or [[]], keyword_args
+ return positional_args or [[]], keyword_args
+
-
-def p_call_build_packed_args(pos, positional_args, keyword_args):
+def p_call_build_packed_args(pos, positional_args, keyword_args):
keyword_dict = None
-
- subtuples = [
- ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg)
- for arg in positional_args
- ]
- # TODO: implement a faster way to join tuples than creating each one and adding them
- arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples)
-
- if keyword_args:
- kwargs = []
- dict_items = []
- for item in keyword_args:
- if isinstance(item, tuple):
- key, value = item
- dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value))
- elif item.is_dict_literal:
- # unpack "**{a:b}" directly
- dict_items.extend(item.key_value_pairs)
- else:
- if dict_items:
- kwargs.append(ExprNodes.DictNode(
- dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
- dict_items = []
- kwargs.append(item)
-
- if dict_items:
- kwargs.append(ExprNodes.DictNode(
- dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
-
- if kwargs:
- if len(kwargs) == 1 and kwargs[0].is_dict_literal:
- # only simple keyword arguments found -> one dict
- keyword_dict = kwargs[0]
- else:
- # at least one **kwargs
- keyword_dict = ExprNodes.MergedDictNode(pos, keyword_args=kwargs)
-
+
+ subtuples = [
+ ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg)
+ for arg in positional_args
+ ]
+ # TODO: implement a faster way to join tuples than creating each one and adding them
+ arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples)
+
+ if keyword_args:
+ kwargs = []
+ dict_items = []
+ for item in keyword_args:
+ if isinstance(item, tuple):
+ key, value = item
+ dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value))
+ elif item.is_dict_literal:
+ # unpack "**{a:b}" directly
+ dict_items.extend(item.key_value_pairs)
+ else:
+ if dict_items:
+ kwargs.append(ExprNodes.DictNode(
+ dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
+ dict_items = []
+ kwargs.append(item)
+
+ if dict_items:
+ kwargs.append(ExprNodes.DictNode(
+ dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
+
+ if kwargs:
+ if len(kwargs) == 1 and kwargs[0].is_dict_literal:
+ # only simple keyword arguments found -> one dict
+ keyword_dict = kwargs[0]
+ else:
+ # at least one **kwargs
+ keyword_dict = ExprNodes.MergedDictNode(pos, keyword_args=kwargs)
+
return arg_tuple, keyword_dict
-
+
def p_call(s, function):
# s.sy == '('
pos = s.position()
- positional_args, keyword_args = p_call_parse_args(s)
+ positional_args, keyword_args = p_call_parse_args(s)
- if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list):
- return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0])
+ if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list):
+ return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0])
else:
- arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
- return ExprNodes.GeneralCallNode(
- pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict)
+ arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
+ return ExprNodes.GeneralCallNode(
+ pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict)
+
-
#lambdef: 'lambda' [varargslist] ':' test
#subscriptlist: subscript (',' subscript)* [',']
@@ -697,14 +697,14 @@ def p_atom(s):
return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value)
elif kind == 'b':
return ExprNodes.BytesNode(pos, value = bytes_value)
- elif kind == 'f':
- return ExprNodes.JoinedStrNode(pos, values = unicode_value)
- elif kind == '':
- return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
+ elif kind == 'f':
+ return ExprNodes.JoinedStrNode(pos, values = unicode_value)
+ elif kind == '':
+ return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
else:
- s.error("invalid string kind '%s'" % kind)
+ s.error("invalid string kind '%s'" % kind)
elif sy == 'IDENT':
- name = s.systring
+ name = s.systring
if name == "None":
result = ExprNodes.NoneNode(pos)
elif name == "True":
@@ -771,9 +771,9 @@ def wrap_compile_time_constant(pos, value):
elif isinstance(value, bool):
return ExprNodes.BoolNode(pos, value=value)
elif isinstance(value, int):
- return ExprNodes.IntNode(pos, value=rep, constant_result=value)
+ return ExprNodes.IntNode(pos, value=rep, constant_result=value)
elif isinstance(value, float):
- return ExprNodes.FloatNode(pos, value=rep, constant_result=value)
+ return ExprNodes.FloatNode(pos, value=rep, constant_result=value)
elif isinstance(value, complex):
node = ExprNodes.ImagNode(pos, value=repr(value.imag), constant_result=complex(0.0, value.imag))
if value.real:
@@ -786,8 +786,8 @@ def wrap_compile_time_constant(pos, value):
elif isinstance(value, _unicode):
return ExprNodes.UnicodeNode(pos, value=EncodedString(value))
elif isinstance(value, _bytes):
- bvalue = bytes_literal(value, 'ascii') # actually: unknown encoding, but BytesLiteral requires one
- return ExprNodes.BytesNode(pos, value=bvalue, constant_result=value)
+ bvalue = bytes_literal(value, 'ascii') # actually: unknown encoding, but BytesLiteral requires one
+ return ExprNodes.BytesNode(pos, value=bvalue, constant_result=value)
elif isinstance(value, tuple):
args = [wrap_compile_time_constant(pos, arg)
for arg in value]
@@ -796,8 +796,8 @@ def wrap_compile_time_constant(pos, value):
else:
# error already reported
return None
- elif not _IS_PY3 and isinstance(value, long):
- return ExprNodes.IntNode(pos, value=rep.rstrip('L'), constant_result=value)
+ elif not _IS_PY3 and isinstance(value, long):
+ return ExprNodes.IntNode(pos, value=rep.rstrip('L'), constant_result=value)
error(pos, "Invalid type for compile-time constant: %r (type %s)"
% (value, value.__class__.__name__))
return None
@@ -806,84 +806,84 @@ def wrap_compile_time_constant(pos, value):
def p_cat_string_literal(s):
# A sequence of one or more adjacent string literals.
# Returns (kind, bytes_value, unicode_value)
- # where kind in ('b', 'c', 'u', 'f', '')
- pos = s.position()
+ # where kind in ('b', 'c', 'u', 'f', '')
+ pos = s.position()
kind, bytes_value, unicode_value = p_string_literal(s)
if kind == 'c' or s.sy != 'BEGIN_STRING':
return kind, bytes_value, unicode_value
- bstrings, ustrings, positions = [bytes_value], [unicode_value], [pos]
+ bstrings, ustrings, positions = [bytes_value], [unicode_value], [pos]
bytes_value = unicode_value = None
while s.sy == 'BEGIN_STRING':
pos = s.position()
next_kind, next_bytes_value, next_unicode_value = p_string_literal(s)
if next_kind == 'c':
error(pos, "Cannot concatenate char literal with another string or char literal")
- continue
+ continue
elif next_kind != kind:
- # concatenating f strings and normal strings is allowed and leads to an f string
- if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])):
- kind = 'f'
- else:
+ # concatenating f strings and normal strings is allowed and leads to an f string
+ if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])):
+ kind = 'f'
+ else:
error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % (
kind, next_kind))
- continue
- bstrings.append(next_bytes_value)
- ustrings.append(next_unicode_value)
- positions.append(pos)
+ continue
+ bstrings.append(next_bytes_value)
+ ustrings.append(next_unicode_value)
+ positions.append(pos)
# join and rewrap the partial literals
if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings:
# Py3 enforced unicode literals are parsed as bytes/unicode combination
- bytes_value = bytes_literal(StringEncoding.join_bytes(bstrings), s.source_encoding)
+ bytes_value = bytes_literal(StringEncoding.join_bytes(bstrings), s.source_encoding)
if kind in ('u', ''):
- unicode_value = EncodedString(u''.join([u for u in ustrings if u is not None]))
- if kind == 'f':
- unicode_value = []
- for u, pos in zip(ustrings, positions):
- if isinstance(u, list):
- unicode_value += u
- else:
- # non-f-string concatenated into the f-string
- unicode_value.append(ExprNodes.UnicodeNode(pos, value=EncodedString(u)))
+ unicode_value = EncodedString(u''.join([u for u in ustrings if u is not None]))
+ if kind == 'f':
+ unicode_value = []
+ for u, pos in zip(ustrings, positions):
+ if isinstance(u, list):
+ unicode_value += u
+ else:
+ # non-f-string concatenated into the f-string
+ unicode_value.append(ExprNodes.UnicodeNode(pos, value=EncodedString(u)))
return kind, bytes_value, unicode_value
-
+
def p_opt_string_literal(s, required_type='u'):
- if s.sy != 'BEGIN_STRING':
- return None
- pos = s.position()
- kind, bytes_value, unicode_value = p_string_literal(s, required_type)
- if required_type == 'u':
- if kind == 'f':
- s.error("f-string not allowed here", pos)
- return unicode_value
- elif required_type == 'b':
- return bytes_value
- else:
- s.error("internal parser configuration error")
-
-
+ if s.sy != 'BEGIN_STRING':
+ return None
+ pos = s.position()
+ kind, bytes_value, unicode_value = p_string_literal(s, required_type)
+ if required_type == 'u':
+ if kind == 'f':
+ s.error("f-string not allowed here", pos)
+ return unicode_value
+ elif required_type == 'b':
+ return bytes_value
+ else:
+ s.error("internal parser configuration error")
+
+
def check_for_non_ascii_characters(string):
for c in string:
if c >= u'\x80':
return True
return False
-
+
def p_string_literal(s, kind_override=None):
# A single string or char literal. Returns (kind, bvalue, uvalue)
- # where kind in ('b', 'c', 'u', 'f', ''). The 'bvalue' is the source
+ # where kind in ('b', 'c', 'u', 'f', ''). The 'bvalue' is the source
# code byte sequence of the string literal, 'uvalue' is the
# decoded Unicode string. Either of the two may be None depending
# on the 'kind' of string, only unprefixed strings have both
- # representations. In f-strings, the uvalue is a list of the Unicode
- # strings and f-string expressions that make up the f-string.
+ # representations. In f-strings, the uvalue is a list of the Unicode
+ # strings and f-string expressions that make up the f-string.
# s.sy == 'BEGIN_STRING'
pos = s.position()
is_python3_source = s.context.language_level >= 3
- has_non_ascii_literal_characters = False
+ has_non_ascii_literal_characters = False
string_start_pos = (pos[0], pos[1], pos[2] + len(s.systring))
- kind_string = s.systring.rstrip('"\'').lower()
+ kind_string = s.systring.rstrip('"\'').lower()
if len(kind_string) > 1:
if len(set(kind_string)) != len(kind_string):
error(pos, 'Duplicate string prefix character')
@@ -893,32 +893,32 @@ def p_string_literal(s, kind_override=None):
error(pos, 'String prefixes b and f cannot be combined')
if 'u' in kind_string and 'f' in kind_string:
error(pos, 'String prefixes u and f cannot be combined')
-
- is_raw = 'r' in kind_string
-
- if 'c' in kind_string:
- # this should never happen, since the lexer does not allow combining c
- # with other prefix characters
- if len(kind_string) != 1:
+
+ is_raw = 'r' in kind_string
+
+ if 'c' in kind_string:
+ # this should never happen, since the lexer does not allow combining c
+ # with other prefix characters
+ if len(kind_string) != 1:
error(pos, 'Invalid string prefix for character literal')
- kind = 'c'
- elif 'f' in kind_string:
+ kind = 'c'
+ elif 'f' in kind_string:
kind = 'f' # u is ignored
is_raw = True # postpone the escape resolution
- elif 'b' in kind_string:
- kind = 'b'
- elif 'u' in kind_string:
- kind = 'u'
- else:
+ elif 'b' in kind_string:
+ kind = 'b'
+ elif 'u' in kind_string:
+ kind = 'u'
+ else:
kind = ''
-
+
if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives:
chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
kind = 'u'
else:
if kind_override is not None and kind_override in 'ub':
kind = kind_override
- if kind in ('u', 'f'): # f-strings are scanned exactly like Unicode literals, but are parsed further later
+ if kind in ('u', 'f'): # f-strings are scanned exactly like Unicode literals, but are parsed further later
chars = StringEncoding.UnicodeLiteralBuilder()
elif kind == '':
chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
@@ -929,17 +929,17 @@ def p_string_literal(s, kind_override=None):
s.next()
sy = s.sy
systr = s.systring
- # print "p_string_literal: sy =", sy, repr(s.systring) ###
+ # print "p_string_literal: sy =", sy, repr(s.systring) ###
if sy == 'CHARS':
chars.append(systr)
- if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
- has_non_ascii_literal_characters = True
+ if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
+ has_non_ascii_literal_characters = True
elif sy == 'ESCAPE':
# in Py2, 'ur' raw unicode strings resolve unicode escapes but nothing else
if is_raw and (is_python3_source or kind != 'u' or systr[1] not in u'Uu'):
chars.append(systr)
if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
- has_non_ascii_literal_characters = True
+ has_non_ascii_literal_characters = True
else:
_append_escape_sequence(kind, chars, systr, s)
elif sy == 'NEWLINE':
@@ -959,18 +959,18 @@ def p_string_literal(s, kind_override=None):
error(pos, u"invalid character literal: %r" % bytes_value)
else:
bytes_value, unicode_value = chars.getstrings()
- if (has_non_ascii_literal_characters
- and is_python3_source and Future.unicode_literals in s.context.future_directives):
+ if (has_non_ascii_literal_characters
+ and is_python3_source and Future.unicode_literals in s.context.future_directives):
# Python 3 forbids literal non-ASCII characters in byte strings
- if kind == 'b':
+ if kind == 'b':
s.error("bytes can only contain ASCII literal characters.", pos=pos)
bytes_value = None
- if kind == 'f':
+ if kind == 'f':
unicode_value = p_f_string(s, unicode_value, string_start_pos, is_raw='r' in kind_string)
s.next()
return (kind, bytes_value, unicode_value)
-
+
def _append_escape_sequence(kind, builder, escape_sequence, s):
c = escape_sequence[1]
if c in u"01234567":
@@ -1043,11 +1043,11 @@ def _f_string_error_pos(pos, string, i):
def p_f_string(s, unicode_value, pos, is_raw):
- # Parses a PEP 498 f-string literal into a list of nodes. Nodes are either UnicodeNodes
- # or FormattedValueNodes.
- values = []
+ # Parses a PEP 498 f-string literal into a list of nodes. Nodes are either UnicodeNodes
+ # or FormattedValueNodes.
+ values = []
next_start = 0
- size = len(unicode_value)
+ size = len(unicode_value)
builder = StringEncoding.UnicodeLiteralBuilder()
_parse_seq = _parse_escape_sequences_raw if is_raw else _parse_escape_sequences
@@ -1063,7 +1063,7 @@ def p_f_string(s, unicode_value, pos, is_raw):
if c == '\\':
if not is_raw and len(part) > 1:
_append_escape_sequence('f', builder, part, s)
- else:
+ else:
builder.append(part)
elif c == '{':
if part == '{{':
@@ -1074,150 +1074,150 @@ def p_f_string(s, unicode_value, pos, is_raw):
values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring()))
builder = StringEncoding.UnicodeLiteralBuilder()
next_start, expr_node = p_f_string_expr(s, unicode_value, pos, next_start, is_raw)
- values.append(expr_node)
+ values.append(expr_node)
elif c == '}':
if part == '}}':
builder.append('}')
else:
error(_f_string_error_pos(pos, unicode_value, end),
"f-string: single '}' is not allowed")
- else:
+ else:
builder.append(part)
-
+
if builder.chars:
values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring()))
- return values
-
-
+ return values
+
+
def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw):
- # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode
- # and the index in the string that follows the expression.
- i = starting_index
- size = len(unicode_value)
- conversion_char = terminal_char = format_spec = None
- format_spec_str = None
- NO_CHAR = 2**30
-
- nested_depth = 0
- quote_char = NO_CHAR
- in_triple_quotes = False
+ # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode
+ # and the index in the string that follows the expression.
+ i = starting_index
+ size = len(unicode_value)
+ conversion_char = terminal_char = format_spec = None
+ format_spec_str = None
+ NO_CHAR = 2**30
+
+ nested_depth = 0
+ quote_char = NO_CHAR
+ in_triple_quotes = False
backslash_reported = False
-
- while True:
- if i >= size:
+
+ while True:
+ if i >= size:
break # error will be reported below
- c = unicode_value[i]
-
- if quote_char != NO_CHAR:
- if c == '\\':
+ c = unicode_value[i]
+
+ if quote_char != NO_CHAR:
+ if c == '\\':
# avoid redundant error reports along '\' sequences
if not backslash_reported:
error(_f_string_error_pos(pos, unicode_value, i),
"backslashes not allowed in f-strings")
backslash_reported = True
- elif c == quote_char:
- if in_triple_quotes:
- if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
- in_triple_quotes = False
- quote_char = NO_CHAR
- i += 2
- else:
- quote_char = NO_CHAR
- elif c in '\'"':
- quote_char = c
- if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
- in_triple_quotes = True
- i += 2
- elif c in '{[(':
- nested_depth += 1
- elif nested_depth != 0 and c in '}])':
- nested_depth -= 1
- elif c == '#':
+ elif c == quote_char:
+ if in_triple_quotes:
+ if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
+ in_triple_quotes = False
+ quote_char = NO_CHAR
+ i += 2
+ else:
+ quote_char = NO_CHAR
+ elif c in '\'"':
+ quote_char = c
+ if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
+ in_triple_quotes = True
+ i += 2
+ elif c in '{[(':
+ nested_depth += 1
+ elif nested_depth != 0 and c in '}])':
+ nested_depth -= 1
+ elif c == '#':
error(_f_string_error_pos(pos, unicode_value, i),
"format string cannot include #")
- elif nested_depth == 0 and c in '!:}':
- # allow != as a special case
- if c == '!' and i + 1 < size and unicode_value[i + 1] == '=':
- i += 1
- continue
-
- terminal_char = c
- break
- i += 1
-
- # normalise line endings as the parser expects that
- expr_str = unicode_value[starting_index:i].replace('\r\n', '\n').replace('\r', '\n')
- expr_pos = (pos[0], pos[1], pos[2] + starting_index + 2) # TODO: find exact code position (concat, multi-line, ...)
-
- if not expr_str.strip():
+ elif nested_depth == 0 and c in '!:}':
+ # allow != as a special case
+ if c == '!' and i + 1 < size and unicode_value[i + 1] == '=':
+ i += 1
+ continue
+
+ terminal_char = c
+ break
+ i += 1
+
+ # normalise line endings as the parser expects that
+ expr_str = unicode_value[starting_index:i].replace('\r\n', '\n').replace('\r', '\n')
+ expr_pos = (pos[0], pos[1], pos[2] + starting_index + 2) # TODO: find exact code position (concat, multi-line, ...)
+
+ if not expr_str.strip():
error(_f_string_error_pos(pos, unicode_value, starting_index),
"empty expression not allowed in f-string")
-
- if terminal_char == '!':
- i += 1
- if i + 2 > size:
+
+ if terminal_char == '!':
+ i += 1
+ if i + 2 > size:
pass # error will be reported below
else:
conversion_char = unicode_value[i]
i += 1
terminal_char = unicode_value[i]
-
- if terminal_char == ':':
- in_triple_quotes = False
- in_string = False
- nested_depth = 0
- start_format_spec = i + 1
- while True:
- if i >= size:
+
+ if terminal_char == ':':
+ in_triple_quotes = False
+ in_string = False
+ nested_depth = 0
+ start_format_spec = i + 1
+ while True:
+ if i >= size:
break # error will be reported below
- c = unicode_value[i]
- if not in_triple_quotes and not in_string:
- if c == '{':
- nested_depth += 1
- elif c == '}':
- if nested_depth > 0:
- nested_depth -= 1
- else:
- terminal_char = c
- break
- if c in '\'"':
- if not in_string and i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
- in_triple_quotes = not in_triple_quotes
- i += 2
- elif not in_triple_quotes:
- in_string = not in_string
- i += 1
-
- format_spec_str = unicode_value[start_format_spec:i]
-
- if terminal_char != '}':
+ c = unicode_value[i]
+ if not in_triple_quotes and not in_string:
+ if c == '{':
+ nested_depth += 1
+ elif c == '}':
+ if nested_depth > 0:
+ nested_depth -= 1
+ else:
+ terminal_char = c
+ break
+ if c in '\'"':
+ if not in_string and i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
+ in_triple_quotes = not in_triple_quotes
+ i += 2
+ elif not in_triple_quotes:
+ in_string = not in_string
+ i += 1
+
+ format_spec_str = unicode_value[start_format_spec:i]
+
+ if terminal_char != '}':
error(_f_string_error_pos(pos, unicode_value, i),
"missing '}' in format string expression" + (
", found '%s'" % terminal_char if terminal_char else ""))
-
- # parse the expression as if it was surrounded by parentheses
- buf = StringIO('(%s)' % expr_str)
- scanner = PyrexScanner(buf, expr_pos[0], parent_scanner=s, source_encoding=s.source_encoding, initial_pos=expr_pos)
- expr = p_testlist(scanner) # TODO is testlist right here?
-
- # validate the conversion char
- if conversion_char is not None and not ExprNodes.FormattedValueNode.find_conversion_func(conversion_char):
+
+ # parse the expression as if it was surrounded by parentheses
+ buf = StringIO('(%s)' % expr_str)
+ scanner = PyrexScanner(buf, expr_pos[0], parent_scanner=s, source_encoding=s.source_encoding, initial_pos=expr_pos)
+ expr = p_testlist(scanner) # TODO is testlist right here?
+
+ # validate the conversion char
+ if conversion_char is not None and not ExprNodes.FormattedValueNode.find_conversion_func(conversion_char):
error(expr_pos, "invalid conversion character '%s'" % conversion_char)
-
- # the format spec is itself treated like an f-string
- if format_spec_str:
+
+ # the format spec is itself treated like an f-string
+ if format_spec_str:
format_spec = ExprNodes.JoinedStrNode(pos, values=p_f_string(s, format_spec_str, pos, is_raw))
-
- return i + 1, ExprNodes.FormattedValueNode(
+
+ return i + 1, ExprNodes.FormattedValueNode(
pos, value=expr, conversion_char=conversion_char, format_spec=format_spec)
-
-
-# since PEP 448:
-# list_display ::= "[" [listmaker] "]"
-# listmaker ::= (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+
+
+# since PEP 448:
+# list_display ::= "[" [listmaker] "]"
+# listmaker ::= (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
# comp_iter ::= comp_for | comp_if
# comp_for ::= ["async"] "for" expression_list "in" testlist [comp_iter]
-# comp_if ::= "if" test [comp_iter]
+# comp_if ::= "if" test [comp_iter]
def p_list_maker(s):
# s.sy == '['
@@ -1225,30 +1225,30 @@ def p_list_maker(s):
s.next()
if s.sy == ']':
s.expect(']')
- return ExprNodes.ListNode(pos, args=[])
-
- expr = p_test_or_starred_expr(s)
+ return ExprNodes.ListNode(pos, args=[])
+
+ expr = p_test_or_starred_expr(s)
if s.sy in ('for', 'async'):
- if expr.is_starred:
- s.error("iterable unpacking cannot be used in comprehension")
+ if expr.is_starred:
+ s.error("iterable unpacking cannot be used in comprehension")
append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
loop = p_comp_for(s, append)
s.expect(']')
return ExprNodes.ComprehensionNode(
- pos, loop=loop, append=append, type=Builtin.list_type,
+ pos, loop=loop, append=append, type=Builtin.list_type,
# list comprehensions leak their loop variable in Py2
- has_local_scope=s.context.language_level >= 3)
-
- # (merged) list literal
- if s.sy == ',':
- s.next()
- exprs = p_test_or_starred_expr_list(s, expr)
- else:
- exprs = [expr]
- s.expect(']')
- return ExprNodes.ListNode(pos, args=exprs)
-
-
+ has_local_scope=s.context.language_level >= 3)
+
+ # (merged) list literal
+ if s.sy == ',':
+ s.next()
+ exprs = p_test_or_starred_expr_list(s, expr)
+ else:
+ exprs = [expr]
+ s.expect(']')
+ return ExprNodes.ListNode(pos, args=exprs)
+
+
def p_comp_iter(s, body):
if s.sy in ('for', 'async'):
return p_comp_for(s, body)
@@ -1283,121 +1283,121 @@ def p_comp_if(s, body):
else_clause = None )
-# since PEP 448:
-#dictorsetmaker: ( ((test ':' test | '**' expr)
-# (comp_for | (',' (test ':' test | '**' expr))* [','])) |
-# ((test | star_expr)
-# (comp_for | (',' (test | star_expr))* [','])) )
-
+# since PEP 448:
+#dictorsetmaker: ( ((test ':' test | '**' expr)
+# (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+# ((test | star_expr)
+# (comp_for | (',' (test | star_expr))* [','])) )
+
def p_dict_or_set_maker(s):
# s.sy == '{'
pos = s.position()
s.next()
if s.sy == '}':
s.next()
- return ExprNodes.DictNode(pos, key_value_pairs=[])
-
- parts = []
- target_type = 0
- last_was_simple_item = False
- while True:
- if s.sy in ('*', '**'):
- # merged set/dict literal
- if target_type == 0:
- target_type = 1 if s.sy == '*' else 2 # 'stars'
- elif target_type != len(s.sy):
- s.error("unexpected %sitem found in %s literal" % (
- s.sy, 'set' if target_type == 1 else 'dict'))
+ return ExprNodes.DictNode(pos, key_value_pairs=[])
+
+ parts = []
+ target_type = 0
+ last_was_simple_item = False
+ while True:
+ if s.sy in ('*', '**'):
+ # merged set/dict literal
+ if target_type == 0:
+ target_type = 1 if s.sy == '*' else 2 # 'stars'
+ elif target_type != len(s.sy):
+ s.error("unexpected %sitem found in %s literal" % (
+ s.sy, 'set' if target_type == 1 else 'dict'))
+ s.next()
+ if s.sy == '*':
+ s.error("expected expression, found '*'")
+ item = p_starred_expr(s)
+ parts.append(item)
+ last_was_simple_item = False
+ else:
+ item = p_test(s)
+ if target_type == 0:
+ target_type = 2 if s.sy == ':' else 1 # dict vs. set
+ if target_type == 2:
+ # dict literal
+ s.expect(':')
+ key = item
+ value = p_test(s)
+ item = ExprNodes.DictItemNode(key.pos, key=key, value=value)
+ if last_was_simple_item:
+ parts[-1].append(item)
+ else:
+ parts.append([item])
+ last_was_simple_item = True
+
+ if s.sy == ',':
s.next()
- if s.sy == '*':
- s.error("expected expression, found '*'")
- item = p_starred_expr(s)
- parts.append(item)
- last_was_simple_item = False
- else:
- item = p_test(s)
- if target_type == 0:
- target_type = 2 if s.sy == ':' else 1 # dict vs. set
- if target_type == 2:
- # dict literal
- s.expect(':')
- key = item
- value = p_test(s)
- item = ExprNodes.DictItemNode(key.pos, key=key, value=value)
- if last_was_simple_item:
- parts[-1].append(item)
- else:
- parts.append([item])
- last_was_simple_item = True
-
- if s.sy == ',':
- s.next()
if s.sy == '}':
break
- else:
- break
-
+ else:
+ break
+
if s.sy in ('for', 'async'):
- # dict/set comprehension
- if len(parts) == 1 and isinstance(parts[0], list) and len(parts[0]) == 1:
- item = parts[0][0]
- if target_type == 2:
- assert isinstance(item, ExprNodes.DictItemNode), type(item)
- comprehension_type = Builtin.dict_type
- append = ExprNodes.DictComprehensionAppendNode(
- item.pos, key_expr=item.key, value_expr=item.value)
- else:
- comprehension_type = Builtin.set_type
- append = ExprNodes.ComprehensionAppendNode(item.pos, expr=item)
+ # dict/set comprehension
+ if len(parts) == 1 and isinstance(parts[0], list) and len(parts[0]) == 1:
+ item = parts[0][0]
+ if target_type == 2:
+ assert isinstance(item, ExprNodes.DictItemNode), type(item)
+ comprehension_type = Builtin.dict_type
+ append = ExprNodes.DictComprehensionAppendNode(
+ item.pos, key_expr=item.key, value_expr=item.value)
+ else:
+ comprehension_type = Builtin.set_type
+ append = ExprNodes.ComprehensionAppendNode(item.pos, expr=item)
loop = p_comp_for(s, append)
s.expect('}')
- return ExprNodes.ComprehensionNode(pos, loop=loop, append=append, type=comprehension_type)
+ return ExprNodes.ComprehensionNode(pos, loop=loop, append=append, type=comprehension_type)
else:
- # syntax error, try to find a good error message
- if len(parts) == 1 and not isinstance(parts[0], list):
- s.error("iterable unpacking cannot be used in comprehension")
- else:
- # e.g. "{1,2,3 for ..."
- s.expect('}')
- return ExprNodes.DictNode(pos, key_value_pairs=[])
-
- s.expect('}')
- if target_type == 1:
- # (merged) set literal
- items = []
- set_items = []
- for part in parts:
- if isinstance(part, list):
- set_items.extend(part)
- else:
- if set_items:
- items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
- set_items = []
- items.append(part)
- if set_items:
- items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
- if len(items) == 1 and items[0].is_set_literal:
- return items[0]
- return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
- else:
- # (merged) dict literal
- items = []
- dict_items = []
- for part in parts:
- if isinstance(part, list):
- dict_items.extend(part)
- else:
- if dict_items:
- items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
- dict_items = []
- items.append(part)
- if dict_items:
- items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
- if len(items) == 1 and items[0].is_dict_literal:
- return items[0]
- return ExprNodes.MergedDictNode(pos, keyword_args=items, reject_duplicates=False)
-
-
+ # syntax error, try to find a good error message
+ if len(parts) == 1 and not isinstance(parts[0], list):
+ s.error("iterable unpacking cannot be used in comprehension")
+ else:
+ # e.g. "{1,2,3 for ..."
+ s.expect('}')
+ return ExprNodes.DictNode(pos, key_value_pairs=[])
+
+ s.expect('}')
+ if target_type == 1:
+ # (merged) set literal
+ items = []
+ set_items = []
+ for part in parts:
+ if isinstance(part, list):
+ set_items.extend(part)
+ else:
+ if set_items:
+ items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
+ set_items = []
+ items.append(part)
+ if set_items:
+ items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
+ if len(items) == 1 and items[0].is_set_literal:
+ return items[0]
+ return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
+ else:
+ # (merged) dict literal
+ items = []
+ dict_items = []
+ for part in parts:
+ if isinstance(part, list):
+ dict_items.extend(part)
+ else:
+ if dict_items:
+ items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
+ dict_items = []
+ items.append(part)
+ if dict_items:
+ items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
+ if len(items) == 1 and items[0].is_dict_literal:
+ return items[0]
+ return ExprNodes.MergedDictNode(pos, keyword_args=items, reject_duplicates=False)
+
+
# NOTE: no longer in Py3 :)
def p_backquote_expr(s):
# s.sy == '`'
@@ -1423,11 +1423,11 @@ def p_simple_expr_list(s, expr=None):
s.next()
return exprs
-
+
def p_test_or_starred_expr_list(s, expr=None):
exprs = expr is not None and [expr] or []
while s.sy not in expr_terminators:
- exprs.append(p_test_or_starred_expr(s))
+ exprs.append(p_test_or_starred_expr(s))
if s.sy != ',':
break
s.next()
@@ -1481,7 +1481,7 @@ def p_genexp(s, expr):
expr_terminators = cython.declare(set, set([
')', ']', '}', ':', '=', 'NEWLINE']))
-
+
#-------------------------------------------------------
#
# Statements
@@ -1495,14 +1495,14 @@ def p_global_statement(s):
names = p_ident_list(s)
return Nodes.GlobalNode(pos, names = names)
-
+
def p_nonlocal_statement(s):
pos = s.position()
s.next()
names = p_ident_list(s)
return Nodes.NonlocalNode(pos, names = names)
-
+
def p_expression_or_assignment(s):
expr = p_testlist_star_expr(s)
if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute):
@@ -1523,7 +1523,7 @@ def p_expression_or_assignment(s):
expr = p_testlist_star_expr(s)
expr_list.append(expr)
if len(expr_list) == 1:
- if re.match(r"([-+*/%^&|]|<<|>>|\*\*|//|@)=", s.sy):
+ if re.match(r"([-+*/%^&|]|<<|>>|\*\*|//|@)=", s.sy):
lhs = expr_list[0]
if isinstance(lhs, ExprNodes.SliceIndexNode):
# implementation requires IndexNode
@@ -1531,7 +1531,7 @@ def p_expression_or_assignment(s):
lhs.pos,
base=lhs.base,
index=make_slice_node(lhs.pos, lhs.start, lhs.stop))
- elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode)):
+ elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode)):
error(lhs.pos, "Illegal operand for inplace operation.")
operator = s.sy[:-1]
s.next()
@@ -1539,17 +1539,17 @@ def p_expression_or_assignment(s):
rhs = p_yield_expression(s)
else:
rhs = p_testlist(s)
- return Nodes.InPlaceAssignmentNode(lhs.pos, operator=operator, lhs=lhs, rhs=rhs)
+ return Nodes.InPlaceAssignmentNode(lhs.pos, operator=operator, lhs=lhs, rhs=rhs)
expr = expr_list[0]
return Nodes.ExprStatNode(expr.pos, expr=expr)
rhs = expr_list[-1]
if len(expr_list) == 2:
- return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs)
+ return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs)
else:
- return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs)
+ return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs)
+
-
def p_print_statement(s):
# s.sy == 'print'
pos = s.position()
@@ -1572,12 +1572,12 @@ def p_print_statement(s):
ends_with_comma = 1
break
args.append(p_test(s))
- arg_tuple = ExprNodes.TupleNode(pos, args=args)
+ arg_tuple = ExprNodes.TupleNode(pos, args=args)
return Nodes.PrintStatNode(pos,
- arg_tuple=arg_tuple, stream=stream,
- append_newline=not ends_with_comma)
+ arg_tuple=arg_tuple, stream=stream,
+ append_newline=not ends_with_comma)
+
-
def p_exec_statement(s):
# s.sy == 'exec'
pos = s.position()
@@ -1670,43 +1670,43 @@ def p_raise_statement(s):
else:
return Nodes.ReraiseStatNode(pos)
-
+
def p_import_statement(s):
# s.sy in ('import', 'cimport')
pos = s.position()
kind = s.sy
s.next()
- items = [p_dotted_name(s, as_allowed=1)]
+ items = [p_dotted_name(s, as_allowed=1)]
while s.sy == ',':
s.next()
- items.append(p_dotted_name(s, as_allowed=1))
+ items.append(p_dotted_name(s, as_allowed=1))
stats = []
- is_absolute = Future.absolute_import in s.context.future_directives
+ is_absolute = Future.absolute_import in s.context.future_directives
for pos, target_name, dotted_name, as_name in items:
if kind == 'cimport':
- stat = Nodes.CImportStatNode(
- pos,
- module_name=dotted_name,
- as_name=as_name,
- is_absolute=is_absolute)
+ stat = Nodes.CImportStatNode(
+ pos,
+ module_name=dotted_name,
+ as_name=as_name,
+ is_absolute=is_absolute)
else:
if as_name and "." in dotted_name:
- name_list = ExprNodes.ListNode(pos, args=[
- ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))])
+ name_list = ExprNodes.ListNode(pos, args=[
+ ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))])
else:
name_list = None
- stat = Nodes.SingleAssignmentNode(
- pos,
- lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
- rhs=ExprNodes.ImportNode(
- pos,
- module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
- level=0 if is_absolute else None,
- name_list=name_list))
+ stat = Nodes.SingleAssignmentNode(
+ pos,
+ lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
+ rhs=ExprNodes.ImportNode(
+ pos,
+ module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
+ level=0 if is_absolute else None,
+ name_list=name_list))
stats.append(stat)
- return Nodes.StatListNode(pos, stats=stats)
+ return Nodes.StatListNode(pos, stats=stats)
+
-
def p_from_import_statement(s, first_statement = 0):
# s.sy == 'from'
pos = s.position()
@@ -1721,7 +1721,7 @@ def p_from_import_statement(s, first_statement = 0):
level = None
if level is not None and s.sy in ('import', 'cimport'):
# we are dealing with "from .. import foo, bar"
- dotted_name_pos, dotted_name = s.position(), s.context.intern_ustring('')
+ dotted_name_pos, dotted_name = s.position(), s.context.intern_ustring('')
else:
if level is None and Future.absolute_import in s.context.future_directives:
level = 0
@@ -1734,7 +1734,7 @@ def p_from_import_statement(s, first_statement = 0):
is_cimport = kind == 'cimport'
is_parenthesized = False
if s.sy == '*':
- imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)]
+ imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)]
s.next()
else:
if s.sy == '(':
@@ -1775,11 +1775,11 @@ def p_from_import_statement(s, first_statement = 0):
items = []
for (name_pos, name, as_name, kind) in imported_names:
imported_name_strings.append(
- ExprNodes.IdentifierStringNode(name_pos, value=name))
+ ExprNodes.IdentifierStringNode(name_pos, value=name))
items.append(
- (name, ExprNodes.NameNode(name_pos, name=as_name or name)))
+ (name, ExprNodes.NameNode(name_pos, name=as_name or name)))
import_list = ExprNodes.ListNode(
- imported_names[0][0], args=imported_name_strings)
+ imported_names[0][0], args=imported_name_strings)
return Nodes.FromImportStatNode(pos,
module = ExprNodes.ImportNode(dotted_name_pos,
module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
@@ -1788,8 +1788,8 @@ def p_from_import_statement(s, first_statement = 0):
items = items)
-imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union']))
-
+imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union']))
+
def p_imported_name(s, is_cimport):
pos = s.position()
kind = None
@@ -1800,7 +1800,7 @@ def p_imported_name(s, is_cimport):
as_name = p_as_name(s)
return (pos, name, as_name, kind)
-
+
def p_dotted_name(s, as_allowed):
pos = s.position()
target_name = p_ident(s)
@@ -1811,9 +1811,9 @@ def p_dotted_name(s, as_allowed):
names.append(p_ident(s))
if as_allowed:
as_name = p_as_name(s)
- return (pos, target_name, s.context.intern_ustring(u'.'.join(names)), as_name)
+ return (pos, target_name, s.context.intern_ustring(u'.'.join(names)), as_name)
+
-
def p_as_name(s):
if s.sy == 'IDENT' and s.systring == 'as':
s.next()
@@ -1821,7 +1821,7 @@ def p_as_name(s):
else:
return None
-
+
def p_assert_statement(s):
# s.sy == 'assert'
pos = s.position()
@@ -1834,7 +1834,7 @@ def p_assert_statement(s):
value = None
return Nodes.AssertStatNode(pos, cond = cond, value = value)
-
+
statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF']))
def p_if_statement(s):
@@ -1874,25 +1874,25 @@ def p_while_statement(s):
condition = test, body = body,
else_clause = else_clause)
-
-def p_for_statement(s, is_async=False):
+
+def p_for_statement(s, is_async=False):
# s.sy == 'for'
pos = s.position()
s.next()
- kw = p_for_bounds(s, allow_testlist=True, is_async=is_async)
+ kw = p_for_bounds(s, allow_testlist=True, is_async=is_async)
body = p_suite(s)
else_clause = p_else_clause(s)
- kw.update(body=body, else_clause=else_clause, is_async=is_async)
+ kw.update(body=body, else_clause=else_clause, is_async=is_async)
return Nodes.ForStatNode(pos, **kw)
-
-def p_for_bounds(s, allow_testlist=True, is_async=False):
+
+def p_for_bounds(s, allow_testlist=True, is_async=False):
target = p_for_target(s)
if s.sy == 'in':
s.next()
- iterator = p_for_iterator(s, allow_testlist, is_async=is_async)
- return dict(target=target, iterator=iterator)
- elif not s.in_python_file and not is_async:
+ iterator = p_for_iterator(s, allow_testlist, is_async=is_async)
+ return dict(target=target, iterator=iterator)
+ elif not s.in_python_file and not is_async:
if s.sy == 'from':
s.next()
bound1 = p_bit_expr(s)
@@ -1962,20 +1962,20 @@ def p_target(s, terminator):
else:
return expr
-
+
def p_for_target(s):
return p_target(s, 'in')
-
-def p_for_iterator(s, allow_testlist=True, is_async=False):
+
+def p_for_iterator(s, allow_testlist=True, is_async=False):
pos = s.position()
if allow_testlist:
expr = p_testlist(s)
else:
expr = p_or_test(s)
- return (ExprNodes.AsyncIteratorNode if is_async else ExprNodes.IteratorNode)(pos, sequence=expr)
+ return (ExprNodes.AsyncIteratorNode if is_async else ExprNodes.IteratorNode)(pos, sequence=expr)
+
-
def p_try_statement(s):
# s.sy == 'try'
pos = s.position()
@@ -2043,14 +2043,14 @@ def p_include_statement(s, ctx):
include_file_path = s.context.find_include_file(include_file_name, pos)
if include_file_path:
s.included_files.append(include_file_name)
- with Utils.open_source_file(include_file_path) as f:
+ with Utils.open_source_file(include_file_path) as f:
if Options.source_root:
import os
rel_path = os.path.relpath(include_file_path, Options.source_root)
else:
rel_path = None
source_desc = FileSourceDescriptor(include_file_path, rel_path)
- s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
+ s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
tree = p_statement_list(s2, ctx)
return tree
else:
@@ -2058,21 +2058,21 @@ def p_include_statement(s, ctx):
else:
return Nodes.PassStatNode(pos)
-
+
def p_with_statement(s):
- s.next() # 'with'
+ s.next() # 'with'
if s.systring == 'template' and not s.in_python_file:
node = p_with_template(s)
else:
node = p_with_items(s)
return node
-
-def p_with_items(s, is_async=False):
+
+def p_with_items(s, is_async=False):
pos = s.position()
if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
- if is_async:
- s.error("with gil/nogil cannot be async")
+ if is_async:
+ s.error("with gil/nogil cannot be async")
state = s.systring
s.next()
if s.sy == ',':
@@ -2080,7 +2080,7 @@ def p_with_items(s, is_async=False):
body = p_with_items(s)
else:
body = p_suite(s)
- return Nodes.GILStatNode(pos, state=state, body=body)
+ return Nodes.GILStatNode(pos, state=state, body=body)
else:
manager = p_test(s)
target = None
@@ -2089,12 +2089,12 @@ def p_with_items(s, is_async=False):
target = p_starred_expr(s)
if s.sy == ',':
s.next()
- body = p_with_items(s, is_async=is_async)
+ body = p_with_items(s, is_async=is_async)
else:
body = p_suite(s)
- return Nodes.WithStatNode(pos, manager=manager, target=target, body=body, is_async=is_async)
+ return Nodes.WithStatNode(pos, manager=manager, target=target, body=body, is_async=is_async)
+
-
def p_with_template(s):
pos = s.position()
templates = []
@@ -2255,13 +2255,13 @@ def p_statement(s, ctx, first_statement = 0):
s.level = ctx.level
decorators = p_decorators(s)
if not ctx.allow_struct_enum_decorator and s.sy not in ('def', 'cdef', 'cpdef', 'class', 'async'):
- if s.sy == 'IDENT' and s.systring == 'async':
- pass # handled below
- else:
- s.error("Decorators can only be followed by functions or classes")
+ if s.sy == 'IDENT' and s.systring == 'async':
+ pass # handled below
+ else:
+ s.error("Decorators can only be followed by functions or classes")
elif s.sy == 'pass' and cdef_flag:
# empty cdef block
- return p_pass_statement(s, with_newline=1)
+ return p_pass_statement(s, with_newline=1)
overridable = 0
if s.sy == 'cdef':
@@ -2275,11 +2275,11 @@ def p_statement(s, ctx, first_statement = 0):
if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
s.error('cdef statement not allowed here')
s.level = ctx.level
- node = p_cdef_statement(s, ctx(overridable=overridable))
+ node = p_cdef_statement(s, ctx(overridable=overridable))
if decorators is not None:
- tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)
+ tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)
if ctx.allow_struct_enum_decorator:
- tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode)
+ tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode)
if not isinstance(node, tup):
s.error("Decorators can only be followed by functions or classes")
node.decorators = decorators
@@ -2322,22 +2322,22 @@ def p_statement(s, ctx, first_statement = 0):
return p_try_statement(s)
elif s.sy == 'with':
return p_with_statement(s)
- elif s.sy == 'async':
- s.next()
- return p_async_statement(s, ctx, decorators)
+ elif s.sy == 'async':
+ s.next()
+ return p_async_statement(s, ctx, decorators)
else:
- if s.sy == 'IDENT' and s.systring == 'async':
- ident_name = s.systring
- # PEP 492 enables the async/await keywords when it spots "async def ..."
- s.next()
- if s.sy == 'def':
- return p_async_statement(s, ctx, decorators)
- elif decorators:
- s.error("Decorators can only be followed by functions or classes")
- s.put_back('IDENT', ident_name) # re-insert original token
- return p_simple_statement_list(s, ctx, first_statement=first_statement)
-
-
+ if s.sy == 'IDENT' and s.systring == 'async':
+ ident_name = s.systring
+ # PEP 492 enables the async/await keywords when it spots "async def ..."
+ s.next()
+ if s.sy == 'def':
+ return p_async_statement(s, ctx, decorators)
+ elif decorators:
+ s.error("Decorators can only be followed by functions or classes")
+ s.put_back('IDENT', ident_name) # re-insert original token
+ return p_simple_statement_list(s, ctx, first_statement=first_statement)
+
+
def p_statement_list(s, ctx, first_statement = 0):
# Parse a series of statements separated by newlines.
pos = s.position()
@@ -2412,7 +2412,7 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None):
arg = Nodes.CComplexBaseTypeNode(base_type.pos,
base_type = base_type, declarator = declarator)
parsed_type = True
- keyword_node = ExprNodes.IdentifierStringNode(arg.pos, value=ident)
+ keyword_node = ExprNodes.IdentifierStringNode(arg.pos, value=ident)
keyword_args.append((keyword_node, arg))
was_keyword = True
@@ -2455,31 +2455,31 @@ def p_calling_convention(s):
else:
return ""
-
+
calling_convention_words = cython.declare(
set, set(["__stdcall", "__cdecl", "__fastcall"]))
-
+
def p_c_complex_base_type(s, templates = None):
# s.sy == '('
pos = s.position()
s.next()
- base_type = p_c_base_type(s, templates=templates)
- declarator = p_c_declarator(s, empty=True)
- type_node = Nodes.CComplexBaseTypeNode(
- pos, base_type=base_type, declarator=declarator)
- if s.sy == ',':
- components = [type_node]
- while s.sy == ',':
- s.next()
- if s.sy == ')':
- break
- base_type = p_c_base_type(s, templates=templates)
- declarator = p_c_declarator(s, empty=True)
- components.append(Nodes.CComplexBaseTypeNode(
- pos, base_type=base_type, declarator=declarator))
- type_node = Nodes.CTupleBaseTypeNode(pos, components = components)
-
+ base_type = p_c_base_type(s, templates=templates)
+ declarator = p_c_declarator(s, empty=True)
+ type_node = Nodes.CComplexBaseTypeNode(
+ pos, base_type=base_type, declarator=declarator)
+ if s.sy == ',':
+ components = [type_node]
+ while s.sy == ',':
+ s.next()
+ if s.sy == ')':
+ break
+ base_type = p_c_base_type(s, templates=templates)
+ declarator = p_c_declarator(s, empty=True)
+ components.append(Nodes.CComplexBaseTypeNode(
+ pos, base_type=base_type, declarator=declarator))
+ type_node = Nodes.CTupleBaseTypeNode(pos, components = components)
+
s.expect(')')
if s.sy == '[':
if is_memoryviewslice_access(s):
@@ -2728,7 +2728,7 @@ special_basic_c_types = cython.declare(dict, {
# name : (signed, longness)
"Py_UNICODE" : (0, 0),
"Py_UCS4" : (0, 0),
- "Py_hash_t" : (2, 0),
+ "Py_hash_t" : (2, 0),
"Py_ssize_t" : (2, 0),
"ssize_t" : (2, 0),
"size_t" : (0, 0),
@@ -2783,7 +2783,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
if s.sy == '(':
s.next()
if s.sy == ')' or looking_at_name(s):
- base = Nodes.CNameDeclaratorNode(pos, name=s.context.intern_ustring(u""), cname=None)
+ base = Nodes.CNameDeclaratorNode(pos, name=s.context.intern_ustring(u""), cname=None)
result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
else:
result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
@@ -2835,8 +2835,8 @@ supported_overloaded_operators = cython.declare(set, set([
'+', '-', '*', '/', '%',
'++', '--', '~', '|', '&', '^', '<<', '>>', ',',
'==', '!=', '>=', '>', '<=', '<',
- '[]', '()', '!', '=',
- 'bool',
+ '[]', '()', '!', '=',
+ 'bool',
]))
def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
@@ -2877,7 +2877,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
else:
rhs = None
if s.sy == 'IDENT':
- name = s.systring
+ name = s.systring
if empty:
error(s.position(), "Declarator should be empty")
s.next()
@@ -2913,13 +2913,13 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
s.error("Overloading operator '%s' not yet supported." % op,
fatal=False)
name += op
- elif op == 'IDENT':
- op = s.systring;
- if op not in supported_overloaded_operators:
- s.error("Overloading operator '%s' not yet supported." % op,
- fatal=False)
- name = name + ' ' + op
- s.next()
+ elif op == 'IDENT':
+ op = s.systring;
+ if op not in supported_overloaded_operators:
+ s.error("Overloading operator '%s' not yet supported." % op,
+ fatal=False)
+ name = name + ' ' + op
+ s.next()
result = Nodes.CNameDeclaratorNode(pos,
name = name, cname = cname, default = rhs)
result.calling_convention = calling_convention
@@ -2955,9 +2955,9 @@ def p_exception_value_clause(s):
name = s.systring
s.next()
exc_val = p_name(s, name)
- elif s.sy == '*':
- exc_val = ExprNodes.CharNode(s.position(), value=u'*')
- s.next()
+ elif s.sy == '*':
+ exc_val = ExprNodes.CharNode(s.position(), value=u'*')
+ s.next()
else:
if s.sy == '?':
exc_check = 1
@@ -2965,7 +2965,7 @@ def p_exception_value_clause(s):
exc_val = p_test(s)
return exc_val, exc_check
-c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':']))
+c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':']))
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
nonempty_declarators = 0, kw_only = 0, annotated = 1):
@@ -3278,14 +3278,14 @@ def p_c_func_or_var_declaration(s, pos, ctx):
is_const_method = 1
else:
is_const_method = 0
- if s.sy == '->':
- # Special enough to give a better error message and keep going.
- s.error(
- "Return type annotation is not allowed in cdef/cpdef signatures. "
- "Please define it before the function name, as in C signatures.",
- fatal=False)
- s.next()
- p_test(s) # Keep going, but ignore result.
+ if s.sy == '->':
+ # Special enough to give a better error message and keep going.
+ s.error(
+ "Return type annotation is not allowed in cdef/cpdef signatures. "
+ "Please define it before the function name, as in C signatures.",
+ fatal=False)
+ s.next()
+ p_test(s) # Keep going, but ignore result.
if s.sy == ':':
if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
s.error("C function definition not allowed here")
@@ -3362,59 +3362,59 @@ def p_decorators(s):
s.next()
decstring = p_dotted_name(s, as_allowed=0)[2]
names = decstring.split('.')
- decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0]))
+ decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0]))
for name in names[1:]:
- decorator = ExprNodes.AttributeNode(
- pos, attribute=s.context.intern_ustring(name), obj=decorator)
+ decorator = ExprNodes.AttributeNode(
+ pos, attribute=s.context.intern_ustring(name), obj=decorator)
if s.sy == '(':
decorator = p_call(s, decorator)
decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
s.expect_newline("Expected a newline after decorator")
return decorators
-
-def _reject_cdef_modifier_in_py(s, name):
- """Step over incorrectly placed cdef modifiers (@see _CDEF_MODIFIERS) to provide a good error message for them.
- """
- if s.sy == 'IDENT' and name in _CDEF_MODIFIERS:
- # Special enough to provide a good error message.
- s.error("Cannot use cdef modifier '%s' in Python function signature. Use a decorator instead." % name, fatal=False)
- return p_ident(s) # Keep going, in case there are other errors.
- return name
-
-
-def p_def_statement(s, decorators=None, is_async_def=False):
+
+def _reject_cdef_modifier_in_py(s, name):
+ """Step over incorrectly placed cdef modifiers (@see _CDEF_MODIFIERS) to provide a good error message for them.
+ """
+ if s.sy == 'IDENT' and name in _CDEF_MODIFIERS:
+ # Special enough to provide a good error message.
+ s.error("Cannot use cdef modifier '%s' in Python function signature. Use a decorator instead." % name, fatal=False)
+ return p_ident(s) # Keep going, in case there are other errors.
+ return name
+
+
+def p_def_statement(s, decorators=None, is_async_def=False):
# s.sy == 'def'
pos = s.position()
- # PEP 492 switches the async/await keywords on in "async def" functions
- if is_async_def:
- s.enter_async()
+ # PEP 492 switches the async/await keywords on in "async def" functions
+ if is_async_def:
+ s.enter_async()
s.next()
- name = _reject_cdef_modifier_in_py(s, p_ident(s))
- s.expect(
- '(',
- "Expected '(', found '%s'. Did you use cdef syntax in a Python declaration? "
- "Use decorators and Python type annotations instead." % (
- s.systring if s.sy == 'IDENT' else s.sy))
+ name = _reject_cdef_modifier_in_py(s, p_ident(s))
+ s.expect(
+ '(',
+ "Expected '(', found '%s'. Did you use cdef syntax in a Python declaration? "
+ "Use decorators and Python type annotations instead." % (
+ s.systring if s.sy == 'IDENT' else s.sy))
args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
s.expect(')')
- _reject_cdef_modifier_in_py(s, s.systring)
+ _reject_cdef_modifier_in_py(s, s.systring)
return_type_annotation = None
if s.sy == '->':
s.next()
return_type_annotation = p_test(s)
- _reject_cdef_modifier_in_py(s, s.systring)
-
+ _reject_cdef_modifier_in_py(s, s.systring)
+
doc, body = p_suite_with_docstring(s, Ctx(level='function'))
- if is_async_def:
- s.exit_async()
-
- return Nodes.DefNode(
- pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg,
- doc=doc, body=body, decorators=decorators, is_async_def=is_async_def,
- return_type_annotation=return_type_annotation)
-
-
+ if is_async_def:
+ s.exit_async()
+
+ return Nodes.DefNode(
+ pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg,
+ doc=doc, body=body, decorators=decorators, is_async_def=is_async_def,
+ return_type_annotation=return_type_annotation)
+
+
def p_varargslist(s, terminator=')', annotated=1):
args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
annotated = annotated)
@@ -3433,8 +3433,8 @@ def p_varargslist(s, terminator=')', annotated=1):
if s.sy == '**':
s.next()
starstar_arg = p_py_arg_decl(s, annotated=annotated)
- if s.sy == ',':
- s.next()
+ if s.sy == ',':
+ s.next()
return (args, star_arg, starstar_arg)
def p_py_arg_decl(s, annotated = 1):
@@ -3446,18 +3446,18 @@ def p_py_arg_decl(s, annotated = 1):
annotation = p_test(s)
return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
-
+
def p_class_statement(s, decorators):
# s.sy == 'class'
pos = s.position()
s.next()
- class_name = EncodedString(p_ident(s))
- class_name.encoding = s.source_encoding # FIXME: why is this needed?
+ class_name = EncodedString(p_ident(s))
+ class_name.encoding = s.source_encoding # FIXME: why is this needed?
arg_tuple = None
keyword_dict = None
if s.sy == '(':
- positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
- arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
+ positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
+ arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
if arg_tuple is None:
# XXX: empty arg_tuple
arg_tuple = ExprNodes.TupleNode(pos, args=[])
@@ -3469,7 +3469,7 @@ def p_class_statement(s, decorators):
doc=doc, body=body, decorators=decorators,
force_py3_semantics=s.context.language_level >= 3)
-
+
def p_c_class_definition(s, pos, ctx):
# s.sy == 'class'
s.next()
@@ -3489,7 +3489,7 @@ def p_c_class_definition(s, pos, ctx):
objstruct_name = None
typeobj_name = None
bases = None
- check_size = None
+ check_size = None
if s.sy == '(':
positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
if keyword_args:
@@ -3501,7 +3501,7 @@ def p_c_class_definition(s, pos, ctx):
if s.sy == '[':
if ctx.visibility not in ('public', 'extern') and not ctx.api:
error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
- objstruct_name, typeobj_name, check_size = p_c_class_options(s)
+ objstruct_name, typeobj_name, check_size = p_c_class_options(s)
if s.sy == ':':
if ctx.level == 'module_pxd':
body_level = 'c_class_pxd'
@@ -3540,16 +3540,16 @@ def p_c_class_definition(s, pos, ctx):
bases = bases,
objstruct_name = objstruct_name,
typeobj_name = typeobj_name,
- check_size = check_size,
+ check_size = check_size,
in_pxd = ctx.level == 'module_pxd',
doc = doc,
body = body)
-
+
def p_c_class_options(s):
objstruct_name = None
typeobj_name = None
- check_size = None
+ check_size = None
s.expect('[')
while 1:
if s.sy != 'IDENT':
@@ -3560,16 +3560,16 @@ def p_c_class_options(s):
elif s.systring == 'type':
s.next()
typeobj_name = p_ident(s)
- elif s.systring == 'check_size':
- s.next()
- check_size = p_ident(s)
- if check_size not in ('ignore', 'warn', 'error'):
- s.error("Expected one of ignore, warn or error, found %r" % check_size)
+ elif s.systring == 'check_size':
+ s.next()
+ check_size = p_ident(s)
+ if check_size not in ('ignore', 'warn', 'error'):
+ s.error("Expected one of ignore, warn or error, found %r" % check_size)
if s.sy != ',':
break
s.next()
- s.expect(']', "Expected 'object', 'type' or 'check_size'")
- return objstruct_name, typeobj_name, check_size
+ s.expect(']', "Expected 'object', 'type' or 'check_size'")
+ return objstruct_name, typeobj_name, check_size
def p_property_decl(s):
@@ -3648,64 +3648,64 @@ def p_code(s, level=None, ctx=Ctx):
repr(s.sy), repr(s.systring)))
return body
-
+
_match_compiler_directive_comment = cython.declare(object, re.compile(
r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match)
-
+
def p_compiler_directive_comments(s):
result = {}
while s.sy == 'commentline':
- pos = s.position()
+ pos = s.position()
m = _match_compiler_directive_comment(s.systring)
if m:
- directives_string = m.group(1).strip()
+ directives_string = m.group(1).strip()
try:
- new_directives = Options.parse_directive_list(directives_string, ignore_unknown=True)
- except ValueError as e:
+ new_directives = Options.parse_directive_list(directives_string, ignore_unknown=True)
+ except ValueError as e:
s.error(e.args[0], fatal=False)
- s.next()
- continue
-
- for name in new_directives:
- if name not in result:
- pass
- elif new_directives[name] == result[name]:
- warning(pos, "Duplicate directive found: %s" % (name,))
- else:
- s.error("Conflicting settings found for top-level directive %s: %r and %r" % (
- name, result[name], new_directives[name]), pos=pos)
-
- if 'language_level' in new_directives:
- # Make sure we apply the language level already to the first token that follows the comments.
- s.context.set_language_level(new_directives['language_level'])
-
- result.update(new_directives)
-
+ s.next()
+ continue
+
+ for name in new_directives:
+ if name not in result:
+ pass
+ elif new_directives[name] == result[name]:
+ warning(pos, "Duplicate directive found: %s" % (name,))
+ else:
+ s.error("Conflicting settings found for top-level directive %s: %r and %r" % (
+ name, result[name], new_directives[name]), pos=pos)
+
+ if 'language_level' in new_directives:
+ # Make sure we apply the language level already to the first token that follows the comments.
+ s.context.set_language_level(new_directives['language_level'])
+
+ result.update(new_directives)
+
s.next()
return result
-
+
def p_module(s, pxd, full_module_name, ctx=Ctx):
pos = s.position()
directive_comments = p_compiler_directive_comments(s)
s.parse_comments = False
- if s.context.language_level is None:
- s.context.set_language_level(2) # Arcadia default.
-
- if s.context.language_level is None:
- s.context.set_language_level(2)
- if pos[0].filename:
- import warnings
- warnings.warn(
- "Cython directive 'language_level' not set, using 2 for now (Py2). "
- "This will change in a later release! File: %s" % pos[0].filename,
- FutureWarning,
- stacklevel=1 if cython.compiled else 2,
- )
-
+ if s.context.language_level is None:
+ s.context.set_language_level(2) # Arcadia default.
+
+ if s.context.language_level is None:
+ s.context.set_language_level(2)
+ if pos[0].filename:
+ import warnings
+ warnings.warn(
+ "Cython directive 'language_level' not set, using 2 for now (Py2). "
+ "This will change in a later release! File: %s" % pos[0].filename,
+ FutureWarning,
+ stacklevel=1 if cython.compiled else 2,
+ )
+
doc = p_doc_string(s)
if pxd:
level = 'module_pxd'
@@ -3720,16 +3720,16 @@ def p_module(s, pxd, full_module_name, ctx=Ctx):
full_module_name = full_module_name,
directive_comments = directive_comments)
-def p_template_definition(s):
- name = p_ident(s)
- if s.sy == '=':
- s.expect('=')
- s.expect('*')
- required = False
- else:
- required = True
- return name, required
-
+def p_template_definition(s):
+ name = p_ident(s)
+ if s.sy == '=':
+ s.expect('=')
+ s.expect('*')
+ required = False
+ else:
+ required = True
+ return name, required
+
def p_cpp_class_definition(s, pos, ctx):
# s.sy == 'cppclass'
s.next()
@@ -3742,21 +3742,21 @@ def p_cpp_class_definition(s, pos, ctx):
error(pos, "Qualified class name not allowed C++ class")
if s.sy == '[':
s.next()
- templates = [p_template_definition(s)]
+ templates = [p_template_definition(s)]
while s.sy == ',':
s.next()
- templates.append(p_template_definition(s))
+ templates.append(p_template_definition(s))
s.expect(']')
- template_names = [name for name, required in templates]
+ template_names = [name for name, required in templates]
else:
templates = None
- template_names = None
+ template_names = None
if s.sy == '(':
s.next()
- base_classes = [p_c_base_type(s, templates = template_names)]
+ base_classes = [p_c_base_type(s, templates = template_names)]
while s.sy == ',':
s.next()
- base_classes.append(p_c_base_type(s, templates = template_names))
+ base_classes.append(p_c_base_type(s, templates = template_names))
s.expect(')')
else:
base_classes = []
@@ -3769,7 +3769,7 @@ def p_cpp_class_definition(s, pos, ctx):
s.expect_indent()
attributes = []
body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
- body_ctx.templates = template_names
+ body_ctx.templates = template_names
while s.sy != 'DEDENT':
if s.sy != 'pass':
attributes.append(p_cpp_class_attribute(s, body_ctx))
@@ -3795,13 +3795,13 @@ def p_cpp_class_attribute(s, ctx):
decorators = p_decorators(s)
if s.systring == 'cppclass':
return p_cpp_class_definition(s, s.position(), ctx)
- elif s.systring == 'ctypedef':
- return p_ctypedef_statement(s, ctx)
- elif s.sy == 'IDENT' and s.systring in struct_enum_union:
- if s.systring != 'enum':
- return p_cpp_class_definition(s, s.position(), ctx)
- else:
- return p_struct_enum(s, s.position(), ctx)
+ elif s.systring == 'ctypedef':
+ return p_ctypedef_statement(s, ctx)
+ elif s.sy == 'IDENT' and s.systring in struct_enum_union:
+ if s.systring != 'enum':
+ return p_cpp_class_definition(s, s.position(), ctx)
+ else:
+ return p_struct_enum(s, s.position(), ctx)
else:
node = p_c_func_or_var_declaration(s, s.position(), ctx)
if decorators is not None:
@@ -3829,7 +3829,7 @@ def print_parse_tree(f, node, level, key = None):
t = type(node)
if t is tuple:
f.write("(%s @ %s\n" % (node[0], node[1]))
- for i in range(2, len(node)):
+ for i in range(2, len(node)):
print_parse_tree(f, node[i], level+1)
f.write("%s)\n" % ind)
return
@@ -3845,7 +3845,7 @@ def print_parse_tree(f, node, level, key = None):
return
elif t is list:
f.write("[\n")
- for i in range(len(node)):
+ for i in range(len(node)):
print_parse_tree(f, node[i], level+1)
f.write("%s]\n" % ind)
return