diff options
author | Aleksandr <[email protected]> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:47:52 +0300 |
commit | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (patch) | |
tree | d16cef493ac1e092b4a03ab9437ec06ffe3d188f /contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py | |
parent | 37de222addabbef336dcaaea5f7c7645a629fc6d (diff) |
Restoring authorship annotation for Aleksandr <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py')
-rw-r--r-- | contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py | 666 |
1 files changed, 333 insertions, 333 deletions
diff --git a/contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py b/contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py index 0da3670caee..88f028aa0ca 100644 --- a/contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py +++ b/contrib/tools/cython/Cython/Compiler/ParseTreeTransforms.py @@ -7,7 +7,7 @@ cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object, error=object, warning=object, copy=object, _unicode=object) import copy -import hashlib +import hashlib from . import PyrexTypes from . import Naming @@ -15,7 +15,7 @@ from . import ExprNodes from . import Nodes from . import Options from . import Builtin -from . import Errors +from . import Errors from .Visitor import VisitorTransform, TreeVisitor from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform @@ -180,7 +180,7 @@ class PostParse(ScopeTrackingTransform): # unpack a lambda expression into the corresponding DefNode collector = YieldNodeCollector() collector.visitchildren(node.result_expr) - if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode): + if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode): body = Nodes.ExprStatNode( node.result_expr.pos, expr=node.result_expr) else: @@ -196,25 +196,25 @@ class PostParse(ScopeTrackingTransform): def visit_GeneratorExpressionNode(self, node): # unpack a generator expression into the corresponding DefNode - collector = YieldNodeCollector() - collector.visitchildren(node.loop) - node.def_node = Nodes.DefNode( - node.pos, name=node.name, doc=None, - args=[], star_arg=None, starstar_arg=None, - body=node.loop, is_async_def=collector.has_await) - self.visitchildren(node) - return node - - def visit_ComprehensionNode(self, node): - # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...) - if not node.has_local_scope: - collector = YieldNodeCollector() - collector.visitchildren(node.loop) - if collector.has_await: - node.has_local_scope = True - self.visitchildren(node) - return node - + collector = YieldNodeCollector() + collector.visitchildren(node.loop) + node.def_node = Nodes.DefNode( + node.pos, name=node.name, doc=None, + args=[], star_arg=None, starstar_arg=None, + body=node.loop, is_async_def=collector.has_await) + self.visitchildren(node) + return node + + def visit_ComprehensionNode(self, node): + # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...) + if not node.has_local_scope: + collector = YieldNodeCollector() + collector.visitchildren(node.loop) + if collector.has_await: + node.has_local_scope = True + self.visitchildren(node) + return node + # cdef variables def handle_bufferdefaults(self, decl): if not isinstance(decl.default, ExprNodes.DictNode): @@ -599,29 +599,29 @@ class PxdPostParse(CythonTransform, SkipDeclarations): else: return node - -class TrackNumpyAttributes(VisitorTransform, SkipDeclarations): - # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two. - def __init__(self): - super(TrackNumpyAttributes, self).__init__() - self.numpy_module_names = set() - - def visit_CImportStatNode(self, node): - if node.module_name == u"numpy": - self.numpy_module_names.add(node.as_name or u"numpy") - return node - - def visit_AttributeNode(self, node): - self.visitchildren(node) + +class TrackNumpyAttributes(VisitorTransform, SkipDeclarations): + # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two. + def __init__(self): + super(TrackNumpyAttributes, self).__init__() + self.numpy_module_names = set() + + def visit_CImportStatNode(self, node): + if node.module_name == u"numpy": + self.numpy_module_names.add(node.as_name or u"numpy") + return node + + def visit_AttributeNode(self, node): + self.visitchildren(node) obj = node.obj if (obj.is_name and obj.name in self.numpy_module_names) or obj.is_numpy_attribute: - node.is_numpy_attribute = True - return node - - visit_Node = VisitorTransform.recurse_to_children - - -class InterpretCompilerDirectives(CythonTransform): + node.is_numpy_attribute = True + return node + + visit_Node = VisitorTransform.recurse_to_children + + +class InterpretCompilerDirectives(CythonTransform): """ After parsing, directives can be stored in a number of places: - #cython-comments at the top of the file (stored in ModuleNode) @@ -841,16 +841,16 @@ class InterpretCompilerDirectives(CythonTransform): if node.name in self.cython_module_names: node.is_cython_module = True else: - directive = self.directive_names.get(node.name) - if directive is not None: - node.cython_attribute = directive - return node - - def visit_NewExprNode(self, node): - self.visit(node.cppclass) - self.visitchildren(node) + directive = self.directive_names.get(node.name) + if directive is not None: + node.cython_attribute = directive return node + def visit_NewExprNode(self, node): + self.visit(node.cppclass) + self.visitchildren(node) + return node + def try_to_parse_directives(self, node): # If node is the contents of an directive (in a with statement or # decorator), returns a list of (directivename, value) pairs. @@ -886,8 +886,8 @@ class InterpretCompilerDirectives(CythonTransform): if optname: directivetype = Options.directive_types.get(optname) if directivetype is bool: - arg = ExprNodes.BoolNode(node.pos, value=True) - return [self.try_to_parse_directive(optname, [arg], None, node.pos)] + arg = ExprNodes.BoolNode(node.pos, value=True) + return [self.try_to_parse_directive(optname, [arg], None, node.pos)] elif directivetype is None: return [(optname, None)] else: @@ -896,25 +896,25 @@ class InterpretCompilerDirectives(CythonTransform): return None def try_to_parse_directive(self, optname, args, kwds, pos): - if optname == 'np_pythran' and not self.context.cpp: - raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname) - elif optname == 'exceptval': - # default: exceptval(None, check=True) - arg_error = len(args) > 1 - check = True - if kwds and kwds.key_value_pairs: - kw = kwds.key_value_pairs[0] - if (len(kwds.key_value_pairs) == 1 and - kw.key.is_string_literal and kw.key.value == 'check' and - isinstance(kw.value, ExprNodes.BoolNode)): - check = kw.value.value - else: - arg_error = True - if arg_error: - raise PostParseError( - pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"') - return ('exceptval', (args[0] if args else None, check)) - + if optname == 'np_pythran' and not self.context.cpp: + raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname) + elif optname == 'exceptval': + # default: exceptval(None, check=True) + arg_error = len(args) > 1 + check = True + if kwds and kwds.key_value_pairs: + kw = kwds.key_value_pairs[0] + if (len(kwds.key_value_pairs) == 1 and + kw.key.is_string_literal and kw.key.value == 'check' and + isinstance(kw.value, ExprNodes.BoolNode)): + check = kw.value.value + else: + arg_error = True + if arg_error: + raise PostParseError( + pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"') + return ('exceptval', (args[0] if args else None, check)) + directivetype = Options.directive_types.get(optname) if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode): return optname, Options.get_directive_defaults()[optname] @@ -945,7 +945,7 @@ class InterpretCompilerDirectives(CythonTransform): 'The %s directive takes no prepositional arguments' % optname) return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs]) elif directivetype is list: - if kwds and len(kwds.key_value_pairs) != 0: + if kwds and len(kwds.key_value_pairs) != 0: raise PostParseError(pos, 'The %s directive takes no keyword arguments' % optname) return optname, [ str(arg.value) for arg in args ] @@ -1014,8 +1014,8 @@ class InterpretCompilerDirectives(CythonTransform): directives = [] realdecs = [] both = [] - # Decorators coming first take precedence. - for dec in node.decorators[::-1]: + # Decorators coming first take precedence. + for dec in node.decorators[::-1]: new_directives = self.try_to_parse_directives(dec.decorator) if new_directives is not None: for directive in new_directives: @@ -1025,15 +1025,15 @@ class InterpretCompilerDirectives(CythonTransform): directives.append(directive) if directive[0] == 'staticmethod': both.append(dec) - # Adapt scope type based on decorators that change it. - if directive[0] == 'cclass' and scope_name == 'class': - scope_name = 'cclass' + # Adapt scope type based on decorators that change it. + if directive[0] == 'cclass' and scope_name == 'class': + scope_name = 'cclass' else: realdecs.append(dec) - if realdecs and (scope_name == 'cclass' or - isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): + if realdecs and (scope_name == 'cclass' or + isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.") - node.decorators = realdecs[::-1] + both[::-1] + node.decorators = realdecs[::-1] + both[::-1] # merge or override repeated directives optdict = {} for directive in directives: @@ -1283,7 +1283,7 @@ class WithTransform(CythonTransform, SkipDeclarations): pos, with_stat=node, test_if_run=False, args=excinfo_target, - await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), + await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), body=Nodes.ReraiseStatNode(pos), ), ], @@ -1305,7 +1305,7 @@ class WithTransform(CythonTransform, SkipDeclarations): test_if_run=True, args=ExprNodes.TupleNode( pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]), - await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), + await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)), handle_error_case=False, ) return node @@ -1376,28 +1376,28 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): elif decorator.is_attribute and decorator.obj.name in properties: handler_name = self._map_property_attribute(decorator.attribute) if handler_name: - if decorator.obj.name != node.name: - # CPython does not generate an error or warning, but not something useful either. - error(decorator_node.pos, - "Mismatching property names, expected '%s', got '%s'" % ( - decorator.obj.name, node.name)) - elif len(node.decorators) > 1: + if decorator.obj.name != node.name: + # CPython does not generate an error or warning, but not something useful either. + error(decorator_node.pos, + "Mismatching property names, expected '%s', got '%s'" % ( + decorator.obj.name, node.name)) + elif len(node.decorators) > 1: return self._reject_decorated_property(node, decorator_node) - else: - return self._add_to_property(properties, node, handler_name, decorator_node) - - # we clear node.decorators, so we need to set the - # is_staticmethod/is_classmethod attributes now - for decorator in node.decorators: - func = decorator.decorator - if func.is_name: - node.is_classmethod |= func.name == 'classmethod' - node.is_staticmethod |= func.name == 'staticmethod' - + else: + return self._add_to_property(properties, node, handler_name, decorator_node) + + # we clear node.decorators, so we need to set the + # is_staticmethod/is_classmethod attributes now + for decorator in node.decorators: + func = decorator.decorator + if func.is_name: + node.is_classmethod |= func.name == 'classmethod' + node.is_staticmethod |= func.name == 'staticmethod' + # transform normal decorators - decs = node.decorators - node.decorators = None - return self.chain_decorators(node, decs, node.name) + decs = node.decorators + node.decorators = None + return self.chain_decorators(node, decs, node.name) @staticmethod def _reject_decorated_property(node, decorator_node): @@ -1531,13 +1531,13 @@ class ForwardDeclareTypes(CythonTransform): def visit_CClassDefNode(self, node): if node.class_name not in self.module_scope.entries: node.declare(self.module_scope) - # Expand fused methods of .pxd declared types to construct the final vtable order. - type = self.module_scope.entries[node.class_name].type - if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope: - scope = type.scope - for entry in scope.cfunc_entries: - if entry.type and entry.type.is_fused: - entry.type.get_all_specialized_function_types() + # Expand fused methods of .pxd declared types to construct the final vtable order. + type = self.module_scope.entries[node.class_name].type + if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope: + scope = type.scope + for entry in scope.cfunc_entries: + if entry.type and entry.type.is_fused: + entry.type.get_all_specialized_function_types() return node @@ -1602,13 +1602,13 @@ if VALUE is not None: return node def visit_ModuleNode(self, node): - # Pickling support requires injecting module-level nodes. - self.extra_module_declarations = [] + # Pickling support requires injecting module-level nodes. + self.extra_module_declarations = [] self.seen_vars_stack.append(set()) node.analyse_declarations(self.current_env()) self.visitchildren(node) self.seen_vars_stack.pop() - node.body.stats.extend(self.extra_module_declarations) + node.body.stats.extend(self.extra_module_declarations) return node def visit_LambdaNode(self, node): @@ -1630,145 +1630,145 @@ if VALUE is not None: stats.append(property) if stats: node.body.stats += stats - if (node.visibility != 'extern' - and not node.scope.lookup('__reduce__') - and not node.scope.lookup('__reduce_ex__')): - self._inject_pickle_methods(node) - return node - - def _inject_pickle_methods(self, node): - env = self.current_env() - if node.scope.directives['auto_pickle'] is False: # None means attempt it. - # Old behavior of not doing anything. - return - auto_pickle_forced = node.scope.directives['auto_pickle'] is True - - all_members = [] - cls = node.entry.type - cinit = None - inherited_reduce = None - while cls is not None: - all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__')) - cinit = cinit or cls.scope.lookup('__cinit__') - inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__') - cls = cls.base_type - all_members.sort(key=lambda e: e.name) - - if inherited_reduce: - # This is not failsafe, as we may not know whether a cimported class defines a __reduce__. - # This is why we define __reduce_cython__ and only replace __reduce__ - # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation. - return - - non_py = [ - e for e in all_members - if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env) - or not e.type.can_coerce_from_pyobject(env)) - ] - - structs = [e for e in all_members if e.type.is_struct_or_union] - - if cinit or non_py or (structs and not auto_pickle_forced): - if cinit: - # TODO(robertwb): We could allow this if __cinit__ has no require arguments. - msg = 'no default __reduce__ due to non-trivial __cinit__' - elif non_py: - msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py) - else: - # Extern structs may be only partially defined. - # TODO(robertwb): Limit the restriction to extern - # (and recursively extern-containing) structs. - msg = ("Pickling of struct members such as %s must be explicitly requested " - "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs)) - - if auto_pickle_forced: - error(node.pos, msg) - - pickle_func = TreeFragment(u""" - def __reduce_cython__(self): - raise TypeError("%(msg)s") - def __setstate_cython__(self, __pyx_state): - raise TypeError("%(msg)s") - """ % {'msg': msg}, - level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) - pickle_func.analyse_declarations(node.scope) - self.visit(pickle_func) - node.body.stats.append(pickle_func) - - else: - for e in all_members: - if not e.type.is_pyobject: - e.type.create_to_py_utility_code(env) - e.type.create_from_py_utility_code(env) - all_members_names = sorted([e.name for e in all_members]) - checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] - unpickle_func_name = '__pyx_unpickle_%s' % node.class_name - - # TODO(robertwb): Move the state into the third argument - # so it can be pickled *after* self is memoized. - unpickle_func = TreeFragment(u""" - def %(unpickle_func_name)s(__pyx_type, long __pyx_checksum, __pyx_state): + if (node.visibility != 'extern' + and not node.scope.lookup('__reduce__') + and not node.scope.lookup('__reduce_ex__')): + self._inject_pickle_methods(node) + return node + + def _inject_pickle_methods(self, node): + env = self.current_env() + if node.scope.directives['auto_pickle'] is False: # None means attempt it. + # Old behavior of not doing anything. + return + auto_pickle_forced = node.scope.directives['auto_pickle'] is True + + all_members = [] + cls = node.entry.type + cinit = None + inherited_reduce = None + while cls is not None: + all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__')) + cinit = cinit or cls.scope.lookup('__cinit__') + inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__') + cls = cls.base_type + all_members.sort(key=lambda e: e.name) + + if inherited_reduce: + # This is not failsafe, as we may not know whether a cimported class defines a __reduce__. + # This is why we define __reduce_cython__ and only replace __reduce__ + # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation. + return + + non_py = [ + e for e in all_members + if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env) + or not e.type.can_coerce_from_pyobject(env)) + ] + + structs = [e for e in all_members if e.type.is_struct_or_union] + + if cinit or non_py or (structs and not auto_pickle_forced): + if cinit: + # TODO(robertwb): We could allow this if __cinit__ has no require arguments. + msg = 'no default __reduce__ due to non-trivial __cinit__' + elif non_py: + msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py) + else: + # Extern structs may be only partially defined. + # TODO(robertwb): Limit the restriction to extern + # (and recursively extern-containing) structs. + msg = ("Pickling of struct members such as %s must be explicitly requested " + "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs)) + + if auto_pickle_forced: + error(node.pos, msg) + + pickle_func = TreeFragment(u""" + def __reduce_cython__(self): + raise TypeError("%(msg)s") + def __setstate_cython__(self, __pyx_state): + raise TypeError("%(msg)s") + """ % {'msg': msg}, + level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) + pickle_func.analyse_declarations(node.scope) + self.visit(pickle_func) + node.body.stats.append(pickle_func) + + else: + for e in all_members: + if not e.type.is_pyobject: + e.type.create_to_py_utility_code(env) + e.type.create_from_py_utility_code(env) + all_members_names = sorted([e.name for e in all_members]) + checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] + unpickle_func_name = '__pyx_unpickle_%s' % node.class_name + + # TODO(robertwb): Move the state into the third argument + # so it can be pickled *after* self is memoized. + unpickle_func = TreeFragment(u""" + def %(unpickle_func_name)s(__pyx_type, long __pyx_checksum, __pyx_state): cdef object __pyx_PickleError cdef object __pyx_result - if __pyx_checksum != %(checksum)s: - from pickle import PickleError as __pyx_PickleError - raise __pyx_PickleError("Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum) - __pyx_result = %(class_name)s.__new__(__pyx_type) - if __pyx_state is not None: - %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) - return __pyx_result - - cdef %(unpickle_func_name)s__set_state(%(class_name)s __pyx_result, tuple __pyx_state): - %(assignments)s - if len(__pyx_state) > %(num_members)d and hasattr(__pyx_result, '__dict__'): - __pyx_result.__dict__.update(__pyx_state[%(num_members)d]) - """ % { - 'unpickle_func_name': unpickle_func_name, - 'checksum': checksum, - 'members': ', '.join(all_members_names), - 'class_name': node.class_name, - 'assignments': '; '.join( - '__pyx_result.%s = __pyx_state[%s]' % (v, ix) - for ix, v in enumerate(all_members_names)), - 'num_members': len(all_members_names), - }, level='module', pipeline=[NormalizeTree(None)]).substitute({}) - unpickle_func.analyse_declarations(node.entry.scope) - self.visit(unpickle_func) - self.extra_module_declarations.append(unpickle_func) - - pickle_func = TreeFragment(u""" - def __reduce_cython__(self): + if __pyx_checksum != %(checksum)s: + from pickle import PickleError as __pyx_PickleError + raise __pyx_PickleError("Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum) + __pyx_result = %(class_name)s.__new__(__pyx_type) + if __pyx_state is not None: + %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) + return __pyx_result + + cdef %(unpickle_func_name)s__set_state(%(class_name)s __pyx_result, tuple __pyx_state): + %(assignments)s + if len(__pyx_state) > %(num_members)d and hasattr(__pyx_result, '__dict__'): + __pyx_result.__dict__.update(__pyx_state[%(num_members)d]) + """ % { + 'unpickle_func_name': unpickle_func_name, + 'checksum': checksum, + 'members': ', '.join(all_members_names), + 'class_name': node.class_name, + 'assignments': '; '.join( + '__pyx_result.%s = __pyx_state[%s]' % (v, ix) + for ix, v in enumerate(all_members_names)), + 'num_members': len(all_members_names), + }, level='module', pipeline=[NormalizeTree(None)]).substitute({}) + unpickle_func.analyse_declarations(node.entry.scope) + self.visit(unpickle_func) + self.extra_module_declarations.append(unpickle_func) + + pickle_func = TreeFragment(u""" + def __reduce_cython__(self): cdef tuple state cdef object _dict - cdef bint use_setstate - state = (%(members)s) - _dict = getattr(self, '__dict__', None) - if _dict is not None: - state += (_dict,) - use_setstate = True - else: - use_setstate = %(any_notnone_members)s - if use_setstate: - return %(unpickle_func_name)s, (type(self), %(checksum)s, None), state - else: - return %(unpickle_func_name)s, (type(self), %(checksum)s, state) - - def __setstate_cython__(self, __pyx_state): - %(unpickle_func_name)s__set_state(self, __pyx_state) - """ % { - 'unpickle_func_name': unpickle_func_name, - 'checksum': checksum, - 'members': ', '.join('self.%s' % v for v in all_members_names) + (',' if len(all_members_names) == 1 else ''), - # Even better, we could check PyType_IS_GC. - 'any_notnone_members' : ' or '.join(['self.%s is not None' % e.name for e in all_members if e.type.is_pyobject] or ['False']), - }, - level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) - pickle_func.analyse_declarations(node.scope) + cdef bint use_setstate + state = (%(members)s) + _dict = getattr(self, '__dict__', None) + if _dict is not None: + state += (_dict,) + use_setstate = True + else: + use_setstate = %(any_notnone_members)s + if use_setstate: + return %(unpickle_func_name)s, (type(self), %(checksum)s, None), state + else: + return %(unpickle_func_name)s, (type(self), %(checksum)s, state) + + def __setstate_cython__(self, __pyx_state): + %(unpickle_func_name)s__set_state(self, __pyx_state) + """ % { + 'unpickle_func_name': unpickle_func_name, + 'checksum': checksum, + 'members': ', '.join('self.%s' % v for v in all_members_names) + (',' if len(all_members_names) == 1 else ''), + # Even better, we could check PyType_IS_GC. + 'any_notnone_members' : ' or '.join(['self.%s is not None' % e.name for e in all_members if e.type.is_pyobject] or ['False']), + }, + level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) + pickle_func.analyse_declarations(node.scope) self.enter_scope(node, node.scope) # functions should be visited in the class scope - self.visit(pickle_func) + self.visit(pickle_func) self.exit_scope() - node.body.stats.append(pickle_func) - + node.body.stats.append(pickle_func) + def _handle_fused_def_decorators(self, old_decorators, env, node): """ Create function calls to the decorators and reassignments to @@ -1868,7 +1868,7 @@ if VALUE is not None: def visit_FuncDefNode(self, node): """ - Analyse a function and its body, as that hasn't happened yet. Also + Analyse a function and its body, as that hasn't happened yet. Also analyse the directive_locals set by @cython.locals(). Then, if we are a function with fused arguments, replace the function @@ -1931,8 +1931,8 @@ if VALUE is not None: binding = self.current_directives.get('binding') rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding) node.code_object = rhs.code_object - if node.is_generator: - node.gbody.code_object = node.code_object + if node.is_generator: + node.gbody.code_object = node.code_object if env.is_py_class_scope: rhs.binding = True @@ -2059,7 +2059,7 @@ if VALUE is not None: # Some nodes are no longer needed after declaration # analysis and can be dropped. The analysis was performed - # on these nodes in a separate recursive process from the + # on these nodes in a separate recursive process from the # enclosing function or module, so we can simply drop them. def visit_CDeclaratorNode(self, node): # necessary to ensure that all CNameDeclaratorNodes are visited. @@ -2354,20 +2354,20 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations): if 'inline' in self.directives: modifiers.append('inline') nogil = self.directives.get('nogil') - except_val = self.directives.get('exceptval') - return_type_node = self.directives.get('returns') - if return_type_node is None and self.directives['annotation_typing']: - return_type_node = node.return_type_annotation - # for Python anntations, prefer safe exception handling by default - if return_type_node is not None and except_val is None: - except_val = (None, True) # except * - elif except_val is None: - # backward compatible default: no exception check - except_val = (None, False) + except_val = self.directives.get('exceptval') + return_type_node = self.directives.get('returns') + if return_type_node is None and self.directives['annotation_typing']: + return_type_node = node.return_type_annotation + # for Python anntations, prefer safe exception handling by default + if return_type_node is not None and except_val is None: + except_val = (None, True) # except * + elif except_val is None: + # backward compatible default: no exception check + except_val = (None, False) if 'ccall' in self.directives: node = node.as_cfunction( overridable=True, modifiers=modifiers, nogil=nogil, - returns=return_type_node, except_val=except_val) + returns=return_type_node, except_val=except_val) return self.visit(node) if 'cfunc' in self.directives: if self.in_py_class: @@ -2375,7 +2375,7 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations): else: node = node.as_cfunction( overridable=False, modifiers=modifiers, nogil=nogil, - returns=return_type_node, except_val=except_val) + returns=return_type_node, except_val=except_val) return self.visit(node) if 'inline' in modifiers: error(node.pos, "Python functions cannot be declared 'inline'") @@ -2531,23 +2531,23 @@ class YieldNodeCollector(TreeVisitor): super(YieldNodeCollector, self).__init__() self.yields = [] self.returns = [] - self.finallys = [] - self.excepts = [] + self.finallys = [] + self.excepts = [] self.has_return_value = False - self.has_yield = False - self.has_await = False + self.has_yield = False + self.has_await = False def visit_Node(self, node): self.visitchildren(node) def visit_YieldExprNode(self, node): self.yields.append(node) - self.has_yield = True + self.has_yield = True self.visitchildren(node) def visit_AwaitExprNode(self, node): - self.yields.append(node) - self.has_await = True + self.yields.append(node) + self.has_await = True self.visitchildren(node) def visit_ReturnStatNode(self, node): @@ -2556,14 +2556,14 @@ class YieldNodeCollector(TreeVisitor): self.has_return_value = True self.returns.append(node) - def visit_TryFinallyStatNode(self, node): - self.visitchildren(node) - self.finallys.append(node) - - def visit_TryExceptStatNode(self, node): - self.visitchildren(node) - self.excepts.append(node) - + def visit_TryFinallyStatNode(self, node): + self.visitchildren(node) + self.finallys.append(node) + + def visit_TryExceptStatNode(self, node): + self.visitchildren(node) + self.excepts.append(node) + def visit_ClassDefNode(self, node): pass @@ -2599,31 +2599,31 @@ class MarkClosureVisitor(CythonTransform): collector.visitchildren(node) if node.is_async_def: - coroutine_type = Nodes.AsyncDefNode - if collector.has_yield: - coroutine_type = Nodes.AsyncGenNode - for yield_expr in collector.yields + collector.returns: - yield_expr.in_async_gen = True - elif self.current_directives['iterable_coroutine']: - coroutine_type = Nodes.IterableAsyncDefNode - elif collector.has_await: - found = next(y for y in collector.yields if y.is_await) - error(found.pos, "'await' not allowed in generators (use 'yield')") - return node - elif collector.has_yield: - coroutine_type = Nodes.GeneratorDefNode + coroutine_type = Nodes.AsyncDefNode + if collector.has_yield: + coroutine_type = Nodes.AsyncGenNode + for yield_expr in collector.yields + collector.returns: + yield_expr.in_async_gen = True + elif self.current_directives['iterable_coroutine']: + coroutine_type = Nodes.IterableAsyncDefNode + elif collector.has_await: + found = next(y for y in collector.yields if y.is_await) + error(found.pos, "'await' not allowed in generators (use 'yield')") + return node + elif collector.has_yield: + coroutine_type = Nodes.GeneratorDefNode else: return node - for i, yield_expr in enumerate(collector.yields, 1): + for i, yield_expr in enumerate(collector.yields, 1): yield_expr.label_num = i - for retnode in collector.returns + collector.finallys + collector.excepts: + for retnode in collector.returns + collector.finallys + collector.excepts: retnode.in_generator = True gbody = Nodes.GeneratorBodyDefNode( - pos=node.pos, name=node.name, body=node.body, - is_async_gen_body=node.is_async_def and collector.has_yield) - coroutine = coroutine_type( + pos=node.pos, name=node.name, body=node.body, + is_async_gen_body=node.is_async_def and collector.has_yield) + coroutine = coroutine_type( pos=node.pos, name=node.name, args=node.args, star_arg=node.star_arg, starstar_arg=node.starstar_arg, doc=node.doc, decorators=node.decorators, @@ -2670,28 +2670,28 @@ class CreateClosureClasses(CythonTransform): def find_entries_used_in_closures(self, node): from_closure = [] in_closure = [] - for scope in node.local_scope.iter_local_scopes(): - for name, entry in scope.entries.items(): - if not name: - continue - if entry.from_closure: - from_closure.append((name, entry)) - elif entry.in_closure: - in_closure.append((name, entry)) + for scope in node.local_scope.iter_local_scopes(): + for name, entry in scope.entries.items(): + if not name: + continue + if entry.from_closure: + from_closure.append((name, entry)) + elif entry.in_closure: + in_closure.append((name, entry)) return from_closure, in_closure def create_class_from_scope(self, node, target_module_scope, inner_node=None): # move local variables into closure if node.is_generator: - for scope in node.local_scope.iter_local_scopes(): - for entry in scope.entries.values(): + for scope in node.local_scope.iter_local_scopes(): + for entry in scope.entries.values(): if not (entry.from_closure or entry.is_pyglobal or entry.is_cglobal): - entry.in_closure = True + entry.in_closure = True from_closure, in_closure = self.find_entries_used_in_closures(node) in_closure.sort() - # Now from the beginning + # Now from the beginning node.needs_closure = False node.needs_outer_scope = False @@ -2733,10 +2733,10 @@ class CreateClosureClasses(CythonTransform): func_scope.scope_class = entry class_scope = entry.type.scope class_scope.is_internal = True - class_scope.is_closure_class_scope = True - if node.is_async_def or node.is_generator: - # Generators need their closure intact during cleanup as they resume to handle GeneratorExit - class_scope.directives['no_gc_clear'] = True + class_scope.is_closure_class_scope = True + if node.is_async_def or node.is_generator: + # Generators need their closure intact during cleanup as they resume to handle GeneratorExit + class_scope.directives['no_gc_clear'] = True if Options.closure_freelist_size: class_scope.directives['freelist'] = Options.closure_freelist_size @@ -2749,12 +2749,12 @@ class CreateClosureClasses(CythonTransform): is_cdef=True) node.needs_outer_scope = True for name, entry in in_closure: - closure_entry = class_scope.declare_var( - pos=entry.pos, - name=entry.name if not entry.in_subscope else None, - cname=entry.cname, - type=entry.type, - is_cdef=True) + closure_entry = class_scope.declare_var( + pos=entry.pos, + name=entry.name if not entry.in_subscope else None, + cname=entry.cname, + type=entry.type, + is_cdef=True) if entry.is_declared_generic: closure_entry.is_declared_generic = 1 node.needs_closure = True @@ -3191,22 +3191,22 @@ class TransformBuiltinMethods(EnvTransform): def visit_GeneralCallNode(self, node): function = node.function.as_cython_attribute() - if function == u'cast': - # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args + if function == u'cast': + # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args args = node.positional_args.args kwargs = node.keyword_args.compile_time_value(None) - if (len(args) != 2 or len(kwargs) > 1 or - (len(kwargs) == 1 and 'typecheck' not in kwargs)): - error(node.function.pos, - u"cast() takes exactly two arguments and an optional typecheck keyword") - else: - type = args[0].analyse_as_type(self.current_env()) - if type: - typecheck = kwargs.get('typecheck', False) - node = ExprNodes.TypecastNode( - node.function.pos, type=type, operand=args[1], typecheck=typecheck) + if (len(args) != 2 or len(kwargs) > 1 or + (len(kwargs) == 1 and 'typecheck' not in kwargs)): + error(node.function.pos, + u"cast() takes exactly two arguments and an optional typecheck keyword") + else: + type = args[0].analyse_as_type(self.current_env()) + if type: + typecheck = kwargs.get('typecheck', False) + node = ExprNodes.TypecastNode( + node.function.pos, type=type, operand=args[1], typecheck=typecheck) else: - error(args[0].pos, "Not a type") + error(args[0].pos, "Not a type") self.visitchildren(node) return node @@ -3239,9 +3239,9 @@ class ReplaceFusedTypeChecks(VisitorTransform): return self.transform(node) def visit_PrimaryCmpNode(self, node): - with Errors.local_errors(ignore=True): - type1 = node.operand1.analyse_as_type(self.local_scope) - type2 = node.operand2.analyse_as_type(self.local_scope) + with Errors.local_errors(ignore=True): + type1 = node.operand1.analyse_as_type(self.local_scope) + type2 = node.operand2.analyse_as_type(self.local_scope) if type1 and type2: false_node = ExprNodes.BoolNode(node.pos, value=False) |