diff options
Diffstat (limited to 'contrib/tools/python3/Python/symtable.c')
| -rw-r--r-- | contrib/tools/python3/Python/symtable.c | 348 |
1 files changed, 262 insertions, 86 deletions
diff --git a/contrib/tools/python3/Python/symtable.c b/contrib/tools/python3/Python/symtable.c index f99ca4fdd06..f60af2b6955 100644 --- a/contrib/tools/python3/Python/symtable.c +++ b/contrib/tools/python3/Python/symtable.c @@ -1,9 +1,11 @@ #include "Python.h" -#include "pycore_ast.h" // identifier, stmt_ty +#include "pycore_ast.h" // stmt_ty #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntryObject -#include "structmember.h" // PyMemberDef + +// Set this to 1 to dump all symtables to stdout for debugging +#define _PY_DUMP_SYMTABLE 0 /* error strings used for warnings */ #define GLOBAL_PARAM \ @@ -56,13 +58,13 @@ #define ANNOTATION_NOT_ALLOWED \ "%s cannot be used within an annotation" -#define TYPEVAR_BOUND_NOT_ALLOWED \ -"%s cannot be used within a TypeVar bound" +#define EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE \ +"%s cannot be used within %s" -#define TYPEALIAS_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_ALIAS \ "%s cannot be used within a type alias" -#define TYPEPARAM_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS \ "%s cannot be used within the definition of a generic" #define DUPLICATE_TYPE_PARAM \ @@ -104,6 +106,8 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_mangled_names = NULL; ste->ste_type = block; + ste->ste_scope_info = NULL; + ste->ste_nested = 0; ste->ste_free = 0; ste->ste_varargs = 0; @@ -172,14 +176,14 @@ ste_dealloc(PySTEntryObject *ste) #define OFF(x) offsetof(PySTEntryObject, x) static PyMemberDef ste_memberlist[] = { - {"id", T_OBJECT, OFF(ste_id), READONLY}, - {"name", T_OBJECT, OFF(ste_name), READONLY}, - {"symbols", T_OBJECT, OFF(ste_symbols), READONLY}, - {"varnames", T_OBJECT, OFF(ste_varnames), READONLY}, - {"children", T_OBJECT, OFF(ste_children), READONLY}, - {"nested", T_INT, OFF(ste_nested), READONLY}, - {"type", T_INT, OFF(ste_type), READONLY}, - {"lineno", T_INT, OFF(ste_lineno), READONLY}, + {"id", _Py_T_OBJECT, OFF(ste_id), Py_READONLY}, + {"name", _Py_T_OBJECT, OFF(ste_name), Py_READONLY}, + {"symbols", _Py_T_OBJECT, OFF(ste_symbols), Py_READONLY}, + {"varnames", _Py_T_OBJECT, OFF(ste_varnames), Py_READONLY}, + {"children", _Py_T_OBJECT, OFF(ste_children), Py_READONLY}, + {"nested", Py_T_INT, OFF(ste_nested), Py_READONLY}, + {"type", Py_T_INT, OFF(ste_type), Py_READONLY}, + {"lineno", Py_T_INT, OFF(ste_lineno), Py_READONLY}, {NULL} }; @@ -253,6 +257,109 @@ static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty); +/* For debugging purposes only */ +#if _PY_DUMP_SYMTABLE +static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) +{ + const char *blocktype = ""; + switch (ste->ste_type) { + case FunctionBlock: blocktype = "FunctionBlock"; break; + case ClassBlock: blocktype = "ClassBlock"; break; + case ModuleBlock: blocktype = "ModuleBlock"; break; + case AnnotationBlock: blocktype = "AnnotationBlock"; break; + case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; + case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; + case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; + } + const char *comptype = ""; + switch (ste->ste_comprehension) { + case ListComprehension: comptype = " ListComprehension"; break; + case DictComprehension: comptype = " DictComprehension"; break; + case SetComprehension: comptype = " SetComprehension"; break; + case GeneratorExpression: comptype = " GeneratorExpression"; break; + case NoComprehension: break; + } + PyObject* msg = PyUnicode_FromFormat( + ( + "%U=== Symtable for %U ===\n" + "%U%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s%s%s%s\n" + "%Ulineno: %d col_offset: %d\n" + "%U--- Symbols ---\n" + ), + prefix, + ste->ste_name, + prefix, + blocktype, + comptype, + prefix, + ste->ste_nested ? " nested" : "", + ste->ste_free ? " free" : "", + ste->ste_child_free ? " child_free" : "", + ste->ste_generator ? " generator" : "", + ste->ste_coroutine ? " coroutine" : "", + ste->ste_varargs ? " varargs" : "", + ste->ste_varkeywords ? " varkeywords" : "", + ste->ste_returns_value ? " returns_value" : "", + ste->ste_needs_class_closure ? " needs_class_closure" : "", + ste->ste_needs_classdict ? " needs_classdict" : "", + ste->ste_comp_inlined ? " comp_inlined" : "", + ste->ste_comp_iter_target ? " comp_iter_target" : "", + ste->ste_can_see_class_scope ? " can_see_class_scope" : "", + prefix, + ste->ste_lineno, + ste->ste_col_offset, + prefix + ); + assert(msg != NULL); + printf("%s", PyUnicode_AsUTF8(msg)); + Py_DECREF(msg); + PyObject *name, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(ste->ste_symbols, &pos, &name, &value)) { + int scope = _PyST_GetScope(ste, name); + long flags = _PyST_GetSymbol(ste, name); + printf("%s %s: ", PyUnicode_AsUTF8(prefix), PyUnicode_AsUTF8(name)); + if (flags & DEF_GLOBAL) printf(" DEF_GLOBAL"); + if (flags & DEF_LOCAL) printf(" DEF_LOCAL"); + if (flags & DEF_PARAM) printf(" DEF_PARAM"); + if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL"); + if (flags & USE) printf(" USE"); + if (flags & DEF_FREE) printf(" DEF_FREE"); + if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS"); + if (flags & DEF_IMPORT) printf(" DEF_IMPORT"); + if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); + if (flags & DEF_COMP_ITER) printf(" DEF_COMP_ITER"); + if (flags & DEF_TYPE_PARAM) printf(" DEF_TYPE_PARAM"); + if (flags & DEF_COMP_CELL) printf(" DEF_COMP_CELL"); + switch (scope) { + case LOCAL: printf(" LOCAL"); break; + case GLOBAL_EXPLICIT: printf(" GLOBAL_EXPLICIT"); break; + case GLOBAL_IMPLICIT: printf(" GLOBAL_IMPLICIT"); break; + case FREE: printf(" FREE"); break; + case CELL: printf(" CELL"); break; + } + printf("\n"); + } + printf("%s--- Children ---\n", PyUnicode_AsUTF8(prefix)); + PyObject *new_prefix = PyUnicode_FromFormat(" %U", prefix); + assert(new_prefix != NULL); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(ste->ste_children); i++) { + PyObject *child = PyList_GetItem(ste->ste_children, i); + assert(child != NULL && PySTEntry_Check(child)); + _dump_symtable((PySTEntryObject *)child, new_prefix); + } + Py_DECREF(new_prefix); +} + +static void dump_symtable(PySTEntryObject* ste) +{ + PyObject *empty = PyUnicode_FromString(""); + assert(empty != NULL); + _dump_symtable(ste, empty); + Py_DECREF(empty); +} +#endif #define DUPLICATE_ARGUMENT \ "duplicate argument '%U' in function definition" @@ -284,7 +391,7 @@ symtable_new(void) } struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; @@ -308,10 +415,10 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) return NULL; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = C_RECURSION_LIMIT; + st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { @@ -357,8 +464,12 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) return NULL; } /* Make the second symbol analysis pass */ - if (symtable_analyze(st)) + if (symtable_analyze(st)) { +#if _PY_DUMP_SYMTABLE + dump_symtable(st->st_top); +#endif return st; + } _PySymtable_Free(st); return NULL; error: @@ -378,25 +489,21 @@ _PySymtable_Free(struct symtable *st) } PySTEntryObject * -PySymtable_Lookup(struct symtable *st, void *key) +_PySymtable_Lookup(struct symtable *st, void *key) { PyObject *k, *v; k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItemWithError(st->st_blocks, k); - Py_DECREF(k); - - if (v) { - assert(PySTEntry_Check(v)); - } - else if (!PyErr_Occurred()) { + if (PyDict_GetItemRef(st->st_blocks, k, &v) == 0) { PyErr_SetString(PyExc_KeyError, "unknown symbol table entry"); } + Py_DECREF(k); - return (PySTEntryObject *)Py_XNewRef(v); + assert(v == NULL || PySTEntry_Check(v)); + return (PySTEntryObject *)v; } long @@ -420,9 +527,9 @@ int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock - || ste->ste_type == TypeVarBoundBlock + || ste->ste_type == TypeVariableBlock || ste->ste_type == TypeAliasBlock - || ste->ste_type == TypeParamBlock; + || ste->ste_type == TypeParametersBlock; } static int @@ -656,6 +763,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *k, *v; Py_ssize_t pos = 0; int remove_dunder_class = 0; + int remove_dunder_classdict = 0; + int remove_dunder_cond_annotations = 0; while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter @@ -675,15 +784,27 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } - // __class__ is never allowed to be free through a class scope (see + // __class__, __classdict__ and __conditional_annotations__ are + // never allowed to be free through a class scope (see // drop_class_free) if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { + (_PyUnicode_EqualToASCIIString(k, "__class__") || + _PyUnicode_EqualToASCIIString(k, "__classdict__") || + _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { scope = GLOBAL_IMPLICIT; if (PySet_Discard(comp_free, k) < 0) { return 0; } - remove_dunder_class = 1; + + if (_PyUnicode_EqualToASCIIString(k, "__class__")) { + remove_dunder_class = 1; + } + else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { + remove_dunder_cond_annotations = 1; + } + else { + remove_dunder_classdict = 1; + } } if (!existing) { // name does not exist in scope, copy from comprehension @@ -716,6 +837,12 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { return 0; } + if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { + return 0; + } + if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { + return 0; + } return 1; } @@ -1048,10 +1175,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions + // we inline all non-generator-expression comprehensions, + // except those in annotation scopes that are nested in classes int inline_comp = entry->ste_comprehension && - !entry->ste_generator; + !entry->ste_generator && + !ste->ste_can_see_class_scope; if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) @@ -1387,7 +1516,7 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, int end_lineno, int end_col_offset) { _Py_block_ty current_type = st->st_cur->ste_type; - if(!symtable_enter_block(st, name, TypeParamBlock, ast, lineno, + if(!symtable_enter_block(st, name, TypeParametersBlock, ast, lineno, col_offset, end_lineno, end_col_offset)) { return 0; } @@ -1742,14 +1871,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Try_kind: VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case TryStar_kind: VISIT_SEQ(st, stmt, s->v.TryStar.body); - VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); + VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); break; case Assert_kind: @@ -1962,20 +2091,20 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) } /* Disallow usage in ClassBlock and type scopes */ if (ste->ste_type == ClassBlock || - ste->ste_type == TypeParamBlock || + ste->ste_type == TypeParametersBlock || ste->ste_type == TypeAliasBlock || - ste->ste_type == TypeVarBoundBlock) { + ste->ste_type == TypeVariableBlock) { switch (ste->ste_type) { case ClassBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS); break; - case TypeParamBlock: + case TypeParametersBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEPARAM); break; case TypeAliasBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEALIAS); break; - case TypeVarBoundBlock: + case TypeVariableBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEVAR_BOUND); break; default: @@ -2047,17 +2176,6 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.UnaryOp.operand); break; case Lambda_kind: { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use lambda in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); if (e->v.Lambda.args->kw_defaults) @@ -2192,6 +2310,47 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } static int +symtable_visit_type_param_bound_or_default( + struct symtable *st, expr_ty e, identifier name, + type_param_ty tp, const char *ste_scope_info) +{ + if (_PyUnicode_Equal(name, &_Py_ID(__classdict__))) { + + PyObject *error_msg = PyUnicode_FromFormat("reserved name '%U' cannot be " + "used for type parameter", name); + PyErr_SetObject(PyExc_SyntaxError, error_msg); + Py_DECREF(error_msg); + PyErr_RangedSyntaxLocationObject(st->st_filename, + tp->lineno, + tp->col_offset + 1, + tp->end_lineno, + tp->end_col_offset + 1); + return 0; + } + + if (e) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + if (!symtable_enter_block(st, name, TypeVariableBlock, (void *)tp, LOCATION(e))) { + return 0; + } + + st->st_cur->ste_can_see_class_scope = is_in_class; + if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) { + VISIT_QUIT(st, 0); + } + + assert(ste_scope_info != NULL); + st->st_cur->ste_scope_info = ste_scope_info; + VISIT(st, expr, e); + + if (!symtable_exit_block(st)) { + return 0; + } + } + return 1; +} + +static int symtable_visit_type_param(struct symtable *st, type_param_ty tp) { if (++st->recursion_depth > st->recursion_limit) { @@ -2203,28 +2362,48 @@ symtable_visit_type_param(struct symtable *st, type_param_ty tp) case TypeVar_kind: if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) VISIT_QUIT(st, 0); - if (tp->v.TypeVar.bound) { - int is_in_class = st->st_cur->ste_can_see_class_scope; - if (!symtable_enter_block(st, tp->v.TypeVar.name, - TypeVarBoundBlock, (void *)tp, - LOCATION(tp))) - VISIT_QUIT(st, 0); - st->st_cur->ste_can_see_class_scope = is_in_class; - if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(tp->v.TypeVar.bound))) { - VISIT_QUIT(st, 0); - } - VISIT(st, expr, tp->v.TypeVar.bound); - if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + + const char *ste_scope_info = NULL; + const expr_ty bound = tp->v.TypeVar.bound; + if (bound != NULL) { + ste_scope_info = bound->kind == Tuple_kind ? "a TypeVar constraint" : "a TypeVar bound"; + } + + // We must use a different key for the bound and default. The obvious choice would be to + // use the .bound and .default_value pointers, but that fails when the expression immediately + // inside the bound or default is a comprehension: we would reuse the same key for + // the comprehension scope. Therefore, use the address + 1 as the second key. + // The only requirement for the key is that it is unique and it matches the logic in + // compile.c where the scope is retrieved. + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name, + tp, ste_scope_info)) { + VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name, + (type_param_ty)((uintptr_t)tp + 1), "a TypeVar default")) { + VISIT_QUIT(st, 0); } break; case TypeVarTuple_kind: - if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name, + tp, "a TypeVarTuple default")) { + VISIT_QUIT(st, 0); + } break; case ParamSpec_kind: - if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name, + tp, "a ParamSpec default")) { + VISIT_QUIT(st, 0); + } break; } VISIT_QUIT(st, 1); @@ -2507,18 +2686,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty value) { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use comprehension in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } - int is_generator = (e->kind == GeneratorExp_kind); comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); @@ -2616,12 +2783,21 @@ symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_t enum _block_type type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); - else if (type == TypeVarBoundBlock) - PyErr_Format(PyExc_SyntaxError, TYPEVAR_BOUND_NOT_ALLOWED, name); - else if (type == TypeAliasBlock) - PyErr_Format(PyExc_SyntaxError, TYPEALIAS_NOT_ALLOWED, name); - else if (type == TypeParamBlock) - PyErr_Format(PyExc_SyntaxError, TYPEPARAM_NOT_ALLOWED, name); + else if (type == TypeVariableBlock) { + const char *info = st->st_cur->ste_scope_info; + assert(info != NULL); // e.g., info == "a ParamSpec default" + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE, name, info); + } + else if (type == TypeAliasBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_ALIAS, name); + } + else if (type == TypeParametersBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS, name); + } else return 1; @@ -2664,7 +2840,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { _PyArena_Free(arena); return NULL; |