diff options
| author | orivej <[email protected]> | 2022-02-10 16:44:49 +0300 |
|---|---|---|
| committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:44:49 +0300 |
| commit | 718c552901d703c502ccbefdfc3c9028d608b947 (patch) | |
| tree | 46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /contrib/tools/python3/src/Modules/parsermodule.c | |
| parent | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff) | |
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/python3/src/Modules/parsermodule.c')
| -rw-r--r-- | contrib/tools/python3/src/Modules/parsermodule.c | 2304 |
1 files changed, 1152 insertions, 1152 deletions
diff --git a/contrib/tools/python3/src/Modules/parsermodule.c b/contrib/tools/python3/src/Modules/parsermodule.c index 24b0ffbe36a..ea94ceb6180 100644 --- a/contrib/tools/python3/src/Modules/parsermodule.c +++ b/contrib/tools/python3/src/Modules/parsermodule.c @@ -1,665 +1,665 @@ -/* parsermodule.c - * - * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic - * Institute and State University, Blacksburg, Virginia, USA. - * Portions copyright 1991-1995 by Stichting Mathematisch Centrum, - * Amsterdam, The Netherlands. Copying is permitted under the terms - * associated with the main Python distribution, with the additional - * restriction that this additional notice be included and maintained - * on all distributed copies. - * - * This module serves to replace the original parser module written - * by Guido. The functionality is not matched precisely, but the - * original may be implemented on top of this. This is desirable - * since the source of the text to be parsed is now divorced from - * this interface. - * - * Unlike the prior interface, the ability to give a parse tree - * produced by Python code as a tuple to the compiler is enabled by - * this module. See the documentation for more details. - * - * I've added some annotations that help with the lint code-checking - * program, but they're not complete by a long shot. The real errors - * that lint detects are gone, but there are still warnings with - * Py_[X]DECREF() and Py_[X]INCREF() macros. The lint annotations - * look like "NOTE(...)". - * - */ - -#include "Python.h" /* general Python API */ -#include "Python-ast.h" /* mod_ty */ +/* parsermodule.c + * + * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic + * Institute and State University, Blacksburg, Virginia, USA. + * Portions copyright 1991-1995 by Stichting Mathematisch Centrum, + * Amsterdam, The Netherlands. Copying is permitted under the terms + * associated with the main Python distribution, with the additional + * restriction that this additional notice be included and maintained + * on all distributed copies. + * + * This module serves to replace the original parser module written + * by Guido. The functionality is not matched precisely, but the + * original may be implemented on top of this. This is desirable + * since the source of the text to be parsed is now divorced from + * this interface. + * + * Unlike the prior interface, the ability to give a parse tree + * produced by Python code as a tuple to the compiler is enabled by + * this module. See the documentation for more details. + * + * I've added some annotations that help with the lint code-checking + * program, but they're not complete by a long shot. The real errors + * that lint detects are gone, but there are still warnings with + * Py_[X]DECREF() and Py_[X]INCREF() macros. The lint annotations + * look like "NOTE(...)". + * + */ + +#include "Python.h" /* general Python API */ +#include "Python-ast.h" /* mod_ty */ #undef Yield /* undefine macro conflicting with <winbase.h> */ #include "ast.h" -#include "graminit.h" /* symbols defined in the grammar */ -#include "node.h" /* internal parser structure */ -#include "errcode.h" /* error codes for PyNode_*() */ -#include "token.h" /* token definitions */ +#include "graminit.h" /* symbols defined in the grammar */ +#include "node.h" /* internal parser structure */ +#include "errcode.h" /* error codes for PyNode_*() */ +#include "token.h" /* token definitions */ /* ISTERMINAL() / ISNONTERMINAL() */ -#include "grammar.h" -#include "parsetok.h" - -extern grammar _PyParser_Grammar; /* From graminit.c */ - -#ifdef lint -#include <note.h> -#else -#define NOTE(x) -#endif - -/* String constants used to initialize module attributes. - * - */ -static const char parser_copyright_string[] = -"Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ -University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ -Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ -Centrum, Amsterdam, The Netherlands."; - - -PyDoc_STRVAR(parser_doc_string, -"This is an interface to Python's internal parser."); - -static const char parser_version_string[] = "0.5"; - - -typedef PyObject* (*SeqMaker) (Py_ssize_t length); -typedef int (*SeqInserter) (PyObject* sequence, - Py_ssize_t index, - PyObject* element); - -/* The function below is copyrighted by Stichting Mathematisch Centrum. The - * original copyright statement is included below, and continues to apply - * in full to the function immediately following. All other material is - * original, copyrighted by Fred L. Drake, Jr. and Virginia Polytechnic - * Institute and State University. Changes were made to comply with the - * new naming conventions. Added arguments to provide support for creating - * lists as well as tuples, and optionally including the line numbers. - */ - - -static PyObject* -node2tuple(node *n, /* node to convert */ - SeqMaker mkseq, /* create sequence */ - SeqInserter addelem, /* func. to add elem. in seq. */ - int lineno, /* include line numbers? */ - int col_offset) /* include column offsets? */ -{ - PyObject *result = NULL, *w; - - if (n == NULL) { - Py_RETURN_NONE; - } - - if (ISNONTERMINAL(TYPE(n))) { - int i; - - result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl)); - if (result == NULL) - goto error; - - w = PyLong_FromLong(TYPE(n)); - if (w == NULL) - goto error; - (void) addelem(result, 0, w); - - for (i = 0; i < NCH(n); i++) { - w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); - if (w == NULL) - goto error; - (void) addelem(result, i+1, w); - } - - if (TYPE(n) == encoding_decl) { - w = PyUnicode_FromString(STR(n)); - if (w == NULL) - goto error; - (void) addelem(result, i+1, w); - } - } - else if (ISTERMINAL(TYPE(n))) { - result = mkseq(2 + lineno + col_offset); - if (result == NULL) - goto error; - - w = PyLong_FromLong(TYPE(n)); - if (w == NULL) - goto error; - (void) addelem(result, 0, w); - - w = PyUnicode_FromString(STR(n)); - if (w == NULL) - goto error; - (void) addelem(result, 1, w); - - if (lineno) { - w = PyLong_FromLong(n->n_lineno); - if (w == NULL) - goto error; - (void) addelem(result, 2, w); - } - - if (col_offset) { - w = PyLong_FromLong(n->n_col_offset); - if (w == NULL) - goto error; - (void) addelem(result, 2 + lineno, w); - } - } - else { - PyErr_SetString(PyExc_SystemError, - "unrecognized parse tree node type"); - return ((PyObject*) NULL); - } - return result; - -error: - Py_XDECREF(result); - return NULL; -} -/* - * End of material copyrighted by Stichting Mathematisch Centrum. - */ - - - -/* There are two types of intermediate objects we're interested in: - * 'eval' and 'exec' types. These constants can be used in the st_type - * field of the object type to identify which any given object represents. - * These should probably go in an external header to allow other extensions - * to use them, but then, we really should be using C++ too. ;-) - */ - -#define PyST_EXPR 1 -#define PyST_SUITE 2 - - -/* These are the internal objects and definitions required to implement the - * ST type. Most of the internal names are more reminiscent of the 'old' - * naming style, but the code uses the new naming convention. - */ - -static PyObject* -parser_error = 0; - - -typedef struct { - PyObject_HEAD /* standard object header */ - node* st_node; /* the node* returned by the parser */ - int st_type; /* EXPR or SUITE ? */ - PyCompilerFlags st_flags; /* Parser and compiler flags */ -} PyST_Object; - - -static void parser_free(PyST_Object *st); -static PyObject* parser_sizeof(PyST_Object *, void *); -static PyObject* parser_richcompare(PyObject *left, PyObject *right, int op); -static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *); -static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *); -static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *); -static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *); -static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *); - -#define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS) - -static PyMethodDef parser_methods[] = { +#include "grammar.h" +#include "parsetok.h" + +extern grammar _PyParser_Grammar; /* From graminit.c */ + +#ifdef lint +#include <note.h> +#else +#define NOTE(x) +#endif + +/* String constants used to initialize module attributes. + * + */ +static const char parser_copyright_string[] = +"Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ +University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ +Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ +Centrum, Amsterdam, The Netherlands."; + + +PyDoc_STRVAR(parser_doc_string, +"This is an interface to Python's internal parser."); + +static const char parser_version_string[] = "0.5"; + + +typedef PyObject* (*SeqMaker) (Py_ssize_t length); +typedef int (*SeqInserter) (PyObject* sequence, + Py_ssize_t index, + PyObject* element); + +/* The function below is copyrighted by Stichting Mathematisch Centrum. The + * original copyright statement is included below, and continues to apply + * in full to the function immediately following. All other material is + * original, copyrighted by Fred L. Drake, Jr. and Virginia Polytechnic + * Institute and State University. Changes were made to comply with the + * new naming conventions. Added arguments to provide support for creating + * lists as well as tuples, and optionally including the line numbers. + */ + + +static PyObject* +node2tuple(node *n, /* node to convert */ + SeqMaker mkseq, /* create sequence */ + SeqInserter addelem, /* func. to add elem. in seq. */ + int lineno, /* include line numbers? */ + int col_offset) /* include column offsets? */ +{ + PyObject *result = NULL, *w; + + if (n == NULL) { + Py_RETURN_NONE; + } + + if (ISNONTERMINAL(TYPE(n))) { + int i; + + result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl)); + if (result == NULL) + goto error; + + w = PyLong_FromLong(TYPE(n)); + if (w == NULL) + goto error; + (void) addelem(result, 0, w); + + for (i = 0; i < NCH(n); i++) { + w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); + if (w == NULL) + goto error; + (void) addelem(result, i+1, w); + } + + if (TYPE(n) == encoding_decl) { + w = PyUnicode_FromString(STR(n)); + if (w == NULL) + goto error; + (void) addelem(result, i+1, w); + } + } + else if (ISTERMINAL(TYPE(n))) { + result = mkseq(2 + lineno + col_offset); + if (result == NULL) + goto error; + + w = PyLong_FromLong(TYPE(n)); + if (w == NULL) + goto error; + (void) addelem(result, 0, w); + + w = PyUnicode_FromString(STR(n)); + if (w == NULL) + goto error; + (void) addelem(result, 1, w); + + if (lineno) { + w = PyLong_FromLong(n->n_lineno); + if (w == NULL) + goto error; + (void) addelem(result, 2, w); + } + + if (col_offset) { + w = PyLong_FromLong(n->n_col_offset); + if (w == NULL) + goto error; + (void) addelem(result, 2 + lineno, w); + } + } + else { + PyErr_SetString(PyExc_SystemError, + "unrecognized parse tree node type"); + return ((PyObject*) NULL); + } + return result; + +error: + Py_XDECREF(result); + return NULL; +} +/* + * End of material copyrighted by Stichting Mathematisch Centrum. + */ + + + +/* There are two types of intermediate objects we're interested in: + * 'eval' and 'exec' types. These constants can be used in the st_type + * field of the object type to identify which any given object represents. + * These should probably go in an external header to allow other extensions + * to use them, but then, we really should be using C++ too. ;-) + */ + +#define PyST_EXPR 1 +#define PyST_SUITE 2 + + +/* These are the internal objects and definitions required to implement the + * ST type. Most of the internal names are more reminiscent of the 'old' + * naming style, but the code uses the new naming convention. + */ + +static PyObject* +parser_error = 0; + + +typedef struct { + PyObject_HEAD /* standard object header */ + node* st_node; /* the node* returned by the parser */ + int st_type; /* EXPR or SUITE ? */ + PyCompilerFlags st_flags; /* Parser and compiler flags */ +} PyST_Object; + + +static void parser_free(PyST_Object *st); +static PyObject* parser_sizeof(PyST_Object *, void *); +static PyObject* parser_richcompare(PyObject *left, PyObject *right, int op); +static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *); +static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *); +static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *); +static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *); +static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *); + +#define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS) + +static PyMethodDef parser_methods[] = { {"compile", (PyCFunction)(void(*)(void))parser_compilest, PUBLIC_METHOD_TYPE, - PyDoc_STR("Compile this ST object into a code object.")}, + PyDoc_STR("Compile this ST object into a code object.")}, {"isexpr", (PyCFunction)(void(*)(void))parser_isexpr, PUBLIC_METHOD_TYPE, - PyDoc_STR("Determines if this ST object was created from an expression.")}, + PyDoc_STR("Determines if this ST object was created from an expression.")}, {"issuite", (PyCFunction)(void(*)(void))parser_issuite, PUBLIC_METHOD_TYPE, - PyDoc_STR("Determines if this ST object was created from a suite.")}, + PyDoc_STR("Determines if this ST object was created from a suite.")}, {"tolist", (PyCFunction)(void(*)(void))parser_st2list, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates a list-tree representation of this ST.")}, + PyDoc_STR("Creates a list-tree representation of this ST.")}, {"totuple", (PyCFunction)(void(*)(void))parser_st2tuple, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates a tuple-tree representation of this ST.")}, - {"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS, - PyDoc_STR("Returns size in memory, in bytes.")}, - {NULL, NULL, 0, NULL} -}; - -static -PyTypeObject PyST_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "parser.st", /* tp_name */ - (int) sizeof(PyST_Object), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)parser_free, /* tp_dealloc */ + PyDoc_STR("Creates a tuple-tree representation of this ST.")}, + {"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS, + PyDoc_STR("Returns size in memory, in bytes.")}, + {NULL, NULL, 0, NULL} +}; + +static +PyTypeObject PyST_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "parser.st", /* tp_name */ + (int) sizeof(PyST_Object), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)parser_free, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - - /* Functions to access object as input/output buffer */ - 0, /* tp_as_buffer */ - - Py_TPFLAGS_DEFAULT, /* tp_flags */ - - /* __doc__ */ - "Intermediate representation of a Python parse tree.", - 0, /* tp_traverse */ - 0, /* tp_clear */ - parser_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - parser_methods, /* tp_methods */ -}; /* PyST_Type */ - - -/* PyST_Type isn't subclassable, so just check ob_type */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + + /* Functions to access object as input/output buffer */ + 0, /* tp_as_buffer */ + + Py_TPFLAGS_DEFAULT, /* tp_flags */ + + /* __doc__ */ + "Intermediate representation of a Python parse tree.", + 0, /* tp_traverse */ + 0, /* tp_clear */ + parser_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + parser_methods, /* tp_methods */ +}; /* PyST_Type */ + + +/* PyST_Type isn't subclassable, so just check ob_type */ #define PyST_Object_Check(v) Py_IS_TYPE(v, &PyST_Type) - -static int -parser_compare_nodes(node *left, node *right) -{ - int j; - - if (TYPE(left) < TYPE(right)) - return (-1); - - if (TYPE(right) < TYPE(left)) - return (1); - - if (ISTERMINAL(TYPE(left))) - return (strcmp(STR(left), STR(right))); - - if (NCH(left) < NCH(right)) - return (-1); - - if (NCH(right) < NCH(left)) - return (1); - - for (j = 0; j < NCH(left); ++j) { - int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j)); - - if (v != 0) - return (v); - } - return (0); -} - -/* parser_richcompare(PyObject* left, PyObject* right, int op) - * - * Comparison function used by the Python operators ==, !=, <, >, <=, >= - * This really just wraps a call to parser_compare_nodes() with some easy - * checks and protection code. - * - */ - -static PyObject * -parser_richcompare(PyObject *left, PyObject *right, int op) -{ - int result; - - /* neither argument should be NULL, unless something's gone wrong */ - if (left == NULL || right == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - - /* both arguments should be instances of PyST_Object */ - if (!PyST_Object_Check(left) || !PyST_Object_Check(right)) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (left == right) - /* if arguments are identical, they're equal */ - result = 0; - else - result = parser_compare_nodes(((PyST_Object *)left)->st_node, - ((PyST_Object *)right)->st_node); - - Py_RETURN_RICHCOMPARE(result, 0, op); -} - -/* parser_newstobject(node* st) - * - * Allocates a new Python object representing an ST. This is simply the - * 'wrapper' object that holds a node* and allows it to be passed around in - * Python code. - * - */ -static PyObject* -parser_newstobject(node *st, int type) -{ - PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type); - - if (o != 0) { - o->st_node = st; - o->st_type = type; + +static int +parser_compare_nodes(node *left, node *right) +{ + int j; + + if (TYPE(left) < TYPE(right)) + return (-1); + + if (TYPE(right) < TYPE(left)) + return (1); + + if (ISTERMINAL(TYPE(left))) + return (strcmp(STR(left), STR(right))); + + if (NCH(left) < NCH(right)) + return (-1); + + if (NCH(right) < NCH(left)) + return (1); + + for (j = 0; j < NCH(left); ++j) { + int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j)); + + if (v != 0) + return (v); + } + return (0); +} + +/* parser_richcompare(PyObject* left, PyObject* right, int op) + * + * Comparison function used by the Python operators ==, !=, <, >, <=, >= + * This really just wraps a call to parser_compare_nodes() with some easy + * checks and protection code. + * + */ + +static PyObject * +parser_richcompare(PyObject *left, PyObject *right, int op) +{ + int result; + + /* neither argument should be NULL, unless something's gone wrong */ + if (left == NULL || right == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + /* both arguments should be instances of PyST_Object */ + if (!PyST_Object_Check(left) || !PyST_Object_Check(right)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (left == right) + /* if arguments are identical, they're equal */ + result = 0; + else + result = parser_compare_nodes(((PyST_Object *)left)->st_node, + ((PyST_Object *)right)->st_node); + + Py_RETURN_RICHCOMPARE(result, 0, op); +} + +/* parser_newstobject(node* st) + * + * Allocates a new Python object representing an ST. This is simply the + * 'wrapper' object that holds a node* and allows it to be passed around in + * Python code. + * + */ +static PyObject* +parser_newstobject(node *st, int type) +{ + PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type); + + if (o != 0) { + o->st_node = st; + o->st_type = type; o->st_flags = _PyCompilerFlags_INIT; - } - else { - PyNode_Free(st); - } - return ((PyObject*)o); -} - - -/* void parser_free(PyST_Object* st) - * - * This is called by a del statement that reduces the reference count to 0. - * - */ -static void -parser_free(PyST_Object *st) -{ - PyNode_Free(st->st_node); - PyObject_Del(st); -} - -static PyObject * -parser_sizeof(PyST_Object *st, void *unused) -{ - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node); - return PyLong_FromSsize_t(res); -} - - -/* parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw) - * - * This provides conversion from a node* to a tuple object that can be - * returned to the Python-level caller. The ST object is not modified. - * - */ -static PyObject* -parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) -{ - int line_info = 0; - int col_info = 0; - PyObject *res = 0; - int ok; - - static char *keywords[] = {"st", "line_info", "col_info", NULL}; - - if (self == NULL || PyModule_Check(self)) { - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2tuple", keywords, - &PyST_Type, &self, &line_info, - &col_info); - } - else - ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:totuple", &keywords[1], - &line_info, &col_info); - if (ok != 0) { - /* - * Convert ST into a tuple representation. Use Guido's function, - * since it's known to work already. - */ - res = node2tuple(((PyST_Object*)self)->st_node, - PyTuple_New, PyTuple_SetItem, line_info, col_info); - } - return (res); -} - - -/* parser_st2list(PyObject* self, PyObject* args, PyObject* kw) - * - * This provides conversion from a node* to a list object that can be - * returned to the Python-level caller. The ST object is not modified. - * - */ -static PyObject* -parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) -{ - int line_info = 0; - int col_info = 0; - PyObject *res = 0; - int ok; - - static char *keywords[] = {"st", "line_info", "col_info", NULL}; - - if (self == NULL || PyModule_Check(self)) - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2list", keywords, - &PyST_Type, &self, &line_info, - &col_info); - else - ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:tolist", &keywords[1], - &line_info, &col_info); - if (ok) { - /* - * Convert ST into a tuple representation. Use Guido's function, - * since it's known to work already. - */ - res = node2tuple(self->st_node, - PyList_New, PyList_SetItem, line_info, col_info); - } - return (res); -} - - -/* parser_compilest(PyObject* self, PyObject* args) - * - * This function creates code objects from the parse tree represented by - * the passed-in data object. An optional file name is passed in as well. - * - */ -static PyObject* -parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) -{ - PyObject* res = NULL; - PyArena* arena = NULL; - mod_ty mod; - PyObject* filename = NULL; - int ok; - - static char *keywords[] = {"st", "filename", NULL}; - - if (self == NULL || PyModule_Check(self)) - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O&:compilest", keywords, - &PyST_Type, &self, - PyUnicode_FSDecoder, &filename); - else - ok = PyArg_ParseTupleAndKeywords(args, kw, "|O&:compile", &keywords[1], - PyUnicode_FSDecoder, &filename); - if (!ok) - goto error; - - if (filename == NULL) { - filename = PyUnicode_FromString("<syntax-tree>"); - if (filename == NULL) - goto error; - } - - arena = PyArena_New(); - if (!arena) - goto error; - - mod = PyAST_FromNodeObject(self->st_node, &self->st_flags, - filename, arena); - if (!mod) - goto error; - - res = (PyObject *)PyAST_CompileObject(mod, filename, - &self->st_flags, -1, arena); -error: - Py_XDECREF(filename); - if (arena != NULL) - PyArena_Free(arena); - return res; -} - - -/* PyObject* parser_isexpr(PyObject* self, PyObject* args) - * PyObject* parser_issuite(PyObject* self, PyObject* args) - * - * Checks the passed-in ST object to determine if it is an expression or - * a statement suite, respectively. The return is a Python truth value. - * - */ -static PyObject* -parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw) -{ - PyObject* res = 0; - int ok; - - static char *keywords[] = {"st", NULL}; - - if (self == NULL || PyModule_Check(self)) - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords, - &PyST_Type, &self); - else - ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]); - - if (ok) { - /* Check to see if the ST represents an expression or not. */ - res = (self->st_type == PyST_EXPR) ? Py_True : Py_False; - Py_INCREF(res); - } - return (res); -} - - -static PyObject* -parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) -{ - PyObject* res = 0; - int ok; - - static char *keywords[] = {"st", NULL}; - - if (self == NULL || PyModule_Check(self)) - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords, - &PyST_Type, &self); - else - ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]); - - if (ok) { - /* Check to see if the ST represents an expression or not. */ - res = (self->st_type == PyST_EXPR) ? Py_False : Py_True; - Py_INCREF(res); - } - return (res); -} - - -/* err_string(const char* message) - * - * Sets the error string for an exception of type ParserError. - * - */ -static void -err_string(const char *message) -{ - PyErr_SetString(parser_error, message); -} - - -/* PyObject* parser_do_parse(PyObject* args, int type) - * - * Internal function to actually execute the parse and return the result if - * successful or set an exception if not. - * - */ -static PyObject* -parser_do_parse(PyObject *args, PyObject *kw, const char *argspec, int type) -{ - char* string = 0; - PyObject* res = 0; - int flags = 0; - perrdetail err; - - static char *keywords[] = {"source", NULL}; - - if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) { - node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL, - &_PyParser_Grammar, - (type == PyST_EXPR) - ? eval_input : file_input, - &err, &flags); - - if (n) { - res = parser_newstobject(n, type); + } + else { + PyNode_Free(st); + } + return ((PyObject*)o); +} + + +/* void parser_free(PyST_Object* st) + * + * This is called by a del statement that reduces the reference count to 0. + * + */ +static void +parser_free(PyST_Object *st) +{ + PyNode_Free(st->st_node); + PyObject_Del(st); +} + +static PyObject * +parser_sizeof(PyST_Object *st, void *unused) +{ + Py_ssize_t res; + + res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node); + return PyLong_FromSsize_t(res); +} + + +/* parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw) + * + * This provides conversion from a node* to a tuple object that can be + * returned to the Python-level caller. The ST object is not modified. + * + */ +static PyObject* +parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) +{ + int line_info = 0; + int col_info = 0; + PyObject *res = 0; + int ok; + + static char *keywords[] = {"st", "line_info", "col_info", NULL}; + + if (self == NULL || PyModule_Check(self)) { + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2tuple", keywords, + &PyST_Type, &self, &line_info, + &col_info); + } + else + ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:totuple", &keywords[1], + &line_info, &col_info); + if (ok != 0) { + /* + * Convert ST into a tuple representation. Use Guido's function, + * since it's known to work already. + */ + res = node2tuple(((PyST_Object*)self)->st_node, + PyTuple_New, PyTuple_SetItem, line_info, col_info); + } + return (res); +} + + +/* parser_st2list(PyObject* self, PyObject* args, PyObject* kw) + * + * This provides conversion from a node* to a list object that can be + * returned to the Python-level caller. The ST object is not modified. + * + */ +static PyObject* +parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) +{ + int line_info = 0; + int col_info = 0; + PyObject *res = 0; + int ok; + + static char *keywords[] = {"st", "line_info", "col_info", NULL}; + + if (self == NULL || PyModule_Check(self)) + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|pp:st2list", keywords, + &PyST_Type, &self, &line_info, + &col_info); + else + ok = PyArg_ParseTupleAndKeywords(args, kw, "|pp:tolist", &keywords[1], + &line_info, &col_info); + if (ok) { + /* + * Convert ST into a tuple representation. Use Guido's function, + * since it's known to work already. + */ + res = node2tuple(self->st_node, + PyList_New, PyList_SetItem, line_info, col_info); + } + return (res); +} + + +/* parser_compilest(PyObject* self, PyObject* args) + * + * This function creates code objects from the parse tree represented by + * the passed-in data object. An optional file name is passed in as well. + * + */ +static PyObject* +parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) +{ + PyObject* res = NULL; + PyArena* arena = NULL; + mod_ty mod; + PyObject* filename = NULL; + int ok; + + static char *keywords[] = {"st", "filename", NULL}; + + if (self == NULL || PyModule_Check(self)) + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O&:compilest", keywords, + &PyST_Type, &self, + PyUnicode_FSDecoder, &filename); + else + ok = PyArg_ParseTupleAndKeywords(args, kw, "|O&:compile", &keywords[1], + PyUnicode_FSDecoder, &filename); + if (!ok) + goto error; + + if (filename == NULL) { + filename = PyUnicode_FromString("<syntax-tree>"); + if (filename == NULL) + goto error; + } + + arena = PyArena_New(); + if (!arena) + goto error; + + mod = PyAST_FromNodeObject(self->st_node, &self->st_flags, + filename, arena); + if (!mod) + goto error; + + res = (PyObject *)PyAST_CompileObject(mod, filename, + &self->st_flags, -1, arena); +error: + Py_XDECREF(filename); + if (arena != NULL) + PyArena_Free(arena); + return res; +} + + +/* PyObject* parser_isexpr(PyObject* self, PyObject* args) + * PyObject* parser_issuite(PyObject* self, PyObject* args) + * + * Checks the passed-in ST object to determine if it is an expression or + * a statement suite, respectively. The return is a Python truth value. + * + */ +static PyObject* +parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw) +{ + PyObject* res = 0; + int ok; + + static char *keywords[] = {"st", NULL}; + + if (self == NULL || PyModule_Check(self)) + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords, + &PyST_Type, &self); + else + ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]); + + if (ok) { + /* Check to see if the ST represents an expression or not. */ + res = (self->st_type == PyST_EXPR) ? Py_True : Py_False; + Py_INCREF(res); + } + return (res); +} + + +static PyObject* +parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) +{ + PyObject* res = 0; + int ok; + + static char *keywords[] = {"st", NULL}; + + if (self == NULL || PyModule_Check(self)) + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords, + &PyST_Type, &self); + else + ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]); + + if (ok) { + /* Check to see if the ST represents an expression or not. */ + res = (self->st_type == PyST_EXPR) ? Py_False : Py_True; + Py_INCREF(res); + } + return (res); +} + + +/* err_string(const char* message) + * + * Sets the error string for an exception of type ParserError. + * + */ +static void +err_string(const char *message) +{ + PyErr_SetString(parser_error, message); +} + + +/* PyObject* parser_do_parse(PyObject* args, int type) + * + * Internal function to actually execute the parse and return the result if + * successful or set an exception if not. + * + */ +static PyObject* +parser_do_parse(PyObject *args, PyObject *kw, const char *argspec, int type) +{ + char* string = 0; + PyObject* res = 0; + int flags = 0; + perrdetail err; + + static char *keywords[] = {"source", NULL}; + + if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) { + node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL, + &_PyParser_Grammar, + (type == PyST_EXPR) + ? eval_input : file_input, + &err, &flags); + + if (n) { + res = parser_newstobject(n, type); if (res) { - ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; + ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; ((PyST_Object *)res)->st_flags.cf_feature_version = PY_MINOR_VERSION; } - } - else { - PyParser_SetError(&err); - } - PyParser_ClearError(&err); - } - return (res); -} - - -/* PyObject* parser_expr(PyObject* self, PyObject* args) - * PyObject* parser_suite(PyObject* self, PyObject* args) - * - * External interfaces to the parser itself. Which is called determines if - * the parser attempts to recognize an expression ('eval' form) or statement - * suite ('exec' form). The real work is done by parser_do_parse() above. - * - */ -static PyObject* -parser_expr(PyST_Object *self, PyObject *args, PyObject *kw) -{ - NOTE(ARGUNUSED(self)) - return (parser_do_parse(args, kw, "s:expr", PyST_EXPR)); -} - - -static PyObject* -parser_suite(PyST_Object *self, PyObject *args, PyObject *kw) -{ - NOTE(ARGUNUSED(self)) - return (parser_do_parse(args, kw, "s:suite", PyST_SUITE)); -} - - - -/* This is the messy part of the code. Conversion from a tuple to an ST - * object requires that the input tuple be valid without having to rely on - * catching an exception from the compiler. This is done to allow the - * compiler itself to remain fast, since most of its input will come from - * the parser directly, and therefore be known to be syntactically correct. - * This validation is done to ensure that we don't core dump the compile - * phase, returning an exception instead. - * - * Two aspects can be broken out in this code: creating a node tree from - * the tuple passed in, and verifying that it is indeed valid. It may be - * advantageous to expand the number of ST types to include funcdefs and - * lambdadefs to take advantage of the optimizer, recognizing those STs - * here. They are not necessary, and not quite as useful in a raw form. - * For now, let's get expressions and suites working reliably. - */ - - -static node* build_node_tree(PyObject *tuple); - -static int -validate_node(node *tree) -{ - int type = TYPE(tree); - int nch = NCH(tree); - state *dfa_state; - int pos, arc; - - assert(ISNONTERMINAL(type)); - type -= NT_OFFSET; - if (type >= _PyParser_Grammar.g_ndfas) { - PyErr_Format(parser_error, "Unrecognized node type %d.", TYPE(tree)); - return 0; - } + } + else { + PyParser_SetError(&err); + } + PyParser_ClearError(&err); + } + return (res); +} + + +/* PyObject* parser_expr(PyObject* self, PyObject* args) + * PyObject* parser_suite(PyObject* self, PyObject* args) + * + * External interfaces to the parser itself. Which is called determines if + * the parser attempts to recognize an expression ('eval' form) or statement + * suite ('exec' form). The real work is done by parser_do_parse() above. + * + */ +static PyObject* +parser_expr(PyST_Object *self, PyObject *args, PyObject *kw) +{ + NOTE(ARGUNUSED(self)) + return (parser_do_parse(args, kw, "s:expr", PyST_EXPR)); +} + + +static PyObject* +parser_suite(PyST_Object *self, PyObject *args, PyObject *kw) +{ + NOTE(ARGUNUSED(self)) + return (parser_do_parse(args, kw, "s:suite", PyST_SUITE)); +} + + + +/* This is the messy part of the code. Conversion from a tuple to an ST + * object requires that the input tuple be valid without having to rely on + * catching an exception from the compiler. This is done to allow the + * compiler itself to remain fast, since most of its input will come from + * the parser directly, and therefore be known to be syntactically correct. + * This validation is done to ensure that we don't core dump the compile + * phase, returning an exception instead. + * + * Two aspects can be broken out in this code: creating a node tree from + * the tuple passed in, and verifying that it is indeed valid. It may be + * advantageous to expand the number of ST types to include funcdefs and + * lambdadefs to take advantage of the optimizer, recognizing those STs + * here. They are not necessary, and not quite as useful in a raw form. + * For now, let's get expressions and suites working reliably. + */ + + +static node* build_node_tree(PyObject *tuple); + +static int +validate_node(node *tree) +{ + int type = TYPE(tree); + int nch = NCH(tree); + state *dfa_state; + int pos, arc; + + assert(ISNONTERMINAL(type)); + type -= NT_OFFSET; + if (type >= _PyParser_Grammar.g_ndfas) { + PyErr_Format(parser_error, "Unrecognized node type %d.", TYPE(tree)); + return 0; + } const dfa *nt_dfa = &_PyParser_Grammar.g_dfa[type]; - REQ(tree, nt_dfa->d_type); - - /* Run the DFA for this nonterminal. */ + REQ(tree, nt_dfa->d_type); + + /* Run the DFA for this nonterminal. */ dfa_state = nt_dfa->d_state; - for (pos = 0; pos < nch; ++pos) { - node *ch = CHILD(tree, pos); - int ch_type = TYPE(ch); + for (pos = 0; pos < nch; ++pos) { + node *ch = CHILD(tree, pos); + int ch_type = TYPE(ch); if ((ch_type >= NT_OFFSET + _PyParser_Grammar.g_ndfas) || (ISTERMINAL(ch_type) && (ch_type >= N_TOKENS)) || (ch_type < 0) @@ -673,30 +673,30 @@ validate_node(node *tree) support type comments here. */ ch_type = func_body_suite; } - for (arc = 0; arc < dfa_state->s_narcs; ++arc) { - short a_label = dfa_state->s_arc[arc].a_lbl; - assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); + for (arc = 0; arc < dfa_state->s_narcs; ++arc) { + short a_label = dfa_state->s_arc[arc].a_lbl; + assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); const char *label_str = _PyParser_Grammar.g_ll.ll_label[a_label].lb_str; if ((_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) && ((ch->n_str == NULL) || (label_str == NULL) || (strcmp(ch->n_str, label_str) == 0)) ) { - /* The child is acceptable; if non-terminal, validate it recursively. */ - if (ISNONTERMINAL(ch_type) && !validate_node(ch)) - return 0; - - /* Update the state, and move on to the next child. */ - dfa_state = &nt_dfa->d_state[dfa_state->s_arc[arc].a_arrow]; - goto arc_found; - } - } - /* What would this state have accepted? */ - { - short a_label = dfa_state->s_arc->a_lbl; - if (!a_label) /* Wouldn't accept any more children */ - goto illegal_num_children; - + /* The child is acceptable; if non-terminal, validate it recursively. */ + if (ISNONTERMINAL(ch_type) && !validate_node(ch)) + return 0; + + /* Update the state, and move on to the next child. */ + dfa_state = &nt_dfa->d_state[dfa_state->s_arc[arc].a_arrow]; + goto arc_found; + } + } + /* What would this state have accepted? */ + { + short a_label = dfa_state->s_arc->a_lbl; + if (!a_label) /* Wouldn't accept any more children */ + goto illegal_num_children; + int next_type = _PyParser_Grammar.g_ll.ll_label[a_label].lb_type; const char *expected_str = _PyParser_Grammar.g_ll.ll_label[a_label].lb_str; @@ -711,512 +711,512 @@ validate_node(node *tree) expected_str); } else { - PyErr_Format(parser_error, "Illegal terminal: expected %s.", - _PyParser_TokenNames[next_type]); - } - return 0; - } - -arc_found: - continue; - } - /* Are we in a final state? If so, return 1 for successful validation. */ - for (arc = 0; arc < dfa_state->s_narcs; ++arc) { - if (!dfa_state->s_arc[arc].a_lbl) { - return 1; - } - } - -illegal_num_children: - PyErr_Format(parser_error, - "Illegal number of children for %s node.", nt_dfa->d_name); - return 0; -} - -/* PyObject* parser_tuple2st(PyObject* self, PyObject* args) - * - * This is the public function, called from the Python code. It receives a - * single tuple object from the caller, and creates an ST object if the - * tuple can be validated. It does this by checking the first code of the - * tuple, and, if acceptable, builds the internal representation. If this - * step succeeds, the internal representation is validated as fully as - * possible with the recursive validate_node() routine defined above. - * - * This function must be changed if support is to be added for PyST_FRAGMENT - * ST objects. - * - */ -static PyObject* -parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) -{ - NOTE(ARGUNUSED(self)) - PyObject *st = 0; - PyObject *tuple; - node *tree; - - static char *keywords[] = {"sequence", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords, - &tuple)) - return (0); - if (!PySequence_Check(tuple)) { - PyErr_SetString(PyExc_ValueError, - "sequence2st() requires a single sequence argument"); - return (0); - } - /* - * Convert the tree to the internal form before checking it. - */ - tree = build_node_tree(tuple); - if (tree != 0) { - node *validation_root = NULL; - int tree_type = 0; - switch (TYPE(tree)) { - case eval_input: - /* Might be an eval form. */ - tree_type = PyST_EXPR; - validation_root = tree; - break; - case encoding_decl: - /* This looks like an encoding_decl so far. */ - if (NCH(tree) == 1) { - tree_type = PyST_SUITE; - validation_root = CHILD(tree, 0); - } - else { - err_string("Error Parsing encoding_decl"); + PyErr_Format(parser_error, "Illegal terminal: expected %s.", + _PyParser_TokenNames[next_type]); } - break; - case file_input: - /* This looks like an exec form so far. */ - tree_type = PyST_SUITE; - validation_root = tree; - break; - default: - /* This is a fragment, at best. */ - err_string("parse tree does not use a valid start symbol"); - } - - if (validation_root != NULL && validate_node(validation_root)) - st = parser_newstobject(tree, tree_type); - else - PyNode_Free(tree); - } - /* Make sure we raise an exception on all errors. We should never - * get this, but we'd do well to be sure something is done. - */ - if (st == NULL && !PyErr_Occurred()) - err_string("unspecified ST error occurred"); - - return st; -} - - -/* node* build_node_children() - * - * Iterate across the children of the current non-terminal node and build - * their structures. If successful, return the root of this portion of - * the tree, otherwise, 0. Any required exception will be specified already, - * and no memory will have been deallocated. - * - */ -static node* -build_node_children(PyObject *tuple, node *root, int *line_num) -{ - Py_ssize_t len = PyObject_Size(tuple); - Py_ssize_t i; - int err; - - if (len < 0) { - return NULL; - } - for (i = 1; i < len; ++i) { - /* elem must always be a sequence, however simple */ - PyObject* elem = PySequence_GetItem(tuple, i); - int ok = elem != NULL; - int type = 0; - char *strn = 0; - - if (ok) - ok = PySequence_Check(elem); - if (ok) { - PyObject *temp = PySequence_GetItem(elem, 0); - if (temp == NULL) - ok = 0; - else { - ok = PyLong_Check(temp); - if (ok) { - type = _PyLong_AsInt(temp); - if (type == -1 && PyErr_Occurred()) { - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - } - Py_DECREF(temp); - } - } - if (!ok) { - PyObject *err = Py_BuildValue("Os", elem, - "Illegal node construct."); - PyErr_SetObject(parser_error, err); - Py_XDECREF(err); - Py_XDECREF(elem); - return NULL; - } - if (ISTERMINAL(type)) { - Py_ssize_t len = PyObject_Size(elem); - PyObject *temp; - const char *temp_str; - - if ((len != 2) && (len != 3)) { - err_string("terminal nodes must have 2 or 3 entries"); - Py_DECREF(elem); - return NULL; - } - temp = PySequence_GetItem(elem, 1); - if (temp == NULL) { - Py_DECREF(elem); - return NULL; - } - if (!PyUnicode_Check(temp)) { - PyErr_Format(parser_error, - "second item in terminal node must be a string," - " found %s", - Py_TYPE(temp)->tp_name); - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - if (len == 3) { - PyObject *o = PySequence_GetItem(elem, 2); - if (o == NULL) { - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - if (PyLong_Check(o)) { - int num = _PyLong_AsInt(o); - if (num == -1 && PyErr_Occurred()) { - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - *line_num = num; - } - else { - PyErr_Format(parser_error, - "third item in terminal node must be an" - " integer, found %s", - Py_TYPE(temp)->tp_name); - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - Py_DECREF(o); - } - temp_str = PyUnicode_AsUTF8AndSize(temp, &len); - if (temp_str == NULL) { - Py_DECREF(temp); - Py_DECREF(elem); - return NULL; - } - strn = (char *)PyObject_MALLOC(len + 1); - if (strn == NULL) { - Py_DECREF(temp); - Py_DECREF(elem); - PyErr_NoMemory(); - return NULL; - } - (void) memcpy(strn, temp_str, len + 1); - Py_DECREF(temp); - } - else if (!ISNONTERMINAL(type)) { - /* - * It has to be one or the other; this is an error. - * Raise an exception. - */ - PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); - PyErr_SetObject(parser_error, err); - Py_XDECREF(err); - Py_DECREF(elem); - return NULL; - } + return 0; + } + +arc_found: + continue; + } + /* Are we in a final state? If so, return 1 for successful validation. */ + for (arc = 0; arc < dfa_state->s_narcs; ++arc) { + if (!dfa_state->s_arc[arc].a_lbl) { + return 1; + } + } + +illegal_num_children: + PyErr_Format(parser_error, + "Illegal number of children for %s node.", nt_dfa->d_name); + return 0; +} + +/* PyObject* parser_tuple2st(PyObject* self, PyObject* args) + * + * This is the public function, called from the Python code. It receives a + * single tuple object from the caller, and creates an ST object if the + * tuple can be validated. It does this by checking the first code of the + * tuple, and, if acceptable, builds the internal representation. If this + * step succeeds, the internal representation is validated as fully as + * possible with the recursive validate_node() routine defined above. + * + * This function must be changed if support is to be added for PyST_FRAGMENT + * ST objects. + * + */ +static PyObject* +parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) +{ + NOTE(ARGUNUSED(self)) + PyObject *st = 0; + PyObject *tuple; + node *tree; + + static char *keywords[] = {"sequence", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords, + &tuple)) + return (0); + if (!PySequence_Check(tuple)) { + PyErr_SetString(PyExc_ValueError, + "sequence2st() requires a single sequence argument"); + return (0); + } + /* + * Convert the tree to the internal form before checking it. + */ + tree = build_node_tree(tuple); + if (tree != 0) { + node *validation_root = NULL; + int tree_type = 0; + switch (TYPE(tree)) { + case eval_input: + /* Might be an eval form. */ + tree_type = PyST_EXPR; + validation_root = tree; + break; + case encoding_decl: + /* This looks like an encoding_decl so far. */ + if (NCH(tree) == 1) { + tree_type = PyST_SUITE; + validation_root = CHILD(tree, 0); + } + else { + err_string("Error Parsing encoding_decl"); + } + break; + case file_input: + /* This looks like an exec form so far. */ + tree_type = PyST_SUITE; + validation_root = tree; + break; + default: + /* This is a fragment, at best. */ + err_string("parse tree does not use a valid start symbol"); + } + + if (validation_root != NULL && validate_node(validation_root)) + st = parser_newstobject(tree, tree_type); + else + PyNode_Free(tree); + } + /* Make sure we raise an exception on all errors. We should never + * get this, but we'd do well to be sure something is done. + */ + if (st == NULL && !PyErr_Occurred()) + err_string("unspecified ST error occurred"); + + return st; +} + + +/* node* build_node_children() + * + * Iterate across the children of the current non-terminal node and build + * their structures. If successful, return the root of this portion of + * the tree, otherwise, 0. Any required exception will be specified already, + * and no memory will have been deallocated. + * + */ +static node* +build_node_children(PyObject *tuple, node *root, int *line_num) +{ + Py_ssize_t len = PyObject_Size(tuple); + Py_ssize_t i; + int err; + + if (len < 0) { + return NULL; + } + for (i = 1; i < len; ++i) { + /* elem must always be a sequence, however simple */ + PyObject* elem = PySequence_GetItem(tuple, i); + int ok = elem != NULL; + int type = 0; + char *strn = 0; + + if (ok) + ok = PySequence_Check(elem); + if (ok) { + PyObject *temp = PySequence_GetItem(elem, 0); + if (temp == NULL) + ok = 0; + else { + ok = PyLong_Check(temp); + if (ok) { + type = _PyLong_AsInt(temp); + if (type == -1 && PyErr_Occurred()) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + } + Py_DECREF(temp); + } + } + if (!ok) { + PyObject *err = Py_BuildValue("Os", elem, + "Illegal node construct."); + PyErr_SetObject(parser_error, err); + Py_XDECREF(err); + Py_XDECREF(elem); + return NULL; + } + if (ISTERMINAL(type)) { + Py_ssize_t len = PyObject_Size(elem); + PyObject *temp; + const char *temp_str; + + if ((len != 2) && (len != 3)) { + err_string("terminal nodes must have 2 or 3 entries"); + Py_DECREF(elem); + return NULL; + } + temp = PySequence_GetItem(elem, 1); + if (temp == NULL) { + Py_DECREF(elem); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format(parser_error, + "second item in terminal node must be a string," + " found %s", + Py_TYPE(temp)->tp_name); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (len == 3) { + PyObject *o = PySequence_GetItem(elem, 2); + if (o == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (PyLong_Check(o)) { + int num = _PyLong_AsInt(o); + if (num == -1 && PyErr_Occurred()) { + Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + *line_num = num; + } + else { + PyErr_Format(parser_error, + "third item in terminal node must be an" + " integer, found %s", + Py_TYPE(temp)->tp_name); + Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + Py_DECREF(o); + } + temp_str = PyUnicode_AsUTF8AndSize(temp, &len); + if (temp_str == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + strn = (char *)PyObject_MALLOC(len + 1); + if (strn == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + PyErr_NoMemory(); + return NULL; + } + (void) memcpy(strn, temp_str, len + 1); + Py_DECREF(temp); + } + else if (!ISNONTERMINAL(type)) { + /* + * It has to be one or the other; this is an error. + * Raise an exception. + */ + PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); + PyErr_SetObject(parser_error, err); + Py_XDECREF(err); + Py_DECREF(elem); + return NULL; + } err = PyNode_AddChild(root, type, strn, *line_num, 0, *line_num, 0); - if (err == E_NOMEM) { - Py_DECREF(elem); - PyObject_FREE(strn); - PyErr_NoMemory(); - return NULL; - } - if (err == E_OVERFLOW) { - Py_DECREF(elem); - PyObject_FREE(strn); - PyErr_SetString(PyExc_ValueError, - "unsupported number of child nodes"); - return NULL; - } - - if (ISNONTERMINAL(type)) { - node* new_child = CHILD(root, i - 1); - - if (new_child != build_node_children(elem, new_child, line_num)) { - Py_DECREF(elem); - return NULL; - } - } - else if (type == NEWLINE) { /* It's true: we increment the */ - ++(*line_num); /* line number *after* the newline! */ - } - Py_DECREF(elem); - } - return root; -} - - -static node* -build_node_tree(PyObject *tuple) -{ - node* res = 0; - PyObject *temp = PySequence_GetItem(tuple, 0); - long num = -1; - - if (temp != NULL) - num = PyLong_AsLong(temp); - Py_XDECREF(temp); - if (ISTERMINAL(num)) { - /* - * The tuple is simple, but it doesn't start with a start symbol. - * Raise an exception now and be done with it. - */ - tuple = Py_BuildValue("Os", tuple, - "Illegal syntax-tree; cannot start with terminal symbol."); - PyErr_SetObject(parser_error, tuple); - Py_XDECREF(tuple); - } - else if (ISNONTERMINAL(num)) { - /* - * Not efficient, but that can be handled later. - */ - int line_num = 0; - PyObject *encoding = NULL; - - if (num == encoding_decl) { - encoding = PySequence_GetItem(tuple, 2); - if (encoding == NULL) { - PyErr_SetString(parser_error, "missed encoding"); - return NULL; - } - if (!PyUnicode_Check(encoding)) { - PyErr_Format(parser_error, - "encoding must be a string, found %.200s", - Py_TYPE(encoding)->tp_name); - Py_DECREF(encoding); - return NULL; - } - /* tuple isn't borrowed anymore here, need to DECREF */ - tuple = PySequence_GetSlice(tuple, 0, 2); - if (tuple == NULL) { - Py_DECREF(encoding); - return NULL; - } - } - res = PyNode_New(num); - if (res != NULL) { - if (res != build_node_children(tuple, res, &line_num)) { - PyNode_Free(res); - res = NULL; - } - if (res && encoding) { - Py_ssize_t len; - const char *temp; - temp = PyUnicode_AsUTF8AndSize(encoding, &len); - if (temp == NULL) { - PyNode_Free(res); - Py_DECREF(encoding); - Py_DECREF(tuple); - return NULL; - } - res->n_str = (char *)PyObject_MALLOC(len + 1); - if (res->n_str == NULL) { - PyNode_Free(res); - Py_DECREF(encoding); - Py_DECREF(tuple); - PyErr_NoMemory(); - return NULL; - } - (void) memcpy(res->n_str, temp, len + 1); - } - } - if (encoding != NULL) { - Py_DECREF(encoding); - Py_DECREF(tuple); - } - } - else { - /* The tuple is illegal -- if the number is neither TERMINAL nor - * NONTERMINAL, we can't use it. Not sure the implementation - * allows this condition, but the API doesn't preclude it. - */ - PyObject *err = Py_BuildValue("Os", tuple, - "Illegal component tuple."); - PyErr_SetObject(parser_error, err); - Py_XDECREF(err); - } - - return (res); -} - - -static PyObject* -pickle_constructor = NULL; - - -static PyObject* -parser__pickler(PyObject *self, PyObject *args) -{ - NOTE(ARGUNUSED(self)) - PyObject *result = NULL; - PyObject *st = NULL; - - if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) { - PyObject *newargs; - PyObject *tuple; - + if (err == E_NOMEM) { + Py_DECREF(elem); + PyObject_FREE(strn); + PyErr_NoMemory(); + return NULL; + } + if (err == E_OVERFLOW) { + Py_DECREF(elem); + PyObject_FREE(strn); + PyErr_SetString(PyExc_ValueError, + "unsupported number of child nodes"); + return NULL; + } + + if (ISNONTERMINAL(type)) { + node* new_child = CHILD(root, i - 1); + + if (new_child != build_node_children(elem, new_child, line_num)) { + Py_DECREF(elem); + return NULL; + } + } + else if (type == NEWLINE) { /* It's true: we increment the */ + ++(*line_num); /* line number *after* the newline! */ + } + Py_DECREF(elem); + } + return root; +} + + +static node* +build_node_tree(PyObject *tuple) +{ + node* res = 0; + PyObject *temp = PySequence_GetItem(tuple, 0); + long num = -1; + + if (temp != NULL) + num = PyLong_AsLong(temp); + Py_XDECREF(temp); + if (ISTERMINAL(num)) { + /* + * The tuple is simple, but it doesn't start with a start symbol. + * Raise an exception now and be done with it. + */ + tuple = Py_BuildValue("Os", tuple, + "Illegal syntax-tree; cannot start with terminal symbol."); + PyErr_SetObject(parser_error, tuple); + Py_XDECREF(tuple); + } + else if (ISNONTERMINAL(num)) { + /* + * Not efficient, but that can be handled later. + */ + int line_num = 0; + PyObject *encoding = NULL; + + if (num == encoding_decl) { + encoding = PySequence_GetItem(tuple, 2); + if (encoding == NULL) { + PyErr_SetString(parser_error, "missed encoding"); + return NULL; + } + if (!PyUnicode_Check(encoding)) { + PyErr_Format(parser_error, + "encoding must be a string, found %.200s", + Py_TYPE(encoding)->tp_name); + Py_DECREF(encoding); + return NULL; + } + /* tuple isn't borrowed anymore here, need to DECREF */ + tuple = PySequence_GetSlice(tuple, 0, 2); + if (tuple == NULL) { + Py_DECREF(encoding); + return NULL; + } + } + res = PyNode_New(num); + if (res != NULL) { + if (res != build_node_children(tuple, res, &line_num)) { + PyNode_Free(res); + res = NULL; + } + if (res && encoding) { + Py_ssize_t len; + const char *temp; + temp = PyUnicode_AsUTF8AndSize(encoding, &len); + if (temp == NULL) { + PyNode_Free(res); + Py_DECREF(encoding); + Py_DECREF(tuple); + return NULL; + } + res->n_str = (char *)PyObject_MALLOC(len + 1); + if (res->n_str == NULL) { + PyNode_Free(res); + Py_DECREF(encoding); + Py_DECREF(tuple); + PyErr_NoMemory(); + return NULL; + } + (void) memcpy(res->n_str, temp, len + 1); + } + } + if (encoding != NULL) { + Py_DECREF(encoding); + Py_DECREF(tuple); + } + } + else { + /* The tuple is illegal -- if the number is neither TERMINAL nor + * NONTERMINAL, we can't use it. Not sure the implementation + * allows this condition, but the API doesn't preclude it. + */ + PyObject *err = Py_BuildValue("Os", tuple, + "Illegal component tuple."); + PyErr_SetObject(parser_error, err); + Py_XDECREF(err); + } + + return (res); +} + + +static PyObject* +pickle_constructor = NULL; + + +static PyObject* +parser__pickler(PyObject *self, PyObject *args) +{ + NOTE(ARGUNUSED(self)) + PyObject *result = NULL; + PyObject *st = NULL; + + if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) { + PyObject *newargs; + PyObject *tuple; + if ((newargs = PyTuple_Pack(2, st, Py_True)) == NULL) return NULL; tuple = parser_st2tuple((PyST_Object*)NULL, newargs, NULL); - if (tuple != NULL) { - result = Py_BuildValue("O(O)", pickle_constructor, tuple); - Py_DECREF(tuple); - } - Py_DECREF(newargs); - } - - return (result); -} - - -/* Functions exported by this module. Most of this should probably - * be converted into an ST object with methods, but that is better - * done directly in Python, allowing subclasses to be created directly. - * We'd really have to write a wrapper around it all anyway to allow - * inheritance. - */ -static PyMethodDef parser_functions[] = { + if (tuple != NULL) { + result = Py_BuildValue("O(O)", pickle_constructor, tuple); + Py_DECREF(tuple); + } + Py_DECREF(newargs); + } + + return (result); +} + + +/* Functions exported by this module. Most of this should probably + * be converted into an ST object with methods, but that is better + * done directly in Python, allowing subclasses to be created directly. + * We'd really have to write a wrapper around it all anyway to allow + * inheritance. + */ +static PyMethodDef parser_functions[] = { {"compilest", (PyCFunction)(void(*)(void))parser_compilest, PUBLIC_METHOD_TYPE, - PyDoc_STR("Compiles an ST object into a code object.")}, + PyDoc_STR("Compiles an ST object into a code object.")}, {"expr", (PyCFunction)(void(*)(void))parser_expr, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates an ST object from an expression.")}, + PyDoc_STR("Creates an ST object from an expression.")}, {"isexpr", (PyCFunction)(void(*)(void))parser_isexpr, PUBLIC_METHOD_TYPE, - PyDoc_STR("Determines if an ST object was created from an expression.")}, + PyDoc_STR("Determines if an ST object was created from an expression.")}, {"issuite", (PyCFunction)(void(*)(void))parser_issuite, PUBLIC_METHOD_TYPE, - PyDoc_STR("Determines if an ST object was created from a suite.")}, + PyDoc_STR("Determines if an ST object was created from a suite.")}, {"suite", (PyCFunction)(void(*)(void))parser_suite, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates an ST object from a suite.")}, + PyDoc_STR("Creates an ST object from a suite.")}, {"sequence2st", (PyCFunction)(void(*)(void))parser_tuple2st, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates an ST object from a tree representation.")}, + PyDoc_STR("Creates an ST object from a tree representation.")}, {"st2tuple", (PyCFunction)(void(*)(void))parser_st2tuple, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates a tuple-tree representation of an ST.")}, + PyDoc_STR("Creates a tuple-tree representation of an ST.")}, {"st2list", (PyCFunction)(void(*)(void))parser_st2list, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates a list-tree representation of an ST.")}, + PyDoc_STR("Creates a list-tree representation of an ST.")}, {"tuple2st", (PyCFunction)(void(*)(void))parser_tuple2st, PUBLIC_METHOD_TYPE, - PyDoc_STR("Creates an ST object from a tree representation.")}, - - /* private stuff: support pickle module */ - {"_pickler", (PyCFunction)parser__pickler, METH_VARARGS, - PyDoc_STR("Returns the pickle magic to allow ST objects to be pickled.")}, - - {NULL, NULL, 0, NULL} - }; - - - -static struct PyModuleDef parsermodule = { - PyModuleDef_HEAD_INIT, - "parser", - NULL, - -1, - parser_functions, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC PyInit_parser(void); /* supply a prototype */ - -PyMODINIT_FUNC -PyInit_parser(void) -{ - PyObject *module, *copyreg; - + PyDoc_STR("Creates an ST object from a tree representation.")}, + + /* private stuff: support pickle module */ + {"_pickler", (PyCFunction)parser__pickler, METH_VARARGS, + PyDoc_STR("Returns the pickle magic to allow ST objects to be pickled.")}, + + {NULL, NULL, 0, NULL} + }; + + + +static struct PyModuleDef parsermodule = { + PyModuleDef_HEAD_INIT, + "parser", + NULL, + -1, + parser_functions, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC PyInit_parser(void); /* supply a prototype */ + +PyMODINIT_FUNC +PyInit_parser(void) +{ + PyObject *module, *copyreg; + if (PyErr_WarnEx(PyExc_DeprecationWarning, "The parser module is deprecated and will be removed " "in future versions of Python", 7) != 0) { return NULL; } - if (PyType_Ready(&PyST_Type) < 0) - return NULL; - module = PyModule_Create(&parsermodule); - if (module == NULL) - return NULL; - - if (parser_error == 0) - parser_error = PyErr_NewException("parser.ParserError", NULL, NULL); - - if (parser_error == 0) - return NULL; - /* CAUTION: The code next used to skip bumping the refcount on - * parser_error. That's a disaster if PyInit_parser() gets called more - * than once. By incref'ing, we ensure that each module dict that - * gets created owns its reference to the shared parser_error object, - * and the file static parser_error vrbl owns a reference too. - */ - Py_INCREF(parser_error); - if (PyModule_AddObject(module, "ParserError", parser_error) != 0) - return NULL; - - Py_INCREF(&PyST_Type); - PyModule_AddObject(module, "STType", (PyObject*)&PyST_Type); - - PyModule_AddStringConstant(module, "__copyright__", - parser_copyright_string); - PyModule_AddStringConstant(module, "__doc__", - parser_doc_string); - PyModule_AddStringConstant(module, "__version__", - parser_version_string); - - /* Register to support pickling. - * If this fails, the import of this module will fail because an - * exception will be raised here; should we clear the exception? - */ - copyreg = PyImport_ImportModuleNoBlock("copyreg"); - if (copyreg != NULL) { - PyObject *func, *pickler; - _Py_IDENTIFIER(pickle); - _Py_IDENTIFIER(sequence2st); - _Py_IDENTIFIER(_pickler); - - func = _PyObject_GetAttrId(copyreg, &PyId_pickle); - pickle_constructor = _PyObject_GetAttrId(module, &PyId_sequence2st); - pickler = _PyObject_GetAttrId(module, &PyId__pickler); - Py_XINCREF(pickle_constructor); - if ((func != NULL) && (pickle_constructor != NULL) - && (pickler != NULL)) { - PyObject *res; - - res = PyObject_CallFunctionObjArgs(func, &PyST_Type, pickler, - pickle_constructor, NULL); - Py_XDECREF(res); - } - Py_XDECREF(func); - Py_XDECREF(pickle_constructor); - Py_XDECREF(pickler); - Py_DECREF(copyreg); - } - return module; -} + if (PyType_Ready(&PyST_Type) < 0) + return NULL; + module = PyModule_Create(&parsermodule); + if (module == NULL) + return NULL; + + if (parser_error == 0) + parser_error = PyErr_NewException("parser.ParserError", NULL, NULL); + + if (parser_error == 0) + return NULL; + /* CAUTION: The code next used to skip bumping the refcount on + * parser_error. That's a disaster if PyInit_parser() gets called more + * than once. By incref'ing, we ensure that each module dict that + * gets created owns its reference to the shared parser_error object, + * and the file static parser_error vrbl owns a reference too. + */ + Py_INCREF(parser_error); + if (PyModule_AddObject(module, "ParserError", parser_error) != 0) + return NULL; + + Py_INCREF(&PyST_Type); + PyModule_AddObject(module, "STType", (PyObject*)&PyST_Type); + + PyModule_AddStringConstant(module, "__copyright__", + parser_copyright_string); + PyModule_AddStringConstant(module, "__doc__", + parser_doc_string); + PyModule_AddStringConstant(module, "__version__", + parser_version_string); + + /* Register to support pickling. + * If this fails, the import of this module will fail because an + * exception will be raised here; should we clear the exception? + */ + copyreg = PyImport_ImportModuleNoBlock("copyreg"); + if (copyreg != NULL) { + PyObject *func, *pickler; + _Py_IDENTIFIER(pickle); + _Py_IDENTIFIER(sequence2st); + _Py_IDENTIFIER(_pickler); + + func = _PyObject_GetAttrId(copyreg, &PyId_pickle); + pickle_constructor = _PyObject_GetAttrId(module, &PyId_sequence2st); + pickler = _PyObject_GetAttrId(module, &PyId__pickler); + Py_XINCREF(pickle_constructor); + if ((func != NULL) && (pickle_constructor != NULL) + && (pickler != NULL)) { + PyObject *res; + + res = PyObject_CallFunctionObjArgs(func, &PyST_Type, pickler, + pickle_constructor, NULL); + Py_XDECREF(res); + } + Py_XDECREF(func); + Py_XDECREF(pickle_constructor); + Py_XDECREF(pickler); + Py_DECREF(copyreg); + } + return module; +} |
