diff options
| author | shadchin <[email protected]> | 2022-04-18 12:39:32 +0300 |
|---|---|---|
| committer | shadchin <[email protected]> | 2022-04-18 12:39:32 +0300 |
| commit | d4be68e361f4258cf0848fc70018dfe37a2acc24 (patch) | |
| tree | 153e294cd97ac8b5d7a989612704a0c1f58e8ad4 /contrib/tools/python3/src/Python/ceval.c | |
| parent | 260c02f5ccf242d9d9b8a873afaf6588c00237d6 (diff) | |
IGNIETFERRO-1816 Update Python 3 from 3.9.12 to 3.10.4
ref:9f96be6d02ee8044fdd6f124b799b270c20ce641
Diffstat (limited to 'contrib/tools/python3/src/Python/ceval.c')
| -rw-r--r-- | contrib/tools/python3/src/Python/ceval.c | 1876 |
1 files changed, 1341 insertions, 535 deletions
diff --git a/contrib/tools/python3/src/Python/ceval.c b/contrib/tools/python3/src/Python/ceval.c index 9a61f8a3c3c..21674e0be13 100644 --- a/contrib/tools/python3/src/Python/ceval.c +++ b/contrib/tools/python3/src/Python/ceval.c @@ -1,4 +1,3 @@ - /* Execute compiled code */ /* XXX TO DO: @@ -7,21 +6,22 @@ */ /* enable more aggressive intra-module optimizations, where available */ +/* affects both release and debug builds - see bpo-43271 */ #define PY_LOCAL_AGGRESSIVE #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_call.h" -#include "pycore_ceval.h" -#include "pycore_code.h" -#include "pycore_initconfig.h" -#include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pylifecycle.h" +#include "pycore_call.h" // _PyObject_FastCallDictTstate() +#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() +#include "pycore_code.h" // _PyCode_InitOpcache() +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" -#include "pycore_tupleobject.h" +#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_tuple.h" // _PyTuple_ITEMS() #include "code.h" #include "dictobject.h" @@ -29,9 +29,17 @@ #include "opcode.h" #include "pydtrace.h" #include "setobject.h" +#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX #include <ctype.h> +typedef struct { + PyCodeObject *code; // The code object for the bounds. May be NULL. + PyCodeAddressRange bounds; // Only valid if code != NULL. + CFrame cframe; +} PyTraceInfo; + + #ifdef Py_DEBUG /* For debugging the interpreter: */ #define LLTRACE 1 /* Low-level trace feature */ @@ -46,10 +54,10 @@ _Py_IDENTIFIER(__name__); /* Forward declarations */ Py_LOCAL_INLINE(PyObject *) call_function( - PyThreadState *tstate, PyObject ***pp_stack, + PyThreadState *tstate, PyTraceInfo *, PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames); static PyObject * do_call_core( - PyThreadState *tstate, PyObject *func, + PyThreadState *tstate, PyTraceInfo *, PyObject *func, PyObject *callargs, PyObject *kwdict); #ifdef LLTRACE @@ -58,16 +66,19 @@ static int prtrace(PyThreadState *, PyObject *, const char *); #endif static int call_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, + PyTraceInfo *, int, PyObject *); static int call_trace_protected(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, + PyTraceInfo *, int, PyObject *); static void call_exc_trace(Py_tracefunc, PyObject *, - PyThreadState *, PyFrameObject *); + PyThreadState *, PyFrameObject *, + PyTraceInfo *trace_info); static int maybe_call_line_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, - int *, int *, int *); -static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *); + PyTraceInfo *, int); +static void maybe_dtrace_line(PyFrameObject *, PyTraceInfo *, int); static void dtrace_function_entry(PyFrameObject *); static void dtrace_function_return(PyFrameObject *); @@ -103,16 +114,19 @@ static long dxp[256]; #endif /* per opcode cache */ -#ifdef Py_DEBUG -// --with-pydebug is used to find memory leak. opcache makes it harder. -// So we disable opcache when Py_DEBUG is defined. -// See bpo-37146 -#define OPCACHE_MIN_RUNS 0 /* disable opcache */ -#else -#define OPCACHE_MIN_RUNS 1024 /* create opcache when code executed this time */ -#endif +static int opcache_min_runs = 1024; /* create opcache when code executed this many times */ +#define OPCODE_CACHE_MAX_TRIES 20 #define OPCACHE_STATS 0 /* Enable stats */ +// This function allows to deactivate the opcode cache. As different cache mechanisms may hold +// references, this can mess with the reference leak detector functionality so the cache needs +// to be deactivated in such scenarios to avoid false positives. See bpo-3714 for more information. +void +_PyEval_DeactivateOpCache(void) +{ + opcache_min_runs = 0; +} + #if OPCACHE_STATS static size_t opcache_code_objects = 0; static size_t opcache_code_objects_extra_mem = 0; @@ -120,6 +134,12 @@ static size_t opcache_code_objects_extra_mem = 0; static size_t opcache_global_opts = 0; static size_t opcache_global_hits = 0; static size_t opcache_global_misses = 0; + +static size_t opcache_attr_opts = 0; +static size_t opcache_attr_hits = 0; +static size_t opcache_attr_misses = 0; +static size_t opcache_attr_deopts = 0; +static size_t opcache_attr_total = 0; #endif @@ -254,8 +274,22 @@ _Py_FatalError_TstateNULL(const char *func) "(the current Python thread state is NULL)"); } +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS +int +_PyEval_ThreadsInitialized(PyInterpreterState *interp) +{ + return gil_created(&interp->ceval.gil); +} int +PyEval_ThreadsInitialized(void) +{ + // Fatal error if there is no current interpreter + PyInterpreterState *interp = PyInterpreterState_Get(); + return _PyEval_ThreadsInitialized(interp); +} +#else +int _PyEval_ThreadsInitialized(_PyRuntimeState *runtime) { return gil_created(&runtime->ceval.gil); @@ -267,18 +301,25 @@ PyEval_ThreadsInitialized(void) _PyRuntimeState *runtime = &_PyRuntime; return _PyEval_ThreadsInitialized(runtime); } +#endif PyStatus _PyEval_InitGIL(PyThreadState *tstate) { - if (!_Py_IsMainInterpreter(tstate)) { +#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return _PyStatus_OK(); } +#endif +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; +#else struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; +#endif assert(!gil_created(gil)); PyThread_init_thread(); @@ -291,16 +332,22 @@ _PyEval_InitGIL(PyThreadState *tstate) } void -_PyEval_FiniGIL(PyThreadState *tstate) +_PyEval_FiniGIL(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(tstate)) { +#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return; } +#endif - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + struct _gil_runtime_state *gil = &interp->ceval.gil; +#else + struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; +#endif if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -343,6 +390,25 @@ _PyEval_Fini(void) opcache_global_opts); fprintf(stderr, "\n"); + + fprintf(stderr, "-- Opcode cache LOAD_ATTR hits = %zd (%d%%)\n", + opcache_attr_hits, + (int) (100.0 * opcache_attr_hits / + opcache_attr_total)); + + fprintf(stderr, "-- Opcode cache LOAD_ATTR misses = %zd (%d%%)\n", + opcache_attr_misses, + (int) (100.0 * opcache_attr_misses / + opcache_attr_total)); + + fprintf(stderr, "-- Opcode cache LOAD_ATTR opts = %zd\n", + opcache_attr_opts); + + fprintf(stderr, "-- Opcode cache LOAD_ATTR deopts = %zd\n", + opcache_attr_deopts); + + fprintf(stderr, "-- Opcode cache LOAD_ATTR total = %zd\n", + opcache_attr_total); #endif } @@ -385,9 +451,13 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + (void)_PyThreadState_Swap(gilstate, tstate); +#else if (_PyThreadState_Swap(gilstate, tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } +#endif } void @@ -407,19 +477,20 @@ PyEval_ReleaseThread(PyThreadState *tstate) #ifdef HAVE_FORK /* This function is called from PyOS_AfterFork_Child to destroy all threads - * which are not running in the child process, and clear internal locks - * which might be held by those threads. - */ - -void -_PyEval_ReInitThreads(_PyRuntimeState *runtime) + which are not running in the child process, and clear internal locks + which might be held by those threads. */ +PyStatus +_PyEval_ReInitThreads(PyThreadState *tstate) { - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - _Py_EnsureTstateNotNULL(tstate); + _PyRuntimeState *runtime = tstate->interp->runtime; +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; +#else struct _gil_runtime_state *gil = &runtime->ceval.gil; +#endif if (!gil_created(gil)) { - return; + return _PyStatus_OK(); } recreate_gil(gil); @@ -427,11 +498,12 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) struct _pending_calls *pending = &tstate->interp->ceval.pending; if (_PyThread_at_fork_reinit(&pending->lock) < 0) { - Py_FatalError("Can't initialize threads for pending calls"); + return _PyStatus_ERR("Can't reinitialize pending calls lock"); } /* Destroy all threads except the current one */ _PyThreadState_DeleteExcept(runtime, tstate); + return _PyStatus_OK(); } #endif @@ -439,22 +511,30 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) raised. */ void -_PyEval_SignalAsyncExc(PyThreadState *tstate) +_PyEval_SignalAsyncExc(PyInterpreterState *interp) { - assert(is_tstate_valid(tstate)); - SIGNAL_ASYNC_EXC(tstate->interp); + SIGNAL_ASYNC_EXC(interp); } PyThreadState * PyEval_SaveThread(void) { _PyRuntimeState *runtime = &_PyRuntime; +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + PyThreadState *old_tstate = _PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, old_tstate); +#else PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); +#endif _Py_EnsureTstateNotNULL(tstate); struct _ceval_runtime_state *ceval = &runtime->ceval; struct _ceval_state *ceval2 = &tstate->interp->ceval; +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + assert(gil_created(&ceval2->gil)); +#else assert(gil_created(&ceval->gil)); +#endif drop_gil(ceval, ceval2, tstate); return tstate; } @@ -618,10 +698,8 @@ handle_signals(PyThreadState *tstate) } static int -make_pending_calls(PyThreadState *tstate) +make_pending_calls(PyInterpreterState *interp) { - assert(is_tstate_valid(tstate)); - /* only execute pending calls on main thread */ if (!_Py_ThreadCanHandlePendingCalls()) { return 0; @@ -636,11 +714,11 @@ make_pending_calls(PyThreadState *tstate) /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(tstate->interp); + UNSIGNAL_PENDING_CALLS(interp); int res = 0; /* perform a bounded number of calls, in case of recursion */ - struct _pending_calls *pending = &tstate->interp->ceval.pending; + struct _pending_calls *pending = &interp->ceval.pending; for (int i=0; i<NPENDINGCALLS; i++) { int (*func)(void *) = NULL; void *arg = NULL; @@ -665,7 +743,7 @@ make_pending_calls(PyThreadState *tstate) error: busy = 0; - SIGNAL_PENDING_CALLS(tstate->interp); + SIGNAL_PENDING_CALLS(interp); return res; } @@ -673,6 +751,7 @@ void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); + assert(is_tstate_valid(tstate)); struct _pending_calls *pending = &tstate->interp->ceval.pending; @@ -680,7 +759,7 @@ _Py_FinishPendingCalls(PyThreadState *tstate) return; } - if (make_pending_calls(tstate) < 0) { + if (make_pending_calls(tstate->interp) < 0) { PyObject *exc, *val, *tb; _PyErr_Fetch(tstate, &exc, &val, &tb); PyErr_BadInternalCall(); @@ -697,6 +776,7 @@ Py_MakePendingCalls(void) assert(PyGILState_Check()); PyThreadState *tstate = _PyThreadState_GET(); + assert(is_tstate_valid(tstate)); /* Python signal handler doesn't really queue a callback: it only signals that a signal was received, see _PyEval_SignalReceived(). */ @@ -705,7 +785,7 @@ Py_MakePendingCalls(void) return res; } - res = make_pending_calls(tstate); + res = make_pending_calls(tstate->interp); if (res != 0) { return res; } @@ -716,16 +796,15 @@ Py_MakePendingCalls(void) /* The interpreter's recursion limit */ #ifndef Py_DEFAULT_RECURSION_LIMIT -#define Py_DEFAULT_RECURSION_LIMIT 1000 +# define Py_DEFAULT_RECURSION_LIMIT 1000 #endif -int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT; - void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { - _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT; +#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS _gil_initialize(&ceval->gil); +#endif } int @@ -741,6 +820,10 @@ _PyEval_InitState(struct _ceval_state *ceval) return -1; } +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + _gil_initialize(&ceval->gil); +#endif + return 0; } @@ -757,8 +840,8 @@ _PyEval_FiniState(struct _ceval_state *ceval) int Py_GetRecursionLimit(void) { - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->interp->ceval.recursion_limit; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return interp->ceval.recursion_limit; } void @@ -766,14 +849,11 @@ Py_SetRecursionLimit(int new_limit) { PyThreadState *tstate = _PyThreadState_GET(); tstate->interp->ceval.recursion_limit = new_limit; - if (_Py_IsMainInterpreter(tstate)) { - _Py_CheckRecursionLimit = new_limit; - } } /* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches _Py_CheckRecursionLimit. - If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit + if the recursion_depth reaches recursion_limit. + If USE_STACKCHECK, the macro decrements recursion_limit to guarantee that _Py_CheckRecursiveCall() is regularly called. Without USE_STACKCHECK, there is no need for this. */ int @@ -788,27 +868,20 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow"); return -1; } - if (_Py_IsMainInterpreter(tstate)) { - /* Needed for ABI backwards-compatibility (see bpo-31857) */ - _Py_CheckRecursionLimit = recursion_limit; - } #endif - if (tstate->recursion_critical) - /* Somebody asked that we don't check for recursion. */ - return 0; - if (tstate->overflowed) { - if (tstate->recursion_depth > recursion_limit + 50 || tstate->overflowed > 50) { + if (tstate->recursion_headroom) { + if (tstate->recursion_depth > recursion_limit + 50) { /* Overflowing while handling an overflow. Give up. */ Py_FatalError("Cannot recover from stack overflow."); } } else { if (tstate->recursion_depth > recursion_limit) { - tstate->overflowed++; + tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "maximum recursion depth exceeded%s", where); - tstate->overflowed--; + tstate->recursion_headroom--; --tstate->recursion_depth; return -1; } @@ -816,21 +889,249 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } + +// PEP 634: Structural Pattern Matching + + +// Return a tuple of values corresponding to keys, with error checks for +// duplicate/missing keys. +static PyObject* +match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys) +{ + assert(PyTuple_CheckExact(keys)); + Py_ssize_t nkeys = PyTuple_GET_SIZE(keys); + if (!nkeys) { + // No keys means no items. + return PyTuple_New(0); + } + PyObject *seen = NULL; + PyObject *dummy = NULL; + PyObject *values = NULL; + // We use the two argument form of map.get(key, default) for two reasons: + // - Atomically check for a key and get its value without error handling. + // - Don't cause key creation or resizing in dict subclasses like + // collections.defaultdict that define __missing__ (or similar). + _Py_IDENTIFIER(get); + PyObject *get = _PyObject_GetAttrId(map, &PyId_get); + if (get == NULL) { + goto fail; + } + seen = PySet_New(NULL); + if (seen == NULL) { + goto fail; + } + // dummy = object() + dummy = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); + if (dummy == NULL) { + goto fail; + } + values = PyList_New(0); + if (values == NULL) { + goto fail; + } + for (Py_ssize_t i = 0; i < nkeys; i++) { + PyObject *key = PyTuple_GET_ITEM(keys, i); + if (PySet_Contains(seen, key) || PySet_Add(seen, key)) { + if (!_PyErr_Occurred(tstate)) { + // Seen it before! + _PyErr_Format(tstate, PyExc_ValueError, + "mapping pattern checks duplicate key (%R)", key); + } + goto fail; + } + PyObject *value = PyObject_CallFunctionObjArgs(get, key, dummy, NULL); + if (value == NULL) { + goto fail; + } + if (value == dummy) { + // key not in map! + Py_DECREF(value); + Py_DECREF(values); + // Return None: + Py_INCREF(Py_None); + values = Py_None; + goto done; + } + PyList_Append(values, value); + Py_DECREF(value); + } + Py_SETREF(values, PyList_AsTuple(values)); + // Success: +done: + Py_DECREF(get); + Py_DECREF(seen); + Py_DECREF(dummy); + return values; +fail: + Py_XDECREF(get); + Py_XDECREF(seen); + Py_XDECREF(dummy); + Py_XDECREF(values); + return NULL; +} + +// Extract a named attribute from the subject, with additional bookkeeping to +// raise TypeErrors for repeated lookups. On failure, return NULL (with no +// error set). Use _PyErr_Occurred(tstate) to disambiguate. +static PyObject* +match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type, + PyObject *name, PyObject *seen) +{ + assert(PyUnicode_CheckExact(name)); + assert(PySet_CheckExact(seen)); + if (PySet_Contains(seen, name) || PySet_Add(seen, name)) { + if (!_PyErr_Occurred(tstate)) { + // Seen it before! + _PyErr_Format(tstate, PyExc_TypeError, + "%s() got multiple sub-patterns for attribute %R", + ((PyTypeObject*)type)->tp_name, name); + } + return NULL; + } + PyObject *attr = PyObject_GetAttr(subject, name); + if (attr == NULL && _PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Clear(tstate); + } + return attr; +} + +// On success (match), return a tuple of extracted attributes. On failure (no +// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate. +static PyObject* +match_class(PyThreadState *tstate, PyObject *subject, PyObject *type, + Py_ssize_t nargs, PyObject *kwargs) +{ + if (!PyType_Check(type)) { + const char *e = "called match pattern must be a type"; + _PyErr_Format(tstate, PyExc_TypeError, e); + return NULL; + } + assert(PyTuple_CheckExact(kwargs)); + // First, an isinstance check: + if (PyObject_IsInstance(subject, type) <= 0) { + return NULL; + } + // So far so good: + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + return NULL; + } + PyObject *attrs = PyList_New(0); + if (attrs == NULL) { + Py_DECREF(seen); + return NULL; + } + // NOTE: From this point on, goto fail on failure: + PyObject *match_args = NULL; + // First, the positional subpatterns: + if (nargs) { + int match_self = 0; + match_args = PyObject_GetAttrString(type, "__match_args__"); + if (match_args) { + if (!PyTuple_CheckExact(match_args)) { + const char *e = "%s.__match_args__ must be a tuple (got %s)"; + _PyErr_Format(tstate, PyExc_TypeError, e, + ((PyTypeObject *)type)->tp_name, + Py_TYPE(match_args)->tp_name); + goto fail; + } + } + else if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Clear(tstate); + // _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not + // define __match_args__. This is natural behavior for subclasses: + // it's as if __match_args__ is some "magic" value that is lost as + // soon as they redefine it. + match_args = PyTuple_New(0); + match_self = PyType_HasFeature((PyTypeObject*)type, + _Py_TPFLAGS_MATCH_SELF); + } + else { + goto fail; + } + assert(PyTuple_CheckExact(match_args)); + Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args); + if (allowed < nargs) { + const char *plural = (allowed == 1) ? "" : "s"; + _PyErr_Format(tstate, PyExc_TypeError, + "%s() accepts %d positional sub-pattern%s (%d given)", + ((PyTypeObject*)type)->tp_name, + allowed, plural, nargs); + goto fail; + } + if (match_self) { + // Easy. Copy the subject itself, and move on to kwargs. + PyList_Append(attrs, subject); + } + else { + for (Py_ssize_t i = 0; i < nargs; i++) { + PyObject *name = PyTuple_GET_ITEM(match_args, i); + if (!PyUnicode_CheckExact(name)) { + _PyErr_Format(tstate, PyExc_TypeError, + "__match_args__ elements must be strings " + "(got %s)", Py_TYPE(name)->tp_name); + goto fail; + } + PyObject *attr = match_class_attr(tstate, subject, type, name, + seen); + if (attr == NULL) { + goto fail; + } + PyList_Append(attrs, attr); + Py_DECREF(attr); + } + } + Py_CLEAR(match_args); + } + // Finally, the keyword subpatterns: + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) { + PyObject *name = PyTuple_GET_ITEM(kwargs, i); + PyObject *attr = match_class_attr(tstate, subject, type, name, seen); + if (attr == NULL) { + goto fail; + } + PyList_Append(attrs, attr); + Py_DECREF(attr); + } + Py_SETREF(attrs, PyList_AsTuple(attrs)); + Py_DECREF(seen); + return attrs; +fail: + // We really don't care whether an error was raised or not... that's our + // caller's problem. All we know is that the match failed. + Py_XDECREF(match_args); + Py_DECREF(seen); + Py_DECREF(attrs); + return NULL; +} + + static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause); static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **); -#define _Py_TracingPossible(ceval) ((ceval)->tracing_possible) - PyObject * PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) { - return PyEval_EvalCodeEx(co, - globals, locals, - (PyObject **)NULL, 0, - (PyObject **)NULL, 0, - (PyObject **)NULL, 0, - NULL, NULL); + PyThreadState *tstate = PyThreadState_GET(); + if (locals == NULL) { + locals = globals; + } + PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + if (builtins == NULL) { + return NULL; + } + PyFrameConstructor desc = { + .fc_globals = globals, + .fc_builtins = builtins, + .fc_name = ((PyCodeObject *)co)->co_name, + .fc_qualname = ((PyCodeObject *)co)->co_name, + .fc_code = co, + .fc_defaults = NULL, + .fc_kwdefaults = NULL, + .fc_closure = NULL + }; + return _PyEval_Vector(tstate, &desc, locals, NULL, 0, NULL); } @@ -870,7 +1171,7 @@ eval_frame_handle_pending(PyThreadState *tstate) /* Pending calls */ struct _ceval_state *ceval2 = &tstate->interp->ceval; if (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)) { - if (make_pending_calls(tstate) != 0) { + if (make_pending_calls(tstate->interp) != 0) { return -1; } } @@ -887,9 +1188,13 @@ eval_frame_handle_pending(PyThreadState *tstate) take_gil(tstate); +#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS + (void)_PyThreadState_Swap(&runtime->gilstate, tstate); +#else if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("orphan tstate"); } +#endif } /* Check for asynchronous exception. */ @@ -916,41 +1221,6 @@ eval_frame_handle_pending(PyThreadState *tstate) return 0; } -PyObject* _Py_HOT_FUNCTION -_PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) -{ - _Py_EnsureTstateNotNULL(tstate); - -#ifdef DXPAIRS - int lastopcode = 0; -#endif - PyObject **stack_pointer; /* Next free slot in value stack */ - const _Py_CODEUNIT *next_instr; - int opcode; /* Current opcode */ - int oparg; /* Current opcode argument, if any */ - PyObject **fastlocals, **freevars; - PyObject *retval = NULL; /* Return value */ - struct _ceval_state * const ceval2 = &tstate->interp->ceval; - _Py_atomic_int * const eval_breaker = &ceval2->eval_breaker; - PyCodeObject *co; - - /* when tracing we set things up so that - - not (instr_lb <= current_bytecode_offset < instr_ub) - - is true when the line being executed has changed. The - initial values are such as to make this false the first - time it is tested. */ - int instr_ub = -1, instr_lb = 0, instr_prev = -1; - - const _Py_CODEUNIT *first_instr; - PyObject *names; - PyObject *consts; - _PyOpcache *co_opcache; - -#ifdef LLTRACE - _Py_IDENTIFIER(__ltrace__); -#endif /* Computed GOTOs, or the-optimization-commonly-but-improperly-known-as-"threaded code" @@ -991,6 +1261,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) -fno-crossjumping). */ +/* Use macros rather than inline functions, to make it as clear as possible + * to the C compiler that the tracing check is a simple test then branch. + * We want to be sure that the compiler knows this before it generates + * the CFG. + */ +#ifdef LLTRACE +#define OR_LLTRACE || lltrace +#else +#define OR_LLTRACE +#endif + +#ifdef WITH_DTRACE +#define OR_DTRACE_LINE || PyDTrace_LINE_ENABLED() +#else +#define OR_DTRACE_LINE +#endif + #ifdef DYNAMIC_EXECUTION_PROFILE #undef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 0 @@ -1009,49 +1296,27 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) #endif #if USE_COMPUTED_GOTOS -/* Import the static jump table */ -#include "opcode_targets.h" - -#define TARGET(op) \ - op: \ - TARGET_##op - -#ifdef LLTRACE -#define FAST_DISPATCH() \ +#define TARGET(op) op: TARGET_##op +#define DISPATCH() \ { \ - if (!lltrace && !_Py_TracingPossible(ceval2) && !PyDTrace_LINE_ENABLED()) { \ - f->f_lasti = INSTR_OFFSET(); \ - NEXTOPARG(); \ - goto *opcode_targets[opcode]; \ + if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) { \ + goto tracing_dispatch; \ } \ - goto fast_next_opcode; \ + f->f_lasti = INSTR_OFFSET(); \ + NEXTOPARG(); \ + goto *opcode_targets[opcode]; \ } #else -#define FAST_DISPATCH() \ - { \ - if (!_Py_TracingPossible(ceval2) && !PyDTrace_LINE_ENABLED()) { \ - f->f_lasti = INSTR_OFFSET(); \ - NEXTOPARG(); \ - goto *opcode_targets[opcode]; \ - } \ - goto fast_next_opcode; \ - } +#define TARGET(op) op +#define DISPATCH() goto predispatch; #endif -#define DISPATCH() \ - { \ - if (!_Py_atomic_load_relaxed(eval_breaker)) { \ - FAST_DISPATCH(); \ - } \ + +#define CHECK_EVAL_BREAKER() \ + if (_Py_atomic_load_relaxed(eval_breaker)) { \ continue; \ } -#else -#define TARGET(op) op -#define FAST_DISPATCH() goto fast_next_opcode -#define DISPATCH() continue -#endif - /* Tuple access macros */ @@ -1064,16 +1329,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) /* Code access macros */ /* The integer overflow is checked by an assertion below. */ -#define INSTR_OFFSET() \ - (sizeof(_Py_CODEUNIT) * (int)(next_instr - first_instr)) +#define INSTR_OFFSET() ((int)(next_instr - first_instr)) #define NEXTOPARG() do { \ _Py_CODEUNIT word = *next_instr; \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ next_instr++; \ } while (0) -#define JUMPTO(x) (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT)) -#define JUMPBY(x) (next_instr += (x) / sizeof(_Py_CODEUNIT)) +#define JUMPTO(x) (next_instr = first_instr + (x)) +#define JUMPBY(x) (next_instr += (x)) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to @@ -1135,7 +1399,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) #define SET_SECOND(v) (stack_pointer[-2] = (v)) #define SET_THIRD(v) (stack_pointer[-3] = (v)) #define SET_FOURTH(v) (stack_pointer[-4] = (v)) -#define SET_VALUE(n, v) (stack_pointer[-(n)] = (v)) #define BASIC_STACKADJ(n) (stack_pointer += n) #define BASIC_PUSH(v) (*stack_pointer++ = (v)) #define BASIC_POP() (*--stack_pointer) @@ -1216,16 +1479,43 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) do { \ co_opcache = NULL; \ if (co->co_opcache != NULL) { \ - unsigned char co_opt_offset = \ + unsigned char co_opcache_offset = \ co->co_opcache_map[next_instr - first_instr]; \ - if (co_opt_offset > 0) { \ - assert(co_opt_offset <= co->co_opcache_size); \ - co_opcache = &co->co_opcache[co_opt_offset - 1]; \ + if (co_opcache_offset > 0) { \ + assert(co_opcache_offset <= co->co_opcache_size); \ + co_opcache = &co->co_opcache[co_opcache_offset - 1]; \ assert(co_opcache != NULL); \ } \ } \ } while (0) +#define OPCACHE_DEOPT() \ + do { \ + if (co_opcache != NULL) { \ + co_opcache->optimized = -1; \ + unsigned char co_opcache_offset = \ + co->co_opcache_map[next_instr - first_instr]; \ + assert(co_opcache_offset <= co->co_opcache_size); \ + co->co_opcache_map[co_opcache_offset] = 0; \ + co_opcache = NULL; \ + } \ + } while (0) + +#define OPCACHE_DEOPT_LOAD_ATTR() \ + do { \ + if (co_opcache != NULL) { \ + OPCACHE_STAT_ATTR_DEOPT(); \ + OPCACHE_DEOPT(); \ + } \ + } while (0) + +#define OPCACHE_MAYBE_DEOPT_LOAD_ATTR() \ + do { \ + if (co_opcache != NULL && --co_opcache->optimized <= 0) { \ + OPCACHE_DEOPT_LOAD_ATTR(); \ + } \ + } while (0) + #if OPCACHE_STATS #define OPCACHE_STAT_GLOBAL_HIT() \ @@ -1243,24 +1533,99 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) if (co->co_opcache != NULL) opcache_global_opts++; \ } while (0) +#define OPCACHE_STAT_ATTR_HIT() \ + do { \ + if (co->co_opcache != NULL) opcache_attr_hits++; \ + } while (0) + +#define OPCACHE_STAT_ATTR_MISS() \ + do { \ + if (co->co_opcache != NULL) opcache_attr_misses++; \ + } while (0) + +#define OPCACHE_STAT_ATTR_OPT() \ + do { \ + if (co->co_opcache!= NULL) opcache_attr_opts++; \ + } while (0) + +#define OPCACHE_STAT_ATTR_DEOPT() \ + do { \ + if (co->co_opcache != NULL) opcache_attr_deopts++; \ + } while (0) + +#define OPCACHE_STAT_ATTR_TOTAL() \ + do { \ + if (co->co_opcache != NULL) opcache_attr_total++; \ + } while (0) + #else /* OPCACHE_STATS */ #define OPCACHE_STAT_GLOBAL_HIT() #define OPCACHE_STAT_GLOBAL_MISS() #define OPCACHE_STAT_GLOBAL_OPT() +#define OPCACHE_STAT_ATTR_HIT() +#define OPCACHE_STAT_ATTR_MISS() +#define OPCACHE_STAT_ATTR_OPT() +#define OPCACHE_STAT_ATTR_DEOPT() +#define OPCACHE_STAT_ATTR_TOTAL() + #endif -/* Start of code */ - /* push frame */ +PyObject* _Py_HOT_FUNCTION +_PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" +#endif + +#ifdef DXPAIRS + int lastopcode = 0; +#endif + PyObject **stack_pointer; /* Next free slot in value stack */ + const _Py_CODEUNIT *next_instr; + int opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ + PyObject **fastlocals, **freevars; + PyObject *retval = NULL; /* Return value */ + _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; + PyCodeObject *co; + + const _Py_CODEUNIT *first_instr; + PyObject *names; + PyObject *consts; + _PyOpcache *co_opcache; + +#ifdef LLTRACE + _Py_IDENTIFIER(__ltrace__); +#endif + if (_Py_EnterRecursiveCall(tstate, "")) { return NULL; } + PyTraceInfo trace_info; + /* Mark trace_info as uninitialized */ + trace_info.code = NULL; + + /* WARNING: Because the CFrame lives on the C stack, + * but can be accessed from a heap allocated object (tstate) + * strict stack discipline must be maintained. + */ + CFrame *prev_cframe = tstate->cframe; + trace_info.cframe.use_tracing = prev_cframe->use_tracing; + trace_info.cframe.previous = prev_cframe; + tstate->cframe = &trace_info.cframe; + + /* push frame */ tstate->frame = f; + co = f->f_code; - if (tstate->use_tracing) { + if (trace_info.cframe.use_tracing) { if (tstate->c_tracefunc != NULL) { /* tstate->c_tracefunc, if defined, is a function that will be called on *every* entry @@ -1277,7 +1642,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) whenever an exception is detected. */ if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, - tstate, f, PyTrace_CALL, Py_None)) { + tstate, f, &trace_info, + PyTrace_CALL, Py_None)) { /* Trace function raised an error */ goto exit_eval_frame; } @@ -1287,7 +1653,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) return itself and isn't called for "line" events */ if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, - tstate, f, PyTrace_CALL, Py_None)) { + tstate, f, &trace_info, + PyTrace_CALL, Py_None)) { /* Profile function raised an error */ goto exit_eval_frame; } @@ -1297,7 +1664,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) if (PyDTrace_FUNCTION_ENTRY_ENABLED()) dtrace_function_entry(f); - co = f->f_code; names = co->co_names; consts = co->co_consts; fastlocals = f->f_localsplus; @@ -1323,19 +1689,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) to the beginning of the combined pair.) */ assert(f->f_lasti >= -1); - next_instr = first_instr; - if (f->f_lasti >= 0) { - assert(f->f_lasti % sizeof(_Py_CODEUNIT) == 0); - next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1; - } - stack_pointer = f->f_stacktop; - assert(stack_pointer != NULL); - f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ - f->f_executing = 1; + next_instr = first_instr + f->f_lasti + 1; + stack_pointer = f->f_valuestack + f->f_stackdepth; + /* Set f->f_stackdepth to -1. + * Update when returning or calling trace function. + Having f_stackdepth <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. + */ + f->f_stackdepth = -1; + f->f_state = FRAME_EXECUTING; - if (co->co_opcache_flag < OPCACHE_MIN_RUNS) { + if (co->co_opcache_flag < opcache_min_runs) { co->co_opcache_flag++; - if (co->co_opcache_flag == OPCACHE_MIN_RUNS) { + if (co->co_opcache_flag == opcache_min_runs) { if (_PyCode_InitOpcache(co) < 0) { goto exit_eval_frame; } @@ -1349,11 +1716,18 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) } #ifdef LLTRACE - lltrace = _PyDict_GetItemId(f->f_globals, &PyId___ltrace__) != NULL; + { + int r = _PyDict_ContainsId(f->f_globals, &PyId___ltrace__); + if (r < 0) { + goto exit_eval_frame; + } + lltrace = r; + } #endif - if (throwflag) /* support for generator.throw() */ + if (throwflag) { /* support for generator.throw() */ goto error; + } #ifdef Py_DEBUG /* _PyEval_EvalFrameDefault() must not be called with an exception set, @@ -1378,10 +1752,10 @@ main_loop: if (_Py_atomic_load_relaxed(eval_breaker)) { opcode = _Py_OPCODE(*next_instr); - if (opcode == SETUP_FINALLY || - opcode == SETUP_WITH || - opcode == BEFORE_ASYNC_WITH || - opcode == YIELD_FROM) { + if (opcode != SETUP_FINALLY && + opcode != SETUP_WITH && + opcode != BEFORE_ASYNC_WITH && + opcode != YIELD_FROM) { /* Few cases where we skip running signal handlers and other pending calls: - If we're about to enter the 'with:'. It will prevent @@ -1398,55 +1772,45 @@ main_loop: running the signal handler and raising KeyboardInterrupt (see bpo-30039). */ - goto fast_next_opcode; - } - - if (eval_frame_handle_pending(tstate) != 0) { - goto error; - } + if (eval_frame_handle_pending(tstate) != 0) { + goto error; + } + } } - fast_next_opcode: + tracing_dispatch: + { + int instr_prev = f->f_lasti; f->f_lasti = INSTR_OFFSET(); + NEXTOPARG(); if (PyDTrace_LINE_ENABLED()) - maybe_dtrace_line(f, &instr_lb, &instr_ub, &instr_prev); + maybe_dtrace_line(f, &trace_info, instr_prev); /* line-by-line tracing support */ - if (_Py_TracingPossible(ceval2) && + if (trace_info.cframe.use_tracing && tstate->c_tracefunc != NULL && !tstate->tracing) { int err; - /* see maybe_call_line_trace + /* see maybe_call_line_trace() for expository comments */ - f->f_stacktop = stack_pointer; + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); err = maybe_call_line_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, - &instr_lb, &instr_ub, &instr_prev); + &trace_info, instr_prev); /* Reload possibly changed frame fields */ JUMPTO(f->f_lasti); - if (f->f_stacktop != NULL) { - stack_pointer = f->f_stacktop; - f->f_stacktop = NULL; - } - if (err) + stack_pointer = f->f_valuestack+f->f_stackdepth; + f->f_stackdepth = -1; + if (err) { /* trace function raised an exception */ goto error; + } + NEXTOPARG(); } - - /* Extract opcode and argument */ - - NEXTOPARG(); - dispatch_opcode: -#ifdef DYNAMIC_EXECUTION_PROFILE -#ifdef DXPAIRS - dxpairs[lastopcode][opcode]++; - lastopcode = opcode; -#endif - dxp[opcode]++; -#endif + } #ifdef LLTRACE /* Instruction tracing */ @@ -1462,15 +1826,33 @@ main_loop: } } #endif +#if USE_COMPUTED_GOTOS == 0 + goto dispatch_opcode; + + predispatch: + if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) { + goto tracing_dispatch; + } + f->f_lasti = INSTR_OFFSET(); + NEXTOPARG(); +#endif + dispatch_opcode: +#ifdef DYNAMIC_EXECUTION_PROFILE +#ifdef DXPAIRS + dxpairs[lastopcode][opcode]++; + lastopcode = opcode; +#endif + dxp[opcode]++; +#endif switch (opcode) { /* BEWARE! It is essential that any operation that fails must goto error - and that all operation that succeed call [FAST_]DISPATCH() ! */ + and that all operation that succeed call DISPATCH() ! */ case TARGET(NOP): { - FAST_DISPATCH(); + DISPATCH(); } case TARGET(LOAD_FAST): { @@ -1483,7 +1865,7 @@ main_loop: } Py_INCREF(value); PUSH(value); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(LOAD_CONST): { @@ -1491,20 +1873,20 @@ main_loop: PyObject *value = GETITEM(consts, oparg); Py_INCREF(value); PUSH(value); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(STORE_FAST): { PREDICTED(STORE_FAST); PyObject *value = POP(); SETLOCAL(oparg, value); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(POP_TOP): { PyObject *value = POP(); Py_DECREF(value); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(ROT_TWO): { @@ -1512,7 +1894,7 @@ main_loop: PyObject *second = SECOND(); SET_TOP(second); SET_SECOND(top); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(ROT_THREE): { @@ -1522,7 +1904,7 @@ main_loop: SET_TOP(second); SET_SECOND(third); SET_THIRD(top); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(ROT_FOUR): { @@ -1534,14 +1916,14 @@ main_loop: SET_SECOND(third); SET_THIRD(fourth); SET_FOURTH(top); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(DUP_TOP): { PyObject *top = TOP(); Py_INCREF(top); PUSH(top); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(DUP_TOP_TWO): { @@ -1552,7 +1934,7 @@ main_loop: STACK_GROW(2); SET_TOP(top); SET_SECOND(second); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(UNARY_POSITIVE): { @@ -1687,7 +2069,7 @@ main_loop: PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; - /* NOTE(haypo): Please don't try to micro-optimize int+int on + /* NOTE(vstinner): Please don't try to micro-optimize int+int on CPython using bytecode, it is simply worthless. See http://bugs.python.org/issue21955 and http://bugs.python.org/issue10044 for the discussion. In short, @@ -2055,6 +2437,8 @@ main_loop: retval = POP(); assert(f->f_iblock == 0); assert(EMPTY()); + f->f_state = FRAME_RETURNED; + f->f_stackdepth = 0; goto exiting; } @@ -2197,34 +2581,52 @@ main_loop: case TARGET(YIELD_FROM): { PyObject *v = POP(); PyObject *receiver = TOP(); - int err; - if (PyGen_CheckExact(receiver) || PyCoro_CheckExact(receiver)) { - retval = _PyGen_Send((PyGenObject *)receiver, v); + PySendResult gen_status; + if (tstate->c_tracefunc == NULL) { + gen_status = PyIter_Send(receiver, v, &retval); } else { _Py_IDENTIFIER(send); - if (v == Py_None) + if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); - else + } + else { retval = _PyObject_CallMethodIdOneArg(receiver, &PyId_send, v); + } + if (retval == NULL) { + if (tstate->c_tracefunc != NULL + && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + gen_status = PYGEN_RETURN; + } + else { + gen_status = PYGEN_ERROR; + } + } + else { + gen_status = PYGEN_NEXT; + } } Py_DECREF(v); - if (retval == NULL) { - PyObject *val; - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f); - err = _PyGen_FetchStopIterationValue(&val); - if (err < 0) - goto error; + if (gen_status == PYGEN_ERROR) { + assert (retval == NULL); + goto error; + } + if (gen_status == PYGEN_RETURN) { + assert (retval != NULL); + Py_DECREF(receiver); - SET_TOP(val); + SET_TOP(retval); + retval = NULL; DISPATCH(); } + assert (gen_status == PYGEN_NEXT); /* receiver remains on stack, retval is value to be yielded */ - f->f_stacktop = stack_pointer; /* and repeat... */ - assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT)); - f->f_lasti -= sizeof(_Py_CODEUNIT); + assert(f->f_lasti > 0); + f->f_lasti -= 1; + f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); goto exiting; } @@ -2240,11 +2642,19 @@ main_loop: } retval = w; } - - f->f_stacktop = stack_pointer; + f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); goto exiting; } + case TARGET(GEN_START): { + PyObject *none = POP(); + assert(none == Py_None); + assert(oparg < 3); + Py_DECREF(none); + DISPATCH(); + } + case TARGET(POP_EXCEPT): { PyObject *type, *value, *traceback; _PyErr_StackItem *exc_info; @@ -2270,12 +2680,15 @@ main_loop: } case TARGET(POP_BLOCK): { - PREDICTED(POP_BLOCK); PyFrame_BlockPop(f); DISPATCH(); } case TARGET(RERAISE): { + assert(f->f_iblock > 0); + if (oparg) { + f->f_lasti = f->f_blockstack[f->f_iblock-1].b_handler; + } PyObject *exc = POP(); PyObject *val = POP(); PyObject *tb = POP(); @@ -2294,7 +2707,7 @@ main_loop: UNWIND_EXCEPT_HANDLER(b); Py_DECREF(POP()); JUMPBY(oparg); - FAST_DISPATCH(); + DISPATCH(); } else { PyObject *val = POP(); @@ -2308,7 +2721,7 @@ main_loop: PyObject *value = PyExc_AssertionError; Py_INCREF(value); PUSH(value); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(LOAD_BUILD_CLASS): { @@ -2571,7 +2984,7 @@ main_loop: (PyDictObject *)f->f_builtins, name); if (v == NULL) { - if (!_PyErr_OCCURRED()) { + if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ format_exc_check_arg(tstate, PyExc_NameError, @@ -2993,7 +3406,190 @@ main_loop: case TARGET(LOAD_ATTR): { PyObject *name = GETITEM(names, oparg); PyObject *owner = TOP(); - PyObject *res = PyObject_GetAttr(owner, name); + + PyTypeObject *type = Py_TYPE(owner); + PyObject *res; + PyObject **dictptr; + PyObject *dict; + _PyOpCodeOpt_LoadAttr *la; + + OPCACHE_STAT_ATTR_TOTAL(); + + OPCACHE_CHECK(); + if (co_opcache != NULL && PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) + { + if (co_opcache->optimized > 0) { + // Fast path -- cache hit makes LOAD_ATTR ~30% faster. + la = &co_opcache->u.la; + if (la->type == type && la->tp_version_tag == type->tp_version_tag) + { + // Hint >= 0 is a dict index; hint == -1 is a dict miss. + // Hint < -1 is an inverted slot offset: offset is strictly > 0, + // so ~offset is strictly < -1 (assuming 2's complement). + if (la->hint < -1) { + // Even faster path -- slot hint. + Py_ssize_t offset = ~la->hint; + // fprintf(stderr, "Using hint for offset %zd\n", offset); + char *addr = (char *)owner + offset; + res = *(PyObject **)addr; + if (res != NULL) { + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + DISPATCH(); + } + // Else slot is NULL. Fall through to slow path to raise AttributeError(name). + // Don't DEOPT, since the slot is still there. + } else { + // Fast path for dict. + assert(type->tp_dict != NULL); + assert(type->tp_dictoffset > 0); + + dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); + dict = *dictptr; + if (dict != NULL && PyDict_CheckExact(dict)) { + Py_ssize_t hint = la->hint; + Py_INCREF(dict); + res = NULL; + assert(!_PyErr_Occurred(tstate)); + la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res); + if (res != NULL) { + assert(la->hint >= 0); + if (la->hint == hint && hint >= 0) { + // Our hint has helped -- cache hit. + OPCACHE_STAT_ATTR_HIT(); + } else { + // The hint we provided didn't work. + // Maybe next time? + OPCACHE_MAYBE_DEOPT_LOAD_ATTR(); + } + + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + Py_DECREF(dict); + DISPATCH(); + } + else { + _PyErr_Clear(tstate); + // This attribute can be missing sometimes; + // we don't want to optimize this lookup. + OPCACHE_DEOPT_LOAD_ATTR(); + Py_DECREF(dict); + } + } + else { + // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } + } + else { + // The type of the object has either been updated, + // or is different. Maybe it will stabilize? + OPCACHE_MAYBE_DEOPT_LOAD_ATTR(); + } + OPCACHE_STAT_ATTR_MISS(); + } + + if (co_opcache != NULL && // co_opcache can be NULL after a DEOPT() call. + type->tp_getattro == PyObject_GenericGetAttr) + { + if (type->tp_dict == NULL) { + if (PyType_Ready(type) < 0) { + Py_DECREF(owner); + SET_TOP(NULL); + goto error; + } + } + PyObject *descr = _PyType_Lookup(type, name); + if (descr != NULL) { + // We found an attribute with a data-like descriptor. + PyTypeObject *dtype = Py_TYPE(descr); + if (dtype == &PyMemberDescr_Type) { // It's a slot + PyMemberDescrObject *member = (PyMemberDescrObject *)descr; + struct PyMemberDef *dmem = member->d_member; + if (dmem->type == T_OBJECT_EX) { + Py_ssize_t offset = dmem->offset; + assert(offset > 0); // 0 would be confused with dict hint == -1 (miss). + + if (co_opcache->optimized == 0) { + // First time we optimize this opcode. + OPCACHE_STAT_ATTR_OPT(); + co_opcache->optimized = OPCODE_CACHE_MAX_TRIES; + // fprintf(stderr, "Setting hint for %s, offset %zd\n", dmem->name, offset); + } + + la = &co_opcache->u.la; + la->type = type; + la->tp_version_tag = type->tp_version_tag; + la->hint = ~offset; + + char *addr = (char *)owner + offset; + res = *(PyObject **)addr; + if (res != NULL) { + Py_INCREF(res); + Py_DECREF(owner); + SET_TOP(res); + + DISPATCH(); + } + // Else slot is NULL. Fall through to slow path to raise AttributeError(name). + } + // Else it's a slot of a different type. We don't handle those. + } + // Else it's some other kind of descriptor that we don't handle. + OPCACHE_DEOPT_LOAD_ATTR(); + } + else if (type->tp_dictoffset > 0) { + // We found an instance with a __dict__. + dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); + dict = *dictptr; + + if (dict != NULL && PyDict_CheckExact(dict)) { + Py_INCREF(dict); + res = NULL; + assert(!_PyErr_Occurred(tstate)); + Py_ssize_t hint = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res); + if (res != NULL) { + Py_INCREF(res); + Py_DECREF(dict); + Py_DECREF(owner); + SET_TOP(res); + + if (co_opcache->optimized == 0) { + // First time we optimize this opcode. + OPCACHE_STAT_ATTR_OPT(); + co_opcache->optimized = OPCODE_CACHE_MAX_TRIES; + } + + la = &co_opcache->u.la; + la->type = type; + la->tp_version_tag = type->tp_version_tag; + assert(hint >= 0); + la->hint = hint; + + DISPATCH(); + } + else { + _PyErr_Clear(tstate); + } + Py_DECREF(dict); + } else { + // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } else { + // The object's class does not have a tp_dictoffset we can use. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } else if (type->tp_getattro != PyObject_GenericGetAttr) { + OPCACHE_DEOPT_LOAD_ATTR(); + } + } + + // Slow path. + res = PyObject_GetAttr(owner, name); Py_DECREF(owner); SET_TOP(res); if (res == NULL) @@ -3019,7 +3615,7 @@ main_loop: case TARGET(IS_OP): { PyObject *right = POP(); PyObject *left = TOP(); - int res = (left == right)^oparg; + int res = Py_Is(left, right) ^ oparg; PyObject *b = res ? Py_True : Py_False; Py_INCREF(b); SET_TOP(b); @@ -3027,7 +3623,7 @@ main_loop: Py_DECREF(right); PREDICT(POP_JUMP_IF_FALSE); PREDICT(POP_JUMP_IF_TRUE); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(CONTAINS_OP): { @@ -3044,7 +3640,7 @@ main_loop: PUSH(b); PREDICT(POP_JUMP_IF_FALSE); PREDICT(POP_JUMP_IF_TRUE); - FAST_DISPATCH(); + DISPATCH(); } #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ @@ -3141,28 +3737,31 @@ main_loop: case TARGET(JUMP_FORWARD): { JUMPBY(oparg); - FAST_DISPATCH(); + DISPATCH(); } case TARGET(POP_JUMP_IF_FALSE): { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = POP(); int err; - if (cond == Py_True) { + if (Py_IsTrue(cond)) { Py_DECREF(cond); - FAST_DISPATCH(); + DISPATCH(); } - if (cond == Py_False) { + if (Py_IsFalse(cond)) { Py_DECREF(cond); JUMPTO(oparg); - FAST_DISPATCH(); + CHECK_EVAL_BREAKER(); + DISPATCH(); } err = PyObject_IsTrue(cond); Py_DECREF(cond); if (err > 0) ; - else if (err == 0) + else if (err == 0) { JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + } else goto error; DISPATCH(); @@ -3172,19 +3771,21 @@ main_loop: PREDICTED(POP_JUMP_IF_TRUE); PyObject *cond = POP(); int err; - if (cond == Py_False) { + if (Py_IsFalse(cond)) { Py_DECREF(cond); - FAST_DISPATCH(); + DISPATCH(); } - if (cond == Py_True) { + if (Py_IsTrue(cond)) { Py_DECREF(cond); JUMPTO(oparg); - FAST_DISPATCH(); + CHECK_EVAL_BREAKER(); + DISPATCH(); } err = PyObject_IsTrue(cond); Py_DECREF(cond); if (err > 0) { JUMPTO(oparg); + CHECK_EVAL_BREAKER(); } else if (err == 0) ; @@ -3196,14 +3797,14 @@ main_loop: case TARGET(JUMP_IF_FALSE_OR_POP): { PyObject *cond = TOP(); int err; - if (cond == Py_True) { + if (Py_IsTrue(cond)) { STACK_SHRINK(1); Py_DECREF(cond); - FAST_DISPATCH(); + DISPATCH(); } - if (cond == Py_False) { + if (Py_IsFalse(cond)) { JUMPTO(oparg); - FAST_DISPATCH(); + DISPATCH(); } err = PyObject_IsTrue(cond); if (err > 0) { @@ -3220,14 +3821,14 @@ main_loop: case TARGET(JUMP_IF_TRUE_OR_POP): { PyObject *cond = TOP(); int err; - if (cond == Py_False) { + if (Py_IsFalse(cond)) { STACK_SHRINK(1); Py_DECREF(cond); - FAST_DISPATCH(); + DISPATCH(); } - if (cond == Py_True) { + if (Py_IsTrue(cond)) { JUMPTO(oparg); - FAST_DISPATCH(); + DISPATCH(); } err = PyObject_IsTrue(cond); if (err > 0) { @@ -3245,18 +3846,110 @@ main_loop: case TARGET(JUMP_ABSOLUTE): { PREDICTED(JUMP_ABSOLUTE); JUMPTO(oparg); -#if FAST_LOOPS - /* Enabling this path speeds-up all while and for-loops by bypassing - the per-loop checks for signals. By default, this should be turned-off - because it prevents detection of a control-break in tight loops like - "while 1: pass". Compile with this option turned-on when you need - the speed-up and do not need break checking inside tight loops (ones - that contain only instructions ending with FAST_DISPATCH). - */ - FAST_DISPATCH(); -#else + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(GET_LEN): { + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(TOP()); + if (len_i < 0) { + goto error; + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + goto error; + } + PUSH(len_o); + DISPATCH(); + } + + case TARGET(MATCH_CLASS): { + // Pop TOS. On success, set TOS to True and TOS1 to a tuple of + // attributes. On failure, set TOS to False. + PyObject *names = POP(); + PyObject *type = TOP(); + PyObject *subject = SECOND(); + assert(PyTuple_CheckExact(names)); + PyObject *attrs = match_class(tstate, subject, type, oparg, names); + Py_DECREF(names); + if (attrs) { + // Success! + assert(PyTuple_CheckExact(attrs)); + Py_DECREF(subject); + SET_SECOND(attrs); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + Py_DECREF(type); + SET_TOP(PyBool_FromLong(!!attrs)); + DISPATCH(); + } + + case TARGET(MATCH_MAPPING): { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + DISPATCH(); + } + + case TARGET(MATCH_SEQUENCE): { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + DISPATCH(); + } + + case TARGET(MATCH_KEYS): { + // On successful match for all keys, PUSH(values) and PUSH(True). + // Otherwise, PUSH(None) and PUSH(False). + PyObject *keys = TOP(); + PyObject *subject = SECOND(); + PyObject *values_or_none = match_keys(tstate, subject, keys); + if (values_or_none == NULL) { + goto error; + } + PUSH(values_or_none); + if (Py_IsNone(values_or_none)) { + Py_INCREF(Py_False); + PUSH(Py_False); + DISPATCH(); + } + assert(PyTuple_CheckExact(values_or_none)); + Py_INCREF(Py_True); + PUSH(Py_True); + DISPATCH(); + } + + case TARGET(COPY_DICT_WITHOUT_KEYS): { + // rest = dict(TOS1) + // for key in TOS: + // del rest[key] + // SET_TOP(rest) + PyObject *keys = TOP(); + PyObject *subject = SECOND(); + PyObject *rest = PyDict_New(); + if (rest == NULL || PyDict_Update(rest, subject)) { + Py_XDECREF(rest); + goto error; + } + // This may seem a bit inefficient, but keys is rarely big enough to + // actually impact runtime. + assert(PyTuple_CheckExact(keys)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(keys); i++) { + if (PyDict_DelItem(rest, PyTuple_GET_ITEM(keys, i))) { + Py_DECREF(rest); + goto error; + } + } + Py_DECREF(keys); + SET_TOP(rest); DISPATCH(); -#endif } case TARGET(GET_ITER): { @@ -3317,7 +4010,7 @@ main_loop: goto error; } else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f); + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info); } _PyErr_Clear(tstate); } @@ -3325,7 +4018,6 @@ main_loop: STACK_SHRINK(1); Py_DECREF(iter); JUMPBY(oparg); - PREDICT(POP_BLOCK); DISPATCH(); } @@ -3414,7 +4106,7 @@ main_loop: exc = TOP(); val = SECOND(); tb = THIRD(); - assert(exc != Py_None); + assert(!Py_IsNone(exc)); assert(!PyLong_Check(exc)); exit_func = PEEK(7); PyObject *stack[4] = {NULL, exc, val, tb}; @@ -3486,7 +4178,7 @@ main_loop: `callable` will be POPed by call_function. NULL will will be POPed manually later. */ - res = call_function(tstate, &sp, oparg, NULL); + res = call_function(tstate, &trace_info, &sp, oparg, NULL); stack_pointer = sp; (void)POP(); /* POP the NULL. */ } @@ -3503,13 +4195,14 @@ main_loop: We'll be passing `oparg + 1` to call_function, to make it accept the `self` as a first argument. */ - res = call_function(tstate, &sp, oparg + 1, NULL); + res = call_function(tstate, &trace_info, &sp, oparg + 1, NULL); stack_pointer = sp; } PUSH(res); if (res == NULL) goto error; + CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3517,12 +4210,13 @@ main_loop: PREDICTED(CALL_FUNCTION); PyObject **sp, *res; sp = stack_pointer; - res = call_function(tstate, &sp, oparg, NULL); + res = call_function(tstate, &trace_info, &sp, oparg, NULL); stack_pointer = sp; PUSH(res); if (res == NULL) { goto error; } + CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3534,7 +4228,7 @@ main_loop: assert(PyTuple_GET_SIZE(names) <= oparg); /* We assume without checking that names contains only strings */ sp = stack_pointer; - res = call_function(tstate, &sp, oparg, names); + res = call_function(tstate, &trace_info, &sp, oparg, names); stack_pointer = sp; PUSH(res); Py_DECREF(names); @@ -3542,6 +4236,7 @@ main_loop: if (res == NULL) { goto error; } + CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3579,7 +4274,7 @@ main_loop: } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs); + result = do_call_core(tstate, &trace_info, func, callargs, kwargs); Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); @@ -3588,6 +4283,7 @@ main_loop: if (result == NULL) { goto error; } + CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3605,10 +4301,10 @@ main_loop: if (oparg & 0x08) { assert(PyTuple_CheckExact(TOP())); - func ->func_closure = POP(); + func->func_closure = POP(); } if (oparg & 0x04) { - assert(PyDict_CheckExact(TOP())); + assert(PyTuple_CheckExact(TOP())); func->func_annotations = POP(); } if (oparg & 0x02) { @@ -3702,6 +4398,14 @@ main_loop: DISPATCH(); } + case TARGET(ROT_N): { + PyObject *top = TOP(); + memmove(&PEEK(oparg - 1), &PEEK(oparg), + sizeof(PyObject*) * (oparg - 1)); + PEEK(oparg) = top; + DISPATCH(); + } + case TARGET(EXTENDED_ARG): { int oldoparg = oparg; NEXTOPARG(); @@ -3741,11 +4445,15 @@ error: /* Log traceback info. */ PyTraceBack_Here(f); - if (tstate->c_tracefunc != NULL) + if (tstate->c_tracefunc != NULL) { + /* Make sure state is set to FRAME_EXECUTING for tracing */ + assert(f->f_state == FRAME_EXECUTING); + f->f_state = FRAME_UNWINDING; call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, - tstate, f); - + tstate, f, &trace_info); + } exception_unwind: + f->f_state = FRAME_UNWINDING; /* Unwind stacks if an exception occurred */ while (f->f_iblock > 0) { /* Pop the current block. */ @@ -3761,7 +4469,7 @@ exception_unwind: int handler = b->b_handler; _PyErr_StackItem *exc_info = tstate->exc_info; /* Beware, this invalidates all b->b_* fields */ - PyFrame_BlockSetup(f, EXCEPT_HANDLER, -1, STACK_LEVEL()); + PyFrame_BlockSetup(f, EXCEPT_HANDLER, f->f_lasti, STACK_LEVEL()); PUSH(exc_info->exc_traceback); PUSH(exc_info->exc_value); if (exc_info->exc_type != NULL) { @@ -3793,17 +4501,8 @@ exception_unwind: PUSH(val); PUSH(exc); JUMPTO(handler); - if (_Py_TracingPossible(ceval2)) { - int needs_new_execution_window = (f->f_lasti < instr_lb || f->f_lasti >= instr_ub); - int needs_line_update = (f->f_lasti == instr_lb || f->f_lasti < instr_prev); - /* Make sure that we trace line after exception if we are in a new execution - * window or we don't need a line update and we are not in the first instruction - * of the line. */ - if (needs_new_execution_window || (!needs_line_update && instr_lb > 0)) { - instr_prev = INT_MAX; - } - } /* Resume normal execution */ + f->f_state = FRAME_EXECUTING; goto main_loop; } } /* unwind stack */ @@ -3820,18 +4519,19 @@ exception_unwind: PyObject *o = POP(); Py_XDECREF(o); } - + f->f_stackdepth = 0; + f->f_state = FRAME_RAISED; exiting: - if (tstate->use_tracing) { + if (trace_info.cframe.use_tracing) { if (tstate->c_tracefunc) { if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, - tstate, f, PyTrace_RETURN, retval)) { + tstate, f, &trace_info, PyTrace_RETURN, retval)) { Py_CLEAR(retval); } } if (tstate->c_profilefunc) { if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, - tstate, f, PyTrace_RETURN, retval)) { + tstate, f, &trace_info, PyTrace_RETURN, retval)) { Py_CLEAR(retval); } } @@ -3839,10 +4539,13 @@ exiting: /* pop frame */ exit_eval_frame: + /* Restore previous cframe */ + tstate->cframe = trace_info.cframe.previous; + tstate->cframe->use_tracing = trace_info.cframe.use_tracing; + if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(f); _Py_LeaveRecursiveCall(tstate); - f->f_executing = 0; tstate->frame = f->f_back; return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); @@ -3850,7 +4553,7 @@ exit_eval_frame: static void format_missing(PyThreadState *tstate, const char *kind, - PyCodeObject *co, PyObject *names) + PyCodeObject *co, PyObject *names, PyObject *qualname) { int err; Py_ssize_t len = PyList_GET_SIZE(names); @@ -3903,7 +4606,7 @@ format_missing(PyThreadState *tstate, const char *kind, return; _PyErr_Format(tstate, PyExc_TypeError, "%U() missing %i required %s argument%s: %U", - co->co_name, + qualname, len, kind, len == 1 ? "" : "s", @@ -3914,7 +4617,7 @@ format_missing(PyThreadState *tstate, const char *kind, static void missing_arguments(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount, - PyObject **fastlocals) + PyObject **fastlocals, PyObject *qualname) { Py_ssize_t i, j = 0; Py_ssize_t start, end; @@ -3946,14 +4649,14 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, } } assert(j == missing); - format_missing(tstate, kind, co, missing_names); + format_missing(tstate, kind, co, missing_names, qualname); Py_DECREF(missing_names); } static void too_many_positional(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t given, Py_ssize_t defcount, - PyObject **fastlocals) + Py_ssize_t given, PyObject *defaults, + PyObject **fastlocals, PyObject *qualname) { int plural; Py_ssize_t kwonly_given = 0; @@ -3968,6 +4671,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, kwonly_given++; } } + Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults); if (defcount) { Py_ssize_t atleast = co_argcount - defcount; plural = 1; @@ -3997,7 +4701,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, } _PyErr_Format(tstate, PyExc_TypeError, "%U() takes %U positional argument%s but %zd%U %s given", - co->co_name, + qualname, sig, plural ? "s" : "", given, @@ -4009,7 +4713,8 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, static int positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t kwcount, PyObject* const* kwnames) + Py_ssize_t kwcount, PyObject* kwnames, + PyObject *qualname) { int posonly_conflicts = 0; PyObject* posonly_names = PyList_New(0); @@ -4019,7 +4724,7 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, for (int k2=0; k2<kwcount; k2++){ /* Compare the pointers first and fallback to PyObject_RichCompareBool*/ - PyObject* kwname = kwnames[k2]; + PyObject* kwname = PyTuple_GET_ITEM(kwnames, k2); if (kwname == posonly_name){ if(PyList_Append(posonly_names, kwname) != 0) { goto fail; @@ -4054,7 +4759,7 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, _PyErr_Format(tstate, PyExc_TypeError, "%U() got some positional-only arguments passed" " as keyword arguments: '%U'", - co->co_name, error_names); + qualname, error_names); Py_DECREF(error_names); goto fail; } @@ -4068,46 +4773,30 @@ fail: } -/* This is gonna seem *real weird*, but if you put some other code between - PyEval_EvalFrame() and _PyEval_EvalFrameDefault() you will need to adjust - the test in the if statements in Misc/gdbinit (pystack and pystackv). */ -PyObject * -_PyEval_EvalCode(PyThreadState *tstate, - PyObject *_co, PyObject *globals, PyObject *locals, +PyFrameObject * +_PyEval_MakeFrameVector(PyThreadState *tstate, + PyFrameConstructor *con, PyObject *locals, PyObject *const *args, Py_ssize_t argcount, - PyObject *const *kwnames, PyObject *const *kwargs, - Py_ssize_t kwcount, int kwstep, - PyObject *const *defs, Py_ssize_t defcount, - PyObject *kwdefs, PyObject *closure, - PyObject *name, PyObject *qualname) + PyObject *kwnames) { assert(is_tstate_valid(tstate)); - PyCodeObject* co = (PyCodeObject*)_co; - PyFrameObject *f; - PyObject *retval = NULL; - PyObject **fastlocals, **freevars; - PyObject *x, *u; + PyCodeObject *co = (PyCodeObject*)con->fc_code; + assert(con->fc_defaults == NULL || PyTuple_CheckExact(con->fc_defaults)); const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; - Py_ssize_t i, j, n; - PyObject *kwdict; - - if (globals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "PyEval_EvalCodeEx: NULL globals"); - return NULL; - } /* Create the frame */ - f = _PyFrame_New_NoTrack(tstate, co, globals, locals); + PyFrameObject *f = _PyFrame_New_NoTrack(tstate, con, locals); if (f == NULL) { return NULL; } - fastlocals = f->f_localsplus; - freevars = f->f_localsplus + co->co_nlocals; + PyObject **fastlocals = f->f_localsplus; + PyObject **freevars = f->f_localsplus + co->co_nlocals; /* Create a dictionary for keyword parameters (**kwags) */ + PyObject *kwdict; + Py_ssize_t i; if (co->co_flags & CO_VARKEYWORDS) { kwdict = PyDict_New(); if (kwdict == NULL) @@ -4123,6 +4812,7 @@ _PyEval_EvalCode(PyThreadState *tstate, } /* Copy all positional arguments into local variables */ + Py_ssize_t j, n; if (argcount > co->co_argcount) { n = co->co_argcount; } @@ -4130,97 +4820,102 @@ _PyEval_EvalCode(PyThreadState *tstate, n = argcount; } for (j = 0; j < n; j++) { - x = args[j]; + PyObject *x = args[j]; Py_INCREF(x); SETLOCAL(j, x); } /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { - u = _PyTuple_FromArray(args + n, argcount - n); + PyObject *u = _PyTuple_FromArray(args + n, argcount - n); if (u == NULL) { goto fail; } SETLOCAL(total_args, u); } - /* Handle keyword arguments passed as two strided arrays */ - kwcount *= kwstep; - for (i = 0; i < kwcount; i += kwstep) { - PyObject **co_varnames; - PyObject *keyword = kwnames[i]; - PyObject *value = kwargs[i]; - Py_ssize_t j; + /* Handle keyword arguments */ + if (kwnames != NULL) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < kwcount; i++) { + PyObject **co_varnames; + PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = args[i+argcount]; + Py_ssize_t j; - if (keyword == NULL || !PyUnicode_Check(keyword)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() keywords must be strings", - co->co_name); - goto fail; - } - - /* Speed hack: do raw pointer compares. As names are - normally interned this should almost always hit. */ - co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *name = co_varnames[j]; - if (name == keyword) { - goto kw_found; + if (keyword == NULL || !PyUnicode_Check(keyword)) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() keywords must be strings", + con->fc_qualname); + goto fail; } - } - /* Slow fallback, just in case */ - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *name = co_varnames[j]; - int cmp = PyObject_RichCompareBool( keyword, name, Py_EQ); - if (cmp > 0) { - goto kw_found; + /* Speed hack: do raw pointer compares. As names are + normally interned this should almost always hit. */ + co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + if (varname == keyword) { + goto kw_found; + } } - else if (cmp < 0) { - goto fail; + + /* Slow fallback, just in case */ + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ); + if (cmp > 0) { + goto kw_found; + } + else if (cmp < 0) { + goto fail; + } } - } - assert(j >= total_args); - if (kwdict == NULL) { + assert(j >= total_args); + if (kwdict == NULL) { - if (co->co_posonlyargcount - && positional_only_passed_as_keyword(tstate, co, - kwcount, kwnames)) - { + if (co->co_posonlyargcount + && positional_only_passed_as_keyword(tstate, co, + kwcount, kwnames, + con->fc_qualname)) + { + goto fail; + } + + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'", + con->fc_qualname, keyword); goto fail; } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'", - co->co_name, keyword); - goto fail; - } - - if (PyDict_SetItem(kwdict, keyword, value) == -1) { - goto fail; - } - continue; + if (PyDict_SetItem(kwdict, keyword, value) == -1) { + goto fail; + } + continue; - kw_found: - if (GETLOCAL(j) != NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got multiple values for argument '%S'", - co->co_name, keyword); - goto fail; + kw_found: + if (GETLOCAL(j) != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got multiple values for argument '%S'", + con->fc_qualname, keyword); + goto fail; + } + Py_INCREF(value); + SETLOCAL(j, value); } - Py_INCREF(value); - SETLOCAL(j, value); } /* Check the number of positional arguments */ if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { - too_many_positional(tstate, co, argcount, defcount, fastlocals); + too_many_positional(tstate, co, argcount, con->fc_defaults, fastlocals, + con->fc_qualname); goto fail; } /* Add missing positional arguments (copy default values from defs) */ if (argcount < co->co_argcount) { + Py_ssize_t defcount = con->fc_defaults == NULL ? 0 : PyTuple_GET_SIZE(con->fc_defaults); Py_ssize_t m = co->co_argcount - defcount; Py_ssize_t missing = 0; for (i = argcount; i < m; i++) { @@ -4229,18 +4924,22 @@ _PyEval_EvalCode(PyThreadState *tstate, } } if (missing) { - missing_arguments(tstate, co, missing, defcount, fastlocals); + missing_arguments(tstate, co, missing, defcount, fastlocals, + con->fc_qualname); goto fail; } if (n > m) i = n - m; else i = 0; - for (; i < defcount; i++) { - if (GETLOCAL(m+i) == NULL) { - PyObject *def = defs[i]; - Py_INCREF(def); - SETLOCAL(m+i, def); + if (defcount) { + PyObject **defs = &PyTuple_GET_ITEM(con->fc_defaults, 0); + for (; i < defcount; i++) { + if (GETLOCAL(m+i) == NULL) { + PyObject *def = defs[i]; + Py_INCREF(def); + SETLOCAL(m+i, def); + } } } } @@ -4249,12 +4948,11 @@ _PyEval_EvalCode(PyThreadState *tstate, if (co->co_kwonlyargcount > 0) { Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { - PyObject *name; if (GETLOCAL(i) != NULL) continue; - name = PyTuple_GET_ITEM(co->co_varnames, i); - if (kwdefs != NULL) { - PyObject *def = PyDict_GetItemWithError(kwdefs, name); + PyObject *varname = PyTuple_GET_ITEM(co->co_varnames, i); + if (con->fc_kwdefaults != NULL) { + PyObject *def = PyDict_GetItemWithError(con->fc_kwdefaults, varname); if (def) { Py_INCREF(def); SETLOCAL(i, def); @@ -4267,7 +4965,8 @@ _PyEval_EvalCode(PyThreadState *tstate, missing++; } if (missing) { - missing_arguments(tstate, co, missing, -1, fastlocals); + missing_arguments(tstate, co, missing, -1, fastlocals, + con->fc_qualname); goto fail; } } @@ -4294,41 +4993,76 @@ _PyEval_EvalCode(PyThreadState *tstate, /* Copy closure variables to free variables */ for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); + PyObject *o = PyTuple_GET_ITEM(con->fc_closure, i); Py_INCREF(o); freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } - /* Handle generator/coroutine/asynchronous generator */ - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { - PyObject *gen; - int is_coro = co->co_flags & CO_COROUTINE; + return f; - /* Don't need to keep the reference to f_back, it will be set - * when the generator is resumed. */ - Py_CLEAR(f->f_back); - - /* Create a new generator that owns the ready to run frame - * and return that as the value. */ - if (is_coro) { - gen = PyCoro_New(f, name, qualname); - } else if (co->co_flags & CO_ASYNC_GENERATOR) { - gen = PyAsyncGen_New(f, name, qualname); - } else { - gen = PyGen_NewWithQualName(f, name, qualname); - } - if (gen == NULL) { - return NULL; - } +fail: /* Jump here from prelude on failure */ + /* decref'ing the frame can cause __del__ methods to get invoked, + which can call back into Python. While we're done with the + current Python frame (f), the associated C stack is still in use, + so recursion_depth must be boosted for the duration. + */ + if (Py_REFCNT(f) > 1) { + Py_DECREF(f); _PyObject_GC_TRACK(f); + } + else { + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + } + return NULL; +} - return gen; +static PyObject * +make_coro(PyFrameConstructor *con, PyFrameObject *f) +{ + assert (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)); + PyObject *gen; + int is_coro = ((PyCodeObject *)con->fc_code)->co_flags & CO_COROUTINE; + + /* Don't need to keep the reference to f_back, it will be set + * when the generator is resumed. */ + Py_CLEAR(f->f_back); + + /* Create a new generator that owns the ready to run frame + * and return that as the value. */ + if (is_coro) { + gen = PyCoro_New(f, con->fc_name, con->fc_qualname); + } else if (((PyCodeObject *)con->fc_code)->co_flags & CO_ASYNC_GENERATOR) { + gen = PyAsyncGen_New(f, con->fc_name, con->fc_qualname); + } else { + gen = PyGen_NewWithQualName(f, con->fc_name, con->fc_qualname); + } + if (gen == NULL) { + return NULL; } - retval = _PyEval_EvalFrame(tstate, f, 0); + _PyObject_GC_TRACK(f); -fail: /* Jump here from prelude on failure */ + return gen; +} + +PyObject * +_PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames) +{ + PyFrameObject *f = _PyEval_MakeFrameVector( + tstate, con, locals, args, argcount, kwnames); + if (f == NULL) { + return NULL; + } + if (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { + return make_coro(con, f); + } + PyObject *retval = _PyEval_EvalFrame(tstate, f, 0); /* decref'ing the frame can cause __del__ methods to get invoked, which can call back into Python. While we're done with the @@ -4347,26 +5081,7 @@ fail: /* Jump here from prelude on failure */ return retval; } - -PyObject * -_PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, - PyObject *const *args, Py_ssize_t argcount, - PyObject *const *kwnames, PyObject *const *kwargs, - Py_ssize_t kwcount, int kwstep, - PyObject *const *defs, Py_ssize_t defcount, - PyObject *kwdefs, PyObject *closure, - PyObject *name, PyObject *qualname) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_EvalCode(tstate, _co, globals, locals, - args, argcount, - kwnames, kwargs, - kwcount, kwstep, - defs, defcount, - kwdefs, closure, - name, qualname); -} - +/* Legacy API */ PyObject * PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, @@ -4374,15 +5089,70 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure) { - return _PyEval_EvalCodeWithName(_co, globals, locals, - args, argcount, - kws, kws != NULL ? kws + 1 : NULL, - kwcount, 2, - defs, defcount, - kwdefs, closure, - NULL, NULL); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *res = NULL; + PyObject *defaults = _PyTuple_FromArray(defs, defcount); + if (defaults == NULL) { + return NULL; + } + PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + if (builtins == NULL) { + Py_DECREF(defaults); + return NULL; + } + if (locals == NULL) { + locals = globals; + } + PyObject *kwnames = NULL; + PyObject *const *allargs; + PyObject **newargs = NULL; + if (kwcount == 0) { + allargs = args; + } + else { + kwnames = PyTuple_New(kwcount); + if (kwnames == NULL) { + goto fail; + } + newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount)); + if (newargs == NULL) { + goto fail; + } + for (int i = 0; i < argcount; i++) { + newargs[i] = args[i]; + } + for (int i = 0; i < kwcount; i++) { + Py_INCREF(kws[2*i]); + PyTuple_SET_ITEM(kwnames, i, kws[2*i]); + newargs[argcount+i] = kws[2*i+1]; + } + allargs = newargs; + } + for (int i = 0; i < kwcount; i++) { + Py_INCREF(kws[2*i]); + PyTuple_SET_ITEM(kwnames, i, kws[2*i]); + } + PyFrameConstructor constr = { + .fc_globals = globals, + .fc_builtins = builtins, + .fc_name = ((PyCodeObject *)_co)->co_name, + .fc_qualname = ((PyCodeObject *)_co)->co_name, + .fc_code = _co, + .fc_defaults = defaults, + .fc_kwdefaults = kwdefs, + .fc_closure = closure + }; + res = _PyEval_Vector(tstate, &constr, locals, + allargs, argcount, + kwnames); +fail: + Py_XDECREF(kwnames); + PyMem_Free(newargs); + Py_DECREF(defaults); + return res; } + static PyObject * special_lookup(PyThreadState *tstate, PyObject *o, _Py_Identifier *id) { @@ -4410,7 +5180,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) type = exc_info->exc_type; value = exc_info->exc_value; tb = exc_info->exc_traceback; - if (type == Py_None || type == NULL) { + if (Py_IsNone(type) || type == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "No active exception to reraise"); return 0; @@ -4468,7 +5238,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) else if (PyExceptionInstance_Check(cause)) { fixed_cause = cause; } - else if (cause == Py_None) { + else if (Py_IsNone(cause)) { Py_DECREF(cause); fixed_cause = NULL; } @@ -4594,7 +5364,6 @@ Error: return 0; } - #ifdef LLTRACE static int prtrace(PyThreadState *tstate, PyObject *v, const char *str) @@ -4614,7 +5383,9 @@ prtrace(PyThreadState *tstate, PyObject *v, const char *str) static void call_exc_trace(Py_tracefunc func, PyObject *self, - PyThreadState *tstate, PyFrameObject *f) + PyThreadState *tstate, + PyFrameObject *f, + PyTraceInfo *trace_info) { PyObject *type, *value, *traceback, *orig_traceback, *arg; int err; @@ -4630,7 +5401,7 @@ call_exc_trace(Py_tracefunc func, PyObject *self, _PyErr_Restore(tstate, type, value, orig_traceback); return; } - err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); + err = call_trace(func, self, tstate, f, trace_info, PyTrace_EXCEPTION, arg); Py_DECREF(arg); if (err == 0) { _PyErr_Restore(tstate, type, value, orig_traceback); @@ -4645,12 +5416,13 @@ call_exc_trace(Py_tracefunc func, PyObject *self, static int call_trace_protected(Py_tracefunc func, PyObject *obj, PyThreadState *tstate, PyFrameObject *frame, + PyTraceInfo *trace_info, int what, PyObject *arg) { PyObject *type, *value, *traceback; int err; _PyErr_Fetch(tstate, &type, &value, &traceback); - err = call_trace(func, obj, tstate, frame, what, arg); + err = call_trace(func, obj, tstate, frame, trace_info, what, arg); if (err == 0) { _PyErr_Restore(tstate, type, value, traceback); @@ -4664,18 +5436,36 @@ call_trace_protected(Py_tracefunc func, PyObject *obj, } } +static void +initialize_trace_info(PyTraceInfo *trace_info, PyFrameObject *frame) +{ + if (trace_info->code != frame->f_code) { + trace_info->code = frame->f_code; + _PyCode_InitAddressRange(frame->f_code, &trace_info->bounds); + } +} + static int call_trace(Py_tracefunc func, PyObject *obj, PyThreadState *tstate, PyFrameObject *frame, + PyTraceInfo *trace_info, int what, PyObject *arg) { int result; if (tstate->tracing) return 0; tstate->tracing++; - tstate->use_tracing = 0; + tstate->cframe->use_tracing = 0; + if (frame->f_lasti < 0) { + frame->f_lineno = frame->f_code->co_firstlineno; + } + else { + initialize_trace_info(trace_info, frame); + frame->f_lineno = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds); + } result = func(obj, frame, what, arg); - tstate->use_tracing = ((tstate->c_tracefunc != NULL) + frame->f_lineno = 0; + tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL)); tstate->tracing--; return result; @@ -4686,15 +5476,15 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) { PyThreadState *tstate = _PyThreadState_GET(); int save_tracing = tstate->tracing; - int save_use_tracing = tstate->use_tracing; + int save_use_tracing = tstate->cframe->use_tracing; PyObject *result; tstate->tracing = 0; - tstate->use_tracing = ((tstate->c_tracefunc != NULL) + tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL)); result = PyObject_Call(func, args, NULL); tstate->tracing = save_tracing; - tstate->use_tracing = save_use_tracing; + tstate->cframe->use_tracing = save_use_tracing; return result; } @@ -4702,36 +5492,27 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) static int maybe_call_line_trace(Py_tracefunc func, PyObject *obj, PyThreadState *tstate, PyFrameObject *frame, - int *instr_lb, int *instr_ub, int *instr_prev) + PyTraceInfo *trace_info, int instr_prev) { int result = 0; - int line = frame->f_lineno; - /* If the last instruction executed isn't in the current - instruction window, reset the window. - */ - if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) { - PyAddrPair bounds; - line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, - &bounds); - *instr_lb = bounds.ap_lower; - *instr_ub = bounds.ap_upper; - } /* If the last instruction falls at the start of a line or if it represents a jump backwards, update the frame's line number and then call the trace function if we're tracing source lines. */ - if ((frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev)) { - frame->f_lineno = line; - if (frame->f_trace_lines) { - result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); + initialize_trace_info(trace_info, frame); + int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds); + int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds); + if (line != -1 && frame->f_trace_lines) { + /* Trace backward edges or if line number has changed */ + if (frame->f_lasti < instr_prev || line != lastline) { + result = call_trace(func, obj, tstate, frame, trace_info, PyTrace_LINE, Py_None); } } /* Always emit an opcode event if we're tracing all opcodes. */ if (frame->f_trace_opcodes) { - result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None); + result = call_trace(func, obj, tstate, frame, trace_info, PyTrace_OPCODE, Py_None); } - *instr_prev = frame->f_lasti; return result; } @@ -4754,7 +5535,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_profilefunc = NULL; tstate->c_profileobj = NULL; /* Must make sure that tracing is not ignored if 'profileobj' is freed */ - tstate->use_tracing = tstate->c_tracefunc != NULL; + tstate->cframe->use_tracing = tstate->c_tracefunc != NULL; Py_XDECREF(profileobj); Py_XINCREF(arg); @@ -4762,7 +5543,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_profilefunc = func; /* Flag that tracing or profiling is turned on */ - tstate->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL); + tstate->cframe->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL); return 0; } @@ -4790,14 +5571,12 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - struct _ceval_state *ceval2 = &tstate->interp->ceval; PyObject *traceobj = tstate->c_traceobj; - ceval2->tracing_possible += (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = NULL; tstate->c_traceobj = NULL; /* Must make sure that profiling is not ignored if 'traceobj' is freed */ - tstate->use_tracing = (tstate->c_profilefunc != NULL); + tstate->cframe->use_tracing = (tstate->c_profilefunc != NULL); Py_XDECREF(traceobj); Py_XINCREF(arg); @@ -4805,7 +5584,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->c_tracefunc = func; /* Flag that tracing or profiling is turned on */ - tstate->use_tracing = ((func != NULL) + tstate->cframe->use_tracing = ((func != NULL) || (tstate->c_profilefunc != NULL)); return 0; @@ -4886,14 +5665,20 @@ PyEval_GetFrame(void) } PyObject * +_PyEval_GetBuiltins(PyThreadState *tstate) +{ + PyFrameObject *frame = tstate->frame; + if (frame != NULL) { + return frame->f_builtins; + } + return tstate->interp->builtins; +} + +PyObject * PyEval_GetBuiltins(void) { PyThreadState *tstate = _PyThreadState_GET(); - PyFrameObject *current_frame = tstate->frame; - if (current_frame == NULL) - return tstate->interp->builtins; - else - return current_frame->f_builtins; + return _PyEval_GetBuiltins(tstate); } /* Convenience function to get a builtin from its name */ @@ -4994,9 +5779,9 @@ PyEval_GetFuncDesc(PyObject *func) } #define C_TRACE(x, call) \ -if (tstate->use_tracing && tstate->c_profilefunc) { \ +if (trace_info->cframe.use_tracing && tstate->c_profilefunc) { \ if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \ - tstate, tstate->frame, \ + tstate, tstate->frame, trace_info, \ PyTrace_C_CALL, func)) { \ x = NULL; \ } \ @@ -5006,13 +5791,13 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \ if (x == NULL) { \ call_trace_protected(tstate->c_profilefunc, \ tstate->c_profileobj, \ - tstate, tstate->frame, \ + tstate, tstate->frame, trace_info, \ PyTrace_C_EXCEPTION, func); \ /* XXX should pass (type, value, tb) */ \ } else { \ if (call_trace(tstate->c_profilefunc, \ tstate->c_profileobj, \ - tstate, tstate->frame, \ + tstate, tstate->frame, trace_info, \ PyTrace_C_RETURN, func)) { \ Py_DECREF(x); \ x = NULL; \ @@ -5027,6 +5812,7 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \ static PyObject * trace_call_function(PyThreadState *tstate, + PyTraceInfo *trace_info, PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) @@ -5061,7 +5847,11 @@ trace_call_function(PyThreadState *tstate, /* Issue #29227: Inline call_function() into _PyEval_EvalFrameDefault() to reduce the stack consumption. */ Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION -call_function(PyThreadState *tstate, PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames) +call_function(PyThreadState *tstate, + PyTraceInfo *trace_info, + PyObject ***pp_stack, + Py_ssize_t oparg, + PyObject *kwnames) { PyObject **pfunc = (*pp_stack) - oparg - 1; PyObject *func = *pfunc; @@ -5070,8 +5860,8 @@ call_function(PyThreadState *tstate, PyObject ***pp_stack, Py_ssize_t oparg, PyO Py_ssize_t nargs = oparg - nkwargs; PyObject **stack = (*pp_stack) - nargs - nkwargs; - if (tstate->use_tracing) { - x = trace_call_function(tstate, func, stack, nargs, kwnames); + if (trace_info->cframe.use_tracing) { + x = trace_call_function(tstate, trace_info, func, stack, nargs, kwnames); } else { x = PyObject_Vectorcall(func, stack, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); @@ -5089,7 +5879,11 @@ call_function(PyThreadState *tstate, PyObject ***pp_stack, Py_ssize_t oparg, PyO } static PyObject * -do_call_core(PyThreadState *tstate, PyObject *func, PyObject *callargs, PyObject *kwdict) +do_call_core(PyThreadState *tstate, + PyTraceInfo *trace_info, + PyObject *func, + PyObject *callargs, + PyObject *kwdict) { PyObject *result; @@ -5099,7 +5893,7 @@ do_call_core(PyThreadState *tstate, PyObject *func, PyObject *callargs, PyObject } else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) { Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - if (nargs > 0 && tstate->use_tracing) { + if (nargs > 0 && trace_info->cframe.use_tracing) { /* We need to create a temporary bound method as argument for profiling. @@ -5135,7 +5929,7 @@ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { PyThreadState *tstate = _PyThreadState_GET(); - if (v != Py_None) { + if (!Py_IsNone(v)) { Py_ssize_t x; if (_PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5459,6 +6253,23 @@ format_exc_check_arg(PyThreadState *tstate, PyObject *exc, return; _PyErr_Format(tstate, exc, format_str, obj_str); + + if (exc == PyExc_NameError) { + // Include the name in the NameError exceptions to offer suggestions later. + _Py_IDENTIFIER(name); + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + if (PyErr_GivenExceptionMatches(value, PyExc_NameError)) { + PyNameErrorObject* exc = (PyNameErrorObject*) value; + if (exc->name == NULL) { + // We do not care if this fails because we are going to restore the + // NameError anyway. + (void)_PyObject_SetAttrId(value, &PyId_name, obj); + } + } + PyErr_Restore(type, value, traceback); + } } static void @@ -5626,7 +6437,7 @@ dtrace_function_entry(PyFrameObject *f) PyCodeObject *code = f->f_code; filename = PyUnicode_AsUTF8(code->co_filename); funcname = PyUnicode_AsUTF8(code->co_name); - lineno = PyCode_Addr2Line(code, f->f_lasti); + lineno = PyFrame_GetLineNumber(f); PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); } @@ -5641,7 +6452,7 @@ dtrace_function_return(PyFrameObject *f) PyCodeObject *code = f->f_code; filename = PyUnicode_AsUTF8(code->co_filename); funcname = PyUnicode_AsUTF8(code->co_name); - lineno = PyCode_Addr2Line(code, f->f_lasti); + lineno = PyFrame_GetLineNumber(f); PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); } @@ -5649,35 +6460,30 @@ dtrace_function_return(PyFrameObject *f) /* DTrace equivalent of maybe_call_line_trace. */ static void maybe_dtrace_line(PyFrameObject *frame, - int *instr_lb, int *instr_ub, int *instr_prev) + PyTraceInfo *trace_info, int instr_prev) { - int line = frame->f_lineno; const char *co_filename, *co_name; /* If the last instruction executed isn't in the current instruction window, reset the window. */ - if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) { - PyAddrPair bounds; - line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, - &bounds); - *instr_lb = bounds.ap_lower; - *instr_ub = bounds.ap_upper; - } + initialize_trace_info(trace_info, frame); + int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds); /* If the last instruction falls at the start of a line or if it represents a jump backwards, update the frame's line number and call the trace function. */ - if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) { - frame->f_lineno = line; - co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); - if (!co_filename) - co_filename = "?"; - co_name = PyUnicode_AsUTF8(frame->f_code->co_name); - if (!co_name) - co_name = "?"; - PyDTrace_LINE(co_filename, co_name, line); - } - *instr_prev = frame->f_lasti; + if (line != frame->f_lineno || frame->f_lasti < instr_prev) { + if (line != -1) { + frame->f_lineno = line; + co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); + if (!co_filename) + co_filename = "?"; + co_name = PyUnicode_AsUTF8(frame->f_code->co_name); + if (!co_name) + co_name = "?"; + PyDTrace_LINE(co_filename, co_name, line); + } + } } |