diff options
Diffstat (limited to 'contrib/tools/python3/Python/generated_cases.c.h')
| -rw-r--r-- | contrib/tools/python3/Python/generated_cases.c.h | 9406 |
1 files changed, 5440 insertions, 3966 deletions
diff --git a/contrib/tools/python3/Python/generated_cases.c.h b/contrib/tools/python3/Python/generated_cases.c.h index 246d37e771e..ad49becdfca 100644 --- a/contrib/tools/python3/Python/generated_cases.c.h +++ b/contrib/tools/python3/Python/generated_cases.c.h @@ -1,1043 +1,3038 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: // Python/bytecodes.c // Do not edit! - TARGET(NOP) { - DISPATCH(); - } +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 - TARGET(RESUME) { - #line 137 "Python/bytecodes.c" - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); - /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(frame->f_code, tstate->interp); - if (err) goto error; - next_instr--; + + TARGET(BEFORE_ASYNC_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + goto error; + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + goto error; } - else if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; + Py_DECREF(mgr); + res = PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; } - #line 24 "Python/generated_cases.c.h" + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_RESUME) { - #line 151 "Python/bytecodes.c" - /* Possible performance enhancement: - * We need to check the eval breaker anyway, can we - * combine the instrument verison check and the eval breaker test? + TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - if (_Py_Instrument(frame->f_code, tstate->interp)) { - goto error; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } - next_instr--; + goto error; } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->prev_instr != next_instr-1) { - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - DISPATCH(); - } - if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); } + Py_DECREF(enter); + goto error; } - #line 55 "Python/generated_cases.c.h" + Py_DECREF(mgr); + res = PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(LOAD_CLOSURE) { - PyObject *value; - #line 179 "Python/bytecodes.c" - /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - #line 66 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST_CHECK) { - PyObject *value; - #line 186 "Python/bytecodes.c" - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - #line 78 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST) { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 90 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP_ADD_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST_AND_CLEAR) { - PyObject *value; - #line 198 "Python/bytecodes.c" - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - #line 102 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_UNICODE + { + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_CONST) { - PREDICTED(LOAD_CONST); - PyObject *value; - #line 204 "Python/bytecodes.c" - value = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(value); - #line 114 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + stack_pointer += -2; DISPATCH(); } - TARGET(STORE_FAST) { - PyObject *value = stack_pointer[-1]; - #line 209 "Python/bytecodes.c" - SETLOCAL(oparg, value); - #line 124 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_FLOAT + { + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; + TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 138 "Python/generated_cases.c.h" - _tmp_2 = value; + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - oparg = (next_instr++)->op.arg; + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 148 "Python/generated_cases.c.h" - _tmp_1 = value; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - STACK_GROW(2); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST__LOAD_CONST) { - PyObject *_tmp_1; - PyObject *_tmp_2; + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 166 "Python/generated_cases.c.h" - _tmp_2 = value; + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } - oparg = (next_instr++)->op.arg; + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_FLOAT { - PyObject *value; - #line 204 "Python/bytecodes.c" - value = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(value); - #line 175 "Python/generated_cases.c.h" - _tmp_1 = value; + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_GROW(2); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(STORE_FAST__LOAD_FAST) { - PyObject *_tmp_1 = stack_pointer[-1]; + TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *value = _tmp_1; - #line 209 "Python/bytecodes.c" - SETLOCAL(oparg, value); - #line 190 "Python/generated_cases.c.h" + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - oparg = (next_instr++)->op.arg; + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_INT { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 199 "Python/generated_cases.c.h" - _tmp_1 = value; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; + } + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *sub; + PyObject *container; + PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; { - PyObject *value = _tmp_1; - #line 209 "Python/bytecodes.c" - SETLOCAL(oparg, value); - #line 213 "Python/generated_cases.c.h" + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - oparg = (next_instr++)->op.arg; + // _BINARY_SUBSCR { - PyObject *value = _tmp_2; - #line 209 "Python/bytecodes.c" - SETLOCAL(oparg, value); - #line 220 "Python/generated_cases.c.h" + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *res; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *container; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(BINARY_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *res; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_CONST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - #line 204 "Python/bytecodes.c" - value = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(value); - #line 234 "Python/generated_cases.c.h" - _tmp_2 = value; + TARGET(BINARY_SUBSCR_STR_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *str; + PyObject *res; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_TUPLE_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *tuple; + PyObject *res; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); } - oparg = (next_instr++)->op.arg; - { - PyObject *value; - #line 192 "Python/bytecodes.c" - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - #line 244 "Python/generated_cases.c.h" - _tmp_1 = value; + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); } - STACK_GROW(2); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } - TARGET(POP_TOP) { - PyObject *value = stack_pointer[-1]; - #line 219 "Python/bytecodes.c" - #line 256 "Python/generated_cases.c.h" - Py_DECREF(value); - STACK_SHRINK(1); + TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + goto error; + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } + } + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(PUSH_NULL) { + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(CACHE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + DISPATCH(); + } + + TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - #line 223 "Python/bytecodes.c" - res = NULL; - #line 266 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _CALL + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(END_FOR) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, CALL); + DEOPT_IF(code->co_kwonlyargcount, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + goto error; + } + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + goto start_frame; + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *func; + PyObject *self; + PyObject *self_or_null; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 { - PyObject *value = _tmp_1; - #line 219 "Python/bytecodes.c" - #line 278 "Python/generated_cases.c.h" - Py_DECREF(value); + DEOPT_IF(tstate->interp->eval_frame, CALL); } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; { - PyObject *value = _tmp_2; - #line 219 "Python/bytecodes.c" - #line 284 "Python/generated_cases.c.h" - Py_DECREF(value); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + } + // _CHECK_RECURSION_REMAINING + { + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int has_self = (self_or_null != NULL); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - STACK_SHRINK(2); DISPATCH(); } - TARGET(INSTRUMENTED_END_FOR) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; - #line 229 "Python/bytecodes.c" - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + TARGET(CALL_BOUND_METHOD_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *method; + PyObject *self; + PyObject **args; + PyObject *self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_RECURSION_REMAINING + { + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _CHECK_METHOD_VERSION + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(null != NULL, CALL); + } + // _EXPAND_METHOD + { + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + } + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = self; + callable = method; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { goto error; } - PyErr_SetRaisedException(NULL); } - #line 304 "Python/generated_cases.c.h" - Py_DECREF(receiver); - Py_DECREF(value); - STACK_SHRINK(2); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } DISPATCH(); } - TARGET(END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; - #line 242 "Python/bytecodes.c" - Py_DECREF(receiver); - #line 316 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = value; + TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; - #line 246 "Python/bytecodes.c" - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - PyErr_SetRaisedException(NULL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - Py_DECREF(receiver); - #line 334 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = value; + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(UNARY_NEGATIVE) { - PyObject *value = stack_pointer[-1]; + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - #line 257 "Python/bytecodes.c" - res = PyNumber_Negative(value); - #line 345 "Python/generated_cases.c.h" - Py_DECREF(value); - #line 259 "Python/bytecodes.c" - if (res == NULL) goto pop_1_error; - #line 349 "Python/generated_cases.c.h" - stack_pointer[-1] = res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(UNARY_NOT) { - PyObject *value = stack_pointer[-1]; + TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - #line 263 "Python/bytecodes.c" - int err = PyObject_IsTrue(value); - #line 359 "Python/generated_cases.c.h" - Py_DECREF(value); - #line 265 "Python/bytecodes.c" - if (err < 0) goto pop_1_error; - if (err == 0) { - res = Py_True; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + goto error; + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + goto error; + } + Py_SETREF(callargs, tuple); + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) goto error; + result = PyObject_Call(func, callargs, kwargs); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } } else { - res = Py_False; + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyObject_Call(func, callargs, kwargs); } - #line 369 "Python/generated_cases.c.h" - stack_pointer[-1] = res; + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(UNARY_INVERT) { - PyObject *value = stack_pointer[-1]; + TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + PyObject *value; PyObject *res; - #line 275 "Python/bytecodes.c" - res = PyNumber_Invert(value); - #line 379 "Python/generated_cases.c.h" + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); - #line 277 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 383 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *prod; - #line 294 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - STAT_INC(BINARY_OP, hit); - prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (prod == NULL) goto pop_2_error; - #line 400 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = prod; + TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + PyObject *value1; + PyObject *value2; + PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *prod; - #line 304 "Python/bytecodes.c" - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - STAT_INC(BINARY_OP, hit); - double dprod = ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 418 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = prod; - next_instr += 1; + TARGET(CALL_ISINSTANCE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + goto error; + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *sub; - #line 313 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - STAT_INC(BINARY_OP, hit); - sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (sub == NULL) goto pop_2_error; - #line 437 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = sub; + TARGET(CALL_KW) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *sub; - #line 323 "Python/bytecodes.c" - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - STAT_INC(BINARY_OP, hit); - double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 454 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = sub; - next_instr += 1; + TARGET(CALL_LEN) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + goto error; + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(callable); + Py_DECREF(arg); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + TARGET(CALL_LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *arg; + PyObject *self; + PyObject *callable; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + DEOPT_IF(self == NULL, CALL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - #line 331 "Python/bytecodes.c" - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - #line 473 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args == 0, CALL); + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + assert(self != NULL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - #line 347 "Python/bytecodes.c" - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST || - true_next.op.code == STORE_FAST__LOAD_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 510 "Python/generated_cases.c.h" - STACK_SHRINK(2); + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args == 0, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + assert(self != NULL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *sum; - #line 376 "Python/bytecodes.c" - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); - STAT_INC(BINARY_OP, hit); - double dsum = ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 526 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = sum; - next_instr += 1; + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_OP_ADD_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *sum; - #line 385 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); - STAT_INC(BINARY_OP, hit); - sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (sum == NULL) goto pop_2_error; - #line 545 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = sum; - next_instr += 1; + TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR) { - PREDICTED(BINARY_SUBSCR); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + TARGET(CALL_NON_PY_GENERAL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; + PyObject **args; + PyObject *self_or_null; PyObject *res; - #line 403 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(PyFunction_Check(callable), CALL); + DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL); } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - res = PyObject_GetItem(container, sub); - #line 570 "Python/generated_cases.c.h" - Py_DECREF(container); - Py_DECREF(sub); - #line 415 "Python/bytecodes.c" - if (res == NULL) goto pop_2_error; - #line 575 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + // _CALL_NON_PY_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + } + // _CHECK_RECURSION_REMAINING + { + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int has_self = (self_or_null != NULL); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_PY_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; + PyObject **args; + PyObject *self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_RECURSION_REMAINING + { + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *arg; + PyObject *null; + PyObject *callable; PyObject *res; - #line 419 "Python/bytecodes.c" - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + res = PyObject_Str(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + // _CHECK_PERIODIC + { } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - #line 600 "Python/generated_cases.c.h" - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(STORE_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; - #line 434 "Python/bytecodes.c" - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *arg; + PyObject *null; + PyObject *callable; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + res = PySequence_Tuple(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _CHECK_PERIODIC + { } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - #line 624 "Python/generated_cases.c.h" - STACK_SHRINK(4); + stack_pointer[-3] = res; + stack_pointer += -2; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *arg; + PyObject *null; + PyObject *callable; PyObject *res; - #line 449 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + DISPATCH(); + } - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - #line 647 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } - TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; - PyObject *res; - #line 465 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; + } + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; + DISPATCH(); + } - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - #line 672 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; + } + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *right; + PyObject *left; PyObject *res; - #line 481 "Python/bytecodes.c" - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - #line 691 "Python/generated_cases.c.h" - Py_DECREF(dict); - Py_DECREF(sub); - #line 489 "Python/bytecodes.c" - if (true) goto pop_2_error; + STAT_INC(COMPARE_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 698 "Python/generated_cases.c.h" - Py_DECREF(dict); - Py_DECREF(sub); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + // _COMPARE_OP + { + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; + } + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - #line 496 "Python/bytecodes.c" - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - #line 733 "Python/generated_cases.c.h" + TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); } - TARGET(LIST_APPEND) { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 521 "Python/bytecodes.c" - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" - STACK_SHRINK(1); - PREDICT(JUMP_BACKWARD); + TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(SET_ADD) { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 526 "Python/bytecodes.c" - int err = PySet_Add(set, v); - #line 752 "Python/generated_cases.c.h" - Py_DECREF(v); - #line 528 "Python/bytecodes.c" - if (err) goto pop_1_error; - #line 756 "Python/generated_cases.c.h" - STACK_SHRINK(1); - PREDICT(JUMP_BACKWARD); + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(STORE_SUBSCR) { - PREDICTED(STORE_SUBSCR); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; - uint16_t counter = read_u16(&next_instr[0].cache); - #line 539 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr--; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *right; + PyObject *left; + PyObject *b; + // _SPECIALIZE_CONTAINS_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ContainsOp(right, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CONTAINS_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. - #endif /* ENABLE_SPECIALIZATION */ - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - #line 784 "Python/generated_cases.c.h" - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - #line 554 "Python/bytecodes.c" - if (err) goto pop_3_error; - #line 790 "Python/generated_cases.c.h" - STACK_SHRINK(3); + // _CONTAINS_OP + { + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONTAINS_OP_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + int res = PyDict_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONTAINS_OP_SET) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + int res = _PySet_Contains((PySetObject *)right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; DISPATCH(); } - TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; - #line 558 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + DISPATCH(); + } - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); + } + DISPATCH(); + } - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - #line 817 "Python/generated_cases.c.h" - STACK_SHRINK(3); + TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(STORE_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; - #line 577 "Python/bytecodes.c" - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - #line 833 "Python/generated_cases.c.h" - STACK_SHRINK(3); + TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = GETLOCAL(oparg); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + goto error; + } + Py_DECREF(oldobj); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + PyObject *v = GETLOCAL(oparg); + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } + SETLOCAL(oparg, NULL); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_Pop(GLOBALS(), name, NULL); + // Can't use ERROR_IF here. + if (err < 0) { + goto error; + } + if (err == 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + DISPATCH(); + } + + TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + goto error; + } + err = PyObject_DelItem(ns, name); + // Can't use ERROR_IF here. + if (err != 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + goto error; + } DISPATCH(); } TARGET(DELETE_SUBSCR) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - #line 585 "Python/bytecodes.c" + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 845 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 588 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 850 "Python/generated_cases.c.h" - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - TARGET(CALL_INTRINSIC_1) { - PyObject *value = stack_pointer[-1]; - PyObject *res; - #line 592 "Python/bytecodes.c" - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 861 "Python/generated_cases.c.h" + TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); + } + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); - #line 595 "Python/bytecodes.c" - if (res == NULL) goto pop_1_error; - #line 865 "Python/generated_cases.c.h" + stack_pointer += -1; + DISPATCH(); + } + + TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(ENTER_EXECUTOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + DISPATCH(); + } + + TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + } + else { + res = value; + } stack_pointer[-1] = res; DISPATCH(); } - TARGET(CALL_INTRINSIC_2) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; PyObject *res; - #line 599 "Python/bytecodes.c" - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 877 "Python/generated_cases.c.h" - Py_DECREF(value2); - Py_DECREF(value1); - #line 602 "Python/bytecodes.c" + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 882 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(RAISE_VARARGS) { - PyObject **args = (stack_pointer - oparg); - #line 606 "Python/bytecodes.c" - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; + TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *iter; + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + STAT_INC(FOR_ITER, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - if (true) { STACK_SHRINK(oparg); goto error; } - #line 912 "Python/generated_cases.c.h" - } - - TARGET(INTERPRETER_EXIT) { - PyObject *retval = stack_pointer[-1]; - #line 630 "Python/bytecodes.c" - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - STACK_SHRINK(1); // Since we're not going to DISPATCH() - assert(EMPTY()); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - #line 928 "Python/generated_cases.c.h" + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); + DISPATCH(); + } + // Common case: no jump, leave it to the code generator + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); } - TARGET(RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; - #line 643 "Python/bytecodes.c" - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 946 "Python/generated_cases.c.h" + TARGET(FOR_ITER_GEN) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + iter = stack_pointer[-1]; + { + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_StackPush(gen_frame, Py_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); + } + // _PUSH_FRAME + new_frame = gen_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; - #line 658 "Python/bytecodes.c" - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 968 "Python/generated_cases.c.h" + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + #endif + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); } - TARGET(RETURN_CONST) { - #line 677 "Python/bytecodes.c" - PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 986 "Python/generated_cases.c.h" + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_CONST) { - #line 693 "Python/bytecodes.c" - PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 1008 "Python/generated_cases.c.h" + TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); } TARGET(GET_AITER) { - PyObject *obj = stack_pointer[-1]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; PyObject *iter; - #line 713 "Python/bytecodes.c" + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1027 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 726 "Python/bytecodes.c" if (true) goto pop_1_error; } - iter = (*getter)(obj); - #line 1034 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 731 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -1045,19 +3040,20 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1049 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } TARGET(GET_ANEXT) { - PyObject *aiter = stack_pointer[-1]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; PyObject *awaitable; - #line 746 "Python/bytecodes.c" + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -1067,7 +3063,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -1081,7 +3076,6 @@ type->tp_name); goto error; } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -1089,36 +3083,29 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); goto error; } else { Py_DECREF(next_iter); } } - - #line 1101 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = awaitable; - PREDICT(LOAD_CONST); + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } TARGET(GET_AWAITABLE) { - PREDICTED(GET_AWAITABLE); - PyObject *iterable = stack_pointer[-1]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; PyObject *iter; - #line 793 "Python/bytecodes.c" + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - - #line 1119 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 800 "Python/bytecodes.c" - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -1132,1517 +3119,1037 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; - - #line 1139 "Python/generated_cases.c.h" stack_pointer[-1] = iter; - PREDICT(LOAD_CONST); DISPATCH(); } - TARGET(SEND) { - PREDICTED(SEND); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; - PyObject *retval; - #line 826 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - JUMPBY(INLINE_CACHE_ENTRIES_SEND); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, next_instr-1); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - goto error; - } - } - Py_DECREF(v); - #line 1198 "Python/generated_cases.c.h" - stack_pointer[-1] = retval; + TARGET(GET_ITER) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(SEND_GEN) { - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; - #line 875 "Python/bytecodes.c" - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - JUMPBY(INLINE_CACHE_ENTRIES_SEND); - DISPATCH_INLINED(gen_frame); - #line 1223 "Python/generated_cases.c.h" - } - - TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; - #line 893 "Python/bytecodes.c" - assert(frame != &entry_frame); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 1245 "Python/generated_cases.c.h" - } - - TARGET(YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; - #line 912 "Python/bytecodes.c" - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - #line 1266 "Python/generated_cases.c.h" - } - - TARGET(POP_EXCEPT) { - PyObject *exc_value = stack_pointer[-1]; - #line 930 "Python/bytecodes.c" - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - #line 1274 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + stack_pointer[0] = len_o; + stack_pointer += 1; DISPATCH(); } - TARGET(RERAISE) { - PyObject *exc = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); - #line 935 "Python/bytecodes.c" - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); goto error; } + iter = iterable; } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; - #line 1301 "Python/generated_cases.c.h" - } - - TARGET(END_ASYNC_FOR) { - PyObject *exc = stack_pointer[-1]; - PyObject *awaitable = stack_pointer[-2]; - #line 956 "Python/bytecodes.c" - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1310 "Python/generated_cases.c.h" - Py_DECREF(awaitable); - Py_DECREF(exc); - #line 959 "Python/bytecodes.c" - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; - } - #line 1321 "Python/generated_cases.c.h" - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(CLEANUP_THROW) { - PyObject *exc_value = stack_pointer[-1]; - PyObject *last_sent_val = stack_pointer[-2]; - PyObject *sub_iter = stack_pointer[-3]; - PyObject *none; - PyObject *value; - #line 969 "Python/bytecodes.c" - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1337 "Python/generated_cases.c.h" - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - #line 974 "Python/bytecodes.c" - none = Py_None; + else if (PyGen_CheckExact(iterable)) { + iter = iterable; } else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { + goto error; + } + Py_DECREF(iterable); } - #line 1349 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = value; - stack_pointer[-2] = none; - DISPATCH(); - } - - TARGET(LOAD_ASSERTION_ERROR) { - PyObject *value; - #line 984 "Python/bytecodes.c" - value = Py_NewRef(PyExc_AssertionError); - #line 1360 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(LOAD_BUILD_CLASS) { - PyObject *bc; - #line 988 "Python/bytecodes.c" - if (PyDict_CheckExact(BUILTINS())) { - bc = _PyDict_GetItemWithError(BUILTINS(), - &_Py_ID(__build_class__)); - if (bc == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - } - if (true) goto error; - } - Py_INCREF(bc); - } - else { - bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__)); - if (bc == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; - } - } - #line 1390 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = bc; + TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(STORE_NAME) { - PyObject *v = stack_pointer[-1]; - #line 1013 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - #line 1405 "Python/generated_cases.c.h" - Py_DECREF(v); - #line 1020 "Python/bytecodes.c" - if (true) goto pop_1_error; - } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - #line 1414 "Python/generated_cases.c.h" - Py_DECREF(v); - #line 1027 "Python/bytecodes.c" - if (err) goto pop_1_error; - #line 1418 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(DELETE_NAME) { - #line 1031 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - goto error; - } - err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - goto error; - } - #line 1441 "Python/generated_cases.c.h" - DISPATCH(); + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + /* Skip 3 cache entries */ + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL); } - TARGET(UNPACK_SEQUENCE) { - PREDICTED(UNPACK_SEQUENCE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; - #line 1057 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject **top = stack_pointer + oparg - 1; - int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1462 "Python/generated_cases.c.h" - Py_DECREF(seq); - #line 1070 "Python/bytecodes.c" - if (res == 0) goto pop_1_error; - #line 1466 "Python/generated_cases.c.h" - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; next_instr += 1; - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); - #line 1074 "Python/bytecodes.c" - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1483 "Python/generated_cases.c.h" - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); } - TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); - #line 1084 "Python/bytecodes.c" - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - #line 1502 "Python/generated_cases.c.h" - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + goto error; + } + } + Py_DECREF(value); + stack_pointer += -1; DISPATCH(); } - TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); - #line 1095 "Python/bytecodes.c" - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - #line 1521 "Python/generated_cases.c.h" - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + goto error; + } + } + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - TARGET(UNPACK_EX) { - PyObject *seq = stack_pointer[-1]; - #line 1106 "Python/bytecodes.c" - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1535 "Python/generated_cases.c.h" - Py_DECREF(seq); - #line 1110 "Python/bytecodes.c" - if (res == 0) goto pop_1_error; - #line 1539 "Python/generated_cases.c.h" - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(STORE_ATTR) { - PREDICTED(STORE_ATTR); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; - uint16_t counter = read_u16(&next_instr[0].cache); - #line 1121 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - next_instr--; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } - STAT_INC(STORE_ATTR, deferred); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - int err = PyObject_SetAttr(owner, name, v); - #line 1566 "Python/generated_cases.c.h" - Py_DECREF(v); - Py_DECREF(owner); - #line 1137 "Python/bytecodes.c" - if (err) goto pop_2_error; - #line 1571 "Python/generated_cases.c.h" - STACK_SHRINK(2); - next_instr += 4; - DISPATCH(); + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } - TARGET(DELETE_ATTR) { - PyObject *owner = stack_pointer[-1]; - #line 1141 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1582 "Python/generated_cases.c.h" - Py_DECREF(owner); - #line 1144 "Python/bytecodes.c" - if (err) goto pop_1_error; - #line 1586 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(STORE_GLOBAL) { - PyObject *v = stack_pointer[-1]; - #line 1148 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1596 "Python/generated_cases.c.h" - Py_DECREF(v); - #line 1151 "Python/bytecodes.c" - if (err) goto pop_1_error; - #line 1600 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(DELETE_GLOBAL) { - #line 1155 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - goto error; - } - #line 1618 "Python/generated_cases.c.h" - DISPATCH(); + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - TARGET(LOAD_LOCALS) { - PyObject *locals; - #line 1169 "Python/bytecodes.c" - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - #line 1632 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = locals; + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *mod_or_class_dict = stack_pointer[-1]; - PyObject *v; - #line 1179 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - if (PyDict_CheckExact(mod_or_class_dict)) { - v = PyDict_GetItemWithError(mod_or_class_dict, name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; - } + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; } else { - v = PyObject_GetItem(mod_or_class_dict, name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); - } - } - if (v == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - Py_DECREF(mod_or_class_dict); - if (true) goto pop_1_error; - } - Py_INCREF(v); - } - else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - v = PyObject_GetItem(GLOBALS(), name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto pop_1_error; - _PyErr_Clear(tstate); - - /* namespace 2: builtins */ - v = PyObject_GetItem(BUILTINS(), name); - if (v == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - Py_DECREF(mod_or_class_dict); - if (true) goto pop_1_error; - } - } - } + Py_DECREF(value); + offset = 0; } - #line 1703 "Python/generated_cases.c.h" - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_NAME) { - PyObject *v; - #line 1244 "Python/bytecodes.c" - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - if (PyDict_CheckExact(mod_or_class_dict)) { - v = PyDict_GetItemWithError(mod_or_class_dict, name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; - } + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; } else { - v = PyObject_GetItem(mod_or_class_dict, name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); - } - } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; - } - else { - if (PyDict_CheckExact(BUILTINS())) { - v = PyDict_GetItemWithError(BUILTINS(), name); - if (v == NULL) { - if (!_PyErr_Occurred(tstate)) { - format_exc_check_arg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - goto error; - } - Py_INCREF(v); - } - else { - v = PyObject_GetItem(BUILTINS(), name); - if (v == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - goto error; - } - } - } + Py_DECREF(value); + offset = oparg; } - #line 1771 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = v; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_GLOBAL) { - PREDICTED(LOAD_GLOBAL); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *null = NULL; - PyObject *v; - #line 1312 "Python/bytecodes.c" + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); - next_instr--; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - format_exc_check_arg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version && tstate->tracing == 0) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + goto error; } - Py_INCREF(v); + next_instr = this_instr; } else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - v = PyObject_GetItem(GLOBALS(), name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error; - _PyErr_Clear(tstate); - - /* namespace 2: builtins */ - v = PyObject_GetItem(BUILTINS(), name); - if (v == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; + DISPATCH(); } } - null = NULL; - #line 1834 "Python/generated_cases.c.h" - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 4; DISPATCH(); } - TARGET(LOAD_GLOBAL_MODULE) { - PyObject *null = NULL; - PyObject *res; - uint16_t index = read_u16(&next_instr[1].cache); - uint16_t version = read_u16(&next_instr[2].cache); - #line 1366 "Python/bytecodes.c" - DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - #line 1859 "Python/generated_cases.c.h" - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 4; - DISPATCH(); + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *null = NULL; - PyObject *res; - uint16_t index = read_u16(&next_instr[1].cache); - uint16_t mod_version = read_u16(&next_instr[2].cache); - uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1379 "Python/bytecodes.c" - DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); - DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); - PyDictObject *mdict = (PyDictObject *)GLOBALS(); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - assert(opcode == LOAD_GLOBAL_BUILTIN); - DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); - DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(bdict->ma_keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - #line 1889 "Python/generated_cases.c.h" - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 4; - DISPATCH(); + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(DELETE_FAST) { - #line 1396 "Python/bytecodes.c" - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); - #line 1903 "Python/generated_cases.c.h" - DISPATCH(); + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) goto error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; } - TARGET(MAKE_CELL) { - #line 1402 "Python/bytecodes.c" - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - goto resume_with_error; - } - SETLOCAL(oparg, cell); - #line 1917 "Python/generated_cases.c.h" - DISPATCH(); + TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; } - TARGET(DELETE_DEREF) { - #line 1413 "Python/bytecodes.c" - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - format_exc_unbound(tstate, frame->f_code, oparg); - goto error; - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - #line 1933 "Python/generated_cases.c.h" + TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_DEREF) { - PyObject *class_dict = stack_pointer[-1]; - PyObject *value; - #line 1426 "Python/bytecodes.c" - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); - name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); - if (PyDict_CheckExact(class_dict)) { - value = PyDict_GetItemWithError(class_dict, name); - if (value != NULL) { - Py_INCREF(value); - } - else if (_PyErr_Occurred(tstate)) { - goto error; + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + /* Skip 1 cache entry */ + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; } - } - else { - value = PyObject_GetItem(class_dict, name); - if (value == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); + if (optimized < 0) goto error; + if (optimized) { + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } - } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - format_exc_unbound(tstate, frame->f_code, oparg); - goto error; + else { + this_instr[1].counter = restart_backoff_counter(counter); } - Py_INCREF(value); - } - Py_DECREF(class_dict); - #line 1973 "Python/generated_cases.c.h" - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_DEREF) { - PyObject *value; - #line 1461 "Python/bytecodes.c" - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - format_exc_unbound(tstate, frame->f_code, oparg); - if (true) goto error; } - Py_INCREF(value); - #line 1988 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(STORE_DEREF) { - PyObject *v = stack_pointer[-1]; - #line 1471 "Python/bytecodes.c" - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - #line 2001 "Python/generated_cases.c.h" - STACK_SHRINK(1); - DISPATCH(); - } - - TARGET(COPY_FREE_VARS) { - #line 1478 "Python/bytecodes.c" - /* Copy closure variables to free variables */ - PyCodeObject *co = frame->f_code; - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } - #line 2018 "Python/generated_cases.c.h" + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ DISPATCH(); } - TARGET(BUILD_STRING) { - PyObject **pieces = (stack_pointer - oparg); - PyObject *str; - #line 1491 "Python/bytecodes.c" - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 2027 "Python/generated_cases.c.h" - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); - } - #line 1493 "Python/bytecodes.c" - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2033 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); } - TARGET(BUILD_TUPLE) { - PyObject **values = (stack_pointer - oparg); - PyObject *tup; - #line 1497 "Python/bytecodes.c" - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2046 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); DISPATCH(); } - TARGET(BUILD_LIST) { - PyObject **values = (stack_pointer - oparg); + TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; PyObject *list; - #line 1502 "Python/bytecodes.c" - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2059 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } TARGET(LIST_EXTEND) { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1507 "Python/bytecodes.c" + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } - #line 2080 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1518 "Python/bytecodes.c" if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - #line 2086 "Python/generated_cases.c.h" Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - TARGET(SET_UPDATE) { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1525 "Python/bytecodes.c" - int err = _PySet_Update(set, iterable); - #line 2097 "Python/generated_cases.c.h" - Py_DECREF(iterable); - #line 1527 "Python/bytecodes.c" - if (err < 0) goto pop_1_error; - #line 2101 "Python/generated_cases.c.h" - STACK_SHRINK(1); - DISPATCH(); - } - - TARGET(BUILD_SET) { - PyObject **values = (stack_pointer - oparg); - PyObject *set; - #line 1531 "Python/bytecodes.c" - set = PySet_New(NULL); - if (set == NULL) - goto error; - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } - } - #line 2124 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; - DISPATCH(); - } - - TARGET(BUILD_MAP) { - PyObject **values = (stack_pointer - oparg*2); - PyObject *map; - #line 1548 "Python/bytecodes.c" - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - #line 2139 "Python/generated_cases.c.h" - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); - } - #line 1553 "Python/bytecodes.c" - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2145 "Python/generated_cases.c.h" - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(SETUP_ANNOTATIONS) { - #line 1557 "Python/bytecodes.c" - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error; - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + (void)this_instr; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); } + STAT_INC(LOAD_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - else { - /* do the same if locals() is not a dict */ - ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error; - _PyErr_Clear(tstate); - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } } else { - Py_DECREF(ann_dict); + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; } } - #line 2193 "Python/generated_cases.c.h" + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); - PyObject *map; - #line 1599 "Python/bytecodes.c" - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - #line 2211 "Python/generated_cases.c.h" - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); } - Py_DECREF(keys); - #line 1609 "Python/bytecodes.c" - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2218 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(DICT_UPDATE) { - PyObject *update = stack_pointer[-1]; - #line 1613 "Python/bytecodes.c" - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); - } - #line 2234 "Python/generated_cases.c.h" - Py_DECREF(update); - #line 1621 "Python/bytecodes.c" - if (true) goto pop_1_error; - } - #line 2239 "Python/generated_cases.c.h" - Py_DECREF(update); + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); - DISPATCH(); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(DICT_MERGE) { - PyObject *update = stack_pointer[-1]; - #line 1627 "Python/bytecodes.c" - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - - if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2252 "Python/generated_cases.c.h" - Py_DECREF(update); - #line 1632 "Python/bytecodes.c" - if (true) goto pop_1_error; + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - #line 2257 "Python/generated_cases.c.h" - Py_DECREF(update); - STACK_SHRINK(1); - PREDICT(CALL_FUNCTION_EX); - DISPATCH(); - } - - TARGET(MAP_ADD) { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; - #line 1639 "Python/bytecodes.c" - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2273 "Python/generated_cases.c.h" - STACK_SHRINK(2); - PREDICT(JUMP_BACKWARD); + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + attr = _PyObject_InlineValues(owner)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - #line 1648 "Python/bytecodes.c" - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - #line 2286 "Python/generated_cases.c.h" - } - - TARGET(LOAD_SUPER_ATTR) { - PREDICTED(LOAD_SUPER_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; - PyObject *res2 = NULL; - PyObject *res; - #line 1662 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); - int load_method = oparg & 1; - #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); - if (err) goto pop_3_error; + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); } - - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; } - #line 2339 "Python/generated_cases.c.h" - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - #line 1704 "Python/bytecodes.c" - if (super == NULL) goto pop_3_error; - res = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (res == NULL) goto pop_3_error; - #line 2348 "Python/generated_cases.c.h" - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 1; - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_ATTR) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; - PyObject *res2 = NULL; - PyObject *res; - #line 1711 "Python/bytecodes.c" - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - #line 2370 "Python/generated_cases.c.h" - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - #line 1718 "Python/bytecodes.c" - if (res == NULL) goto pop_3_error; - #line 2376 "Python/generated_cases.c.h" - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 1; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_METHOD) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; - PyObject *res2; - PyObject *res; - #line 1722 "Python/bytecodes.c" - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (res2 == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - if (method_found) { - res = self; // transfer ownership - } else { - Py_DECREF(self); - res = res2; - res2 = NULL; + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; } - #line 2414 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; - stack_pointer[-2] = res2; - next_instr += 1; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } - TARGET(LOAD_ATTR) { - PREDICTED(LOAD_ATTR); + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - #line 1761 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); - next_instr--; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - - meth | self | arg1 | ... | argN - */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ - #line 2462 "Python/generated_cases.c.h" - Py_DECREF(owner); - #line 1795 "Python/bytecodes.c" - if (meth == NULL) goto pop_1_error; - res2 = NULL; - res = meth; - } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } - else { - /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); - #line 2473 "Python/generated_cases.c.h" - Py_DECREF(owner); - #line 1804 "Python/bytecodes.c" - if (res == NULL) goto pop_1_error; + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } - #line 2478 "Python/generated_cases.c.h" - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; - DISPATCH(); - } - - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 1809 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - assert(tp->tp_dictoffset < 0); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; - #line 2505 "Python/generated_cases.c.h" - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 1825 "Python/bytecodes.c" - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; - DEOPT_IF(res == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; - #line 2533 "Python/generated_cases.c.h" - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; - DISPATCH(); - } - - TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 1841 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); - uint16_t hint = index; - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t dict_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - DEOPT_IF(res == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; - #line 2575 "Python/generated_cases.c.h" - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(LOAD_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 1871 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - char *addr = (char *)owner + index; - res = *(PyObject **)addr; - DEOPT_IF(res == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; - #line 2600 "Python/generated_cases.c.h" - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; DISPATCH(); } - TARGET(LOAD_ATTR_CLASS) { - PyObject *cls = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 1884 "Python/bytecodes.c" - - DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, - LOAD_ATTR); - assert(type_version != 0); - - STAT_INC(LOAD_ATTR, hit); - res2 = NULL; - res = descr; - assert(res != NULL); - Py_INCREF(res); - #line 2627 "Python/generated_cases.c.h" - Py_DECREF(cls); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *fget = read_obj(&next_instr[5].cache); - #line 1899 "Python/bytecodes.c" + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); @@ -2654,2156 +4161,2123 @@ Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); - new_frame->localsplus[0] = owner; - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - #line 2665 "Python/generated_cases.c.h" - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1925 "Python/bytecodes.c" - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - - PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); - #line 2699 "Python/generated_cases.c.h" } - TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 1953 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - Py_DECREF(old_value); + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - Py_DECREF(owner); - #line 2725 "Python/generated_cases.c.h" - STACK_SHRINK(2); - next_instr += 4; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t hint = read_u16(&next_instr[3].cache); - #line 1973 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + // _CHECK_ATTR_WITH_HINT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); } - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - dict->ma_version_tag = new_version; // PEP 509 - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - Py_DECREF(owner); - #line 2776 "Python/generated_cases.c.h" - STACK_SHRINK(2); - next_instr += 4; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } - TARGET(STORE_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - #line 2015 "Python/bytecodes.c" - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - #line 2797 "Python/generated_cases.c.h" - STACK_SHRINK(2); - next_instr += 4; + TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; + } + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } - TARGET(COMPARE_OP) { - PREDICTED(COMPARE_OP); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *res; - #line 2034 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert((oparg >> 4) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg>>4); - #line 2822 "Python/generated_cases.c.h" - Py_DECREF(left); - Py_DECREF(right); - #line 2047 "Python/bytecodes.c" - if (res == NULL) goto pop_2_error; - #line 2827 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(COMPARE_OP_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *res; - #line 2051 "Python/bytecodes.c" - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2849 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + PyObject *value; + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; + } + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(COMPARE_OP_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *res; - #line 2065 "Python/bytecodes.c" - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2875 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(COMPARE_OP_STR) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *res; - #line 2083 "Python/bytecodes.c" - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >>4) == Py_EQ || (oparg >>4) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - #line 2898 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(IS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *b; - #line 2097 "Python/bytecodes.c" - int res = Py_Is(left, right) ^ oparg; - #line 2911 "Python/generated_cases.c.h" - Py_DECREF(left); - Py_DECREF(right); - #line 2099 "Python/bytecodes.c" - b = res ? Py_True : Py_False; - #line 2916 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = b; + TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(CONTAINS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *b; - #line 2103 "Python/bytecodes.c" - int res = PySequence_Contains(right, left); - #line 2928 "Python/generated_cases.c.h" - Py_DECREF(left); - Py_DECREF(right); - #line 2105 "Python/bytecodes.c" - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - #line 2934 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = b; + TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } - TARGET(CHECK_EG_MATCH) { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; - PyObject *rest; - PyObject *match; - #line 2110 "Python/bytecodes.c" - if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2947 "Python/generated_cases.c.h" - Py_DECREF(exc_value); - Py_DECREF(match_type); - #line 2112 "Python/bytecodes.c" - if (true) goto pop_2_error; + TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + goto error; } - - match = NULL; - rest = NULL; - int res = exception_group_match(frame, exc_value, match_type, - &match, &rest); - #line 2958 "Python/generated_cases.c.h" - Py_DECREF(exc_value); - Py_DECREF(match_type); - #line 2120 "Python/bytecodes.c" - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + if (!value) { + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + goto error; + } } - #line 2970 "Python/generated_cases.c.h" - stack_pointer[-1] = match; - stack_pointer[-2] = rest; + Py_DECREF(class_dict); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(CHECK_EXC_MATCH) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *b; - #line 2131 "Python/bytecodes.c" - assert(PyExceptionInstance_Check(left)); - if (check_except_type_valid(tstate, right) < 0) { - #line 2983 "Python/generated_cases.c.h" - Py_DECREF(right); - #line 2134 "Python/bytecodes.c" - if (true) goto pop_1_error; + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; } - - int res = PyErr_GivenExceptionMatches(left, right); - #line 2990 "Python/generated_cases.c.h" - Py_DECREF(right); - #line 2139 "Python/bytecodes.c" - b = res ? Py_True : Py_False; - #line 2994 "Python/generated_cases.c.h" - stack_pointer[-1] = b; + if (v == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error; + if (v == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error; + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto pop_1_error; + } + } + } + } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; DISPATCH(); } - TARGET(IMPORT_NAME) { - PyObject *fromlist = stack_pointer[-1]; - PyObject *level = stack_pointer[-2]; + TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; PyObject *res; - #line 2143 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - res = import_name(tstate, frame, name, fromlist, level); - #line 3006 "Python/generated_cases.c.h" - Py_DECREF(level); - Py_DECREF(fromlist); - #line 2146 "Python/bytecodes.c" - if (res == NULL) goto pop_2_error; - #line 3011 "Python/generated_cases.c.h" - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; + } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } + } + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } - TARGET(IMPORT_FROM) { - PyObject *from = stack_pointer[-1]; + TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; - #line 2150 "Python/bytecodes.c" - PyObject *name = GETITEM(frame->f_code->co_names, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - #line 3024 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _GUARD_BUILTINS_VERSION + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } - TARGET(JUMP_FORWARD) { - #line 2156 "Python/bytecodes.c" - JUMPBY(oparg); - #line 3033 "Python/generated_cases.c.h" + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } - TARGET(JUMP_BACKWARD) { - PREDICTED(JUMP_BACKWARD); - #line 2160 "Python/bytecodes.c" - assert(oparg < INSTR_OFFSET()); - JUMPBY(-oparg); - #line 3042 "Python/generated_cases.c.h" - CHECK_EVAL_BREAKER(); + TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } - TARGET(POP_JUMP_IF_FALSE) { - PREDICTED(POP_JUMP_IF_FALSE); - PyObject *cond = stack_pointer[-1]; - #line 2166 "Python/bytecodes.c" - if (Py_IsFalse(cond)) { - JUMPBY(oparg); + TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } - else if (!Py_IsTrue(cond)) { - int err = PyObject_IsTrue(cond); - #line 3056 "Python/generated_cases.c.h" - Py_DECREF(cond); - #line 2172 "Python/bytecodes.c" - if (err == 0) { - JUMPBY(oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + goto error; } - else { - if (err < 0) goto pop_1_error; + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + goto error; + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } } } - #line 3066 "Python/generated_cases.c.h" - STACK_SHRINK(1); + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond = stack_pointer[-1]; - #line 2182 "Python/bytecodes.c" - if (Py_IsTrue(cond)) { - JUMPBY(oparg); + TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *class; + PyObject *global_super; + PyObject *self; + PyObject *attr; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_SUPER_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - else if (!Py_IsFalse(cond)) { - int err = PyObject_IsTrue(cond); - #line 3079 "Python/generated_cases.c.h" - Py_DECREF(cond); - #line 2188 "Python/bytecodes.c" - if (err > 0) { - JUMPBY(oparg); + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; } - else { - if (err < 0) goto pop_1_error; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; } - #line 3089 "Python/generated_cases.c.h" - STACK_SHRINK(1); + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + (oparg & 1); DISPATCH(); } - TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *value = stack_pointer[-1]; - #line 2198 "Python/bytecodes.c" - if (!Py_IsNone(value)) { - #line 3098 "Python/generated_cases.c.h" - Py_DECREF(value); - #line 2200 "Python/bytecodes.c" - JUMPBY(oparg); - } - #line 3103 "Python/generated_cases.c.h" - STACK_SHRINK(1); + TARGET(LOAD_SUPER_ATTR_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + /* Skip 1 cache entry */ + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2; DISPATCH(); } - TARGET(POP_JUMP_IF_NONE) { - PyObject *value = stack_pointer[-1]; - #line 2205 "Python/bytecodes.c" - if (Py_IsNone(value)) { - JUMPBY(oparg); + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + /* Skip 1 cache entry */ + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; } - else { - #line 3115 "Python/generated_cases.c.h" - Py_DECREF(value); - #line 2210 "Python/bytecodes.c" + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; } - #line 3119 "Python/generated_cases.c.h" - STACK_SHRINK(1); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2214 "Python/bytecodes.c" - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); - #line 3132 "Python/generated_cases.c.h" + TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + goto error; + } + SETLOCAL(oparg, cell); DISPATCH(); } - TARGET(GET_LEN) { - PyObject *obj = stack_pointer[-1]; - PyObject *len_o; - #line 2223 "Python/bytecodes.c" - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - #line 3145 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = len_o; + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + goto error; + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; + DISPATCH(); + } + + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + PyObject *value; + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; DISPATCH(); } TARGET(MATCH_CLASS) { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; - #line 2231 "Python/bytecodes.c" + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); - attrs = match_class(tstate, subject, type, oparg, names); - #line 3161 "Python/generated_cases.c.h" + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2236 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! attrs = Py_None; // Failure! } - #line 3173 "Python/generated_cases.c.h" - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; DISPATCH(); } TARGET(MATCH_MAPPING) { - PyObject *subject = stack_pointer[-1]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; PyObject *res; - #line 2246 "Python/bytecodes.c" + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 3185 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; - PREDICT(POP_JUMP_IF_FALSE); + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } TARGET(MATCH_SEQUENCE) { - PyObject *subject = stack_pointer[-1]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; PyObject *res; - #line 2252 "Python/bytecodes.c" + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 3198 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; - PREDICT(POP_JUMP_IF_FALSE); + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(MATCH_KEYS) { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; - PyObject *values_or_none; - #line 2258 "Python/bytecodes.c" - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = match_keys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - #line 3213 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); DISPATCH(); } - TARGET(GET_ITER) { - PyObject *iterable = stack_pointer[-1]; - PyObject *iter; - #line 2264 "Python/bytecodes.c" - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - #line 3225 "Python/generated_cases.c.h" - Py_DECREF(iterable); - #line 2267 "Python/bytecodes.c" - if (iter == NULL) goto pop_1_error; - #line 3229 "Python/generated_cases.c.h" - stack_pointer[-1] = iter; + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + stack_pointer += -1; DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable = stack_pointer[-1]; - PyObject *iter; - #line 2271 "Python/bytecodes.c" - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - goto error; + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + /* Skip 1 cache entry */ + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); } - iter = iterable; } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; + // _POP_JUMP_IF_TRUE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - goto error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + /* Skip 1 cache entry */ + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); } - #line 3260 "Python/generated_cases.c.h" - Py_DECREF(iterable); - #line 2294 "Python/bytecodes.c" } - #line 3264 "Python/generated_cases.c.h" - stack_pointer[-1] = iter; - PREDICT(LOAD_CONST); + // _POP_JUMP_IF_FALSE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } + stack_pointer += -1; DISPATCH(); } - TARGET(FOR_ITER) { - PREDICTED(FOR_ITER); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter = stack_pointer[-1]; - PyObject *next; - #line 2313 "Python/bytecodes.c" + TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + PyObject *cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; - } - monitor_raise(tstate, frame, next_instr-1); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); - DISPATCH(); - } - // Common case: no jump, leave it to the code generator - #line 3306 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = next; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_TOP) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + stack_pointer += -1; DISPATCH(); } - TARGET(INSTRUMENTED_FOR_ITER) { - #line 2346 "Python/bytecodes.c" - _Py_CODEUNIT *here = next_instr-1; - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; } else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; - } - monitor_raise(tstate, frame, here); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + prev_exc = Py_None; } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3340 "Python/generated_cases.c.h" + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } - TARGET(FOR_ITER_LIST) { - PyObject *iter = stack_pointer[-1]; - PyObject *next; - #line 2374 "Python/bytecodes.c" - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - _PyListIterObject *it = (_PyListIterObject *)iter; - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyList_GET_SIZE(seq)) { - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_list; // End of this instruction - } - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); - DISPATCH(); - end_for_iter_list: - // Common case: no jump, leave it to the code generator - #line 3367 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = next; + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + PyObject *res; + res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(FOR_ITER_TUPLE) { - PyObject *iter = stack_pointer[-1]; - PyObject *next; - #line 2396 "Python/bytecodes.c" - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyTuple_GET_SIZE(seq)) { - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_tuple; // End of this instruction + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + PyObject **args; + args = &stack_pointer[-oparg]; + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - it->it_seq = NULL; - Py_DECREF(seq); + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); - DISPATCH(); - end_for_iter_tuple: - // Common case: no jump, leave it to the code generator - #line 3397 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = next; + if (true) { stack_pointer += -oparg; goto error; } + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + goto error; + } + } + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); DISPATCH(); } - TARGET(FOR_ITER_RANGE) { - PyObject *iter = stack_pointer[-1]; - PyObject *next; - #line 2418 "Python/bytecodes.c" - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); - DISPATCH(); + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + assert(frame == tstate->current_frame); + #ifdef Py_GIL_DISABLED + // For thread-safety, we need to check instrumentation version + // even when tracing. Otherwise, another thread may concurrently + // re-write the bytecode while we are executing this function. + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); + } + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ + } } - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) { - goto error; + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } - #line 3425 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } - TARGET(FOR_ITER_GEN) { - PyObject *iter = stack_pointer[-1]; - #line 2438 "Python/bytecodes.c" - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - DISPATCH_INLINED(gen_frame); - #line 3450 "Python/generated_cases.c.h" + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); } - TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr = stack_pointer[-1]; - PyObject *exit; - PyObject *res; - #line 2456 "Python/bytecodes.c" - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - goto error; - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - goto error; + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; + // _LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); } - #line 3480 "Python/generated_cases.c.h" - Py_DECREF(mgr); - #line 2479 "Python/bytecodes.c" - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; + // _POP_FRAME + retval = value; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + LLTRACE_RESUME_FRAME(); } - #line 3489 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; - stack_pointer[-2] = exit; - PREDICT(GET_AWAITABLE); DISPATCH(); } - TARGET(BEFORE_WITH) { - PyObject *mgr = stack_pointer[-1]; - PyObject *exit; + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); PyObject *res; - #line 2489 "Python/bytecodes.c" - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - goto error; - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { goto error; } - #line 3527 "Python/generated_cases.c.h" - Py_DECREF(mgr); - #line 2515 "Python/bytecodes.c" - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - #line 3536 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; - stack_pointer[-2] = exit; + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + res = (PyObject *)gen; + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + LOAD_SP(); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(WITH_EXCEPT_START) { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; - PyObject *res; - #line 2524 "Python/bytecodes.c" - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; + TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + LLTRACE_RESUME_FRAME(); + DISPATCH(); + } - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; + TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - else { - Py_DECREF(tb); + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + _PyEval_MonitorRaise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + goto error; + } + } + Py_DECREF(v); } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - #line 3574 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[-1] = retval; DISPATCH(); } - TARGET(PUSH_EXC_INFO) { - PyObject *new_exc = stack_pointer[-1]; - PyObject *prev_exc; - #line 2552 "Python/bytecodes.c" - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + PyObject *v; + PyObject *receiver; + /* Skip 1 cache entry */ + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; + } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; } else { - prev_exc = Py_None; + Py_DECREF(ann_dict); } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - #line 3593 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = new_exc; - stack_pointer[-2] = prev_exc; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t keys_version = read_u32(&next_instr[3].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2564 "Python/bytecodes.c" - /* Cached method object */ - PyTypeObject *self_cls = Py_TYPE(self); - assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - res2 = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res = self; - assert(oparg & 1); - #line 3624 "Python/generated_cases.c.h" - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2583 "Python/bytecodes.c" - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; - assert(oparg & 1); - #line 3648 "Python/generated_cases.c.h" - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); + } + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2595 "Python/bytecodes.c" - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = self_cls->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)self + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; - assert(oparg & 1); - #line 3676 "Python/generated_cases.c.h" - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(KW_NAMES) { - #line 2611 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); - kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3689 "Python/generated_cases.c.h" + TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; + PyObject *owner; + PyObject *v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; DISPATCH(); } - TARGET(INSTRUMENTED_CALL) { - #line 2617 "Python/bytecodes.c" - int is_meth = PEEK(oparg+2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(total_args + 1); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); - if (err) goto error; - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - GO_TO_INSTRUCTION(CALL); - #line 3707 "Python/generated_cases.c.h" - } - - TARGET(CALL) { - PREDICTED(CALL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2662 "Python/bytecodes.c" - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; - } - #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_Call(callable, next_instr, total_args, kwnames); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { - is_meth = 1; // For consistenct; it's dead, though - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - int positional_args = total_args - KWNAMES_LEN(); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + // _GUARD_DORV_NO_DICT { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - kwnames = NULL; - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR); + DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); } else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } + Py_DECREF(old_value); } + Py_DECREF(owner); } - kwnames = NULL; - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3799 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + stack_pointer += -2; DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2750 "Python/bytecodes.c" - DEOPT_IF(method != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // callable - PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // method - Py_DECREF(callable); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3821 "Python/generated_cases.c.h" - } - - TARGET(CALL_PY_EXACT_ARGS) { - PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2762 "Python/bytecodes.c" - assert(kwnames == NULL); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; - int argcount = oparg; - if (is_meth) { - callable = method; - args--; - argcount++; + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != argcount, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); } - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - #line 3856 "Python/generated_cases.c.h" + stack_pointer += -2; + DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2790 "Python/bytecodes.c" - assert(kwnames == NULL); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; - int argcount = oparg; - if (is_meth) { - callable = method; - args--; - argcount++; + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!PyDict_CheckExact((PyObject *)dict), STORE_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - #line 3900 "Python/generated_cases.c.h" - } - - TARGET(CALL_NO_KW_TYPE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2828 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - #line 3918 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + dict->ma_version_tag = new_version; // PEP 509 + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + Py_DECREF(owner); + stack_pointer += -2; DISPATCH(); } - TARGET(CALL_NO_KW_STR_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2840 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3942 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_NO_KW_TUPLE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2854 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3967 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_BUILTIN_CLASS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2868 "Python/bytecodes.c" - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; - } - int kwnames_len = KWNAMES_LEN(); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, - total_args - kwnames_len, kwnames); - kwnames = NULL; - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4003 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; DISPATCH(); } - TARGET(CALL_NO_KW_BUILTIN_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2893 "Python/bytecodes.c" - /* Builtin METH_O functions */ - assert(kwnames == NULL); - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4045 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; DISPATCH(); } - TARGET(CALL_NO_KW_BUILTIN_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2924 "Python/bytecodes.c" - /* Builtin METH_FASTCALL functions, without keywords */ - assert(kwnames == NULL); - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - #line 4091 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2959 "Python/bytecodes.c" - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc( - PyCFunction_GET_SELF(callable), - args, - total_args - KWNAMES_LEN(), - kwnames - ); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - kwnames = NULL; - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4137 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_NO_KW_LEN) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 2994 "Python/bytecodes.c" - assert(kwnames == NULL); - /* len(o) */ - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - goto error; + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4176 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; DISPATCH(); } - TARGET(CALL_NO_KW_ISINSTANCE) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 3021 "Python/bytecodes.c" - assert(kwnames == NULL); - /* isinstance(o, o2) */ - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - callable = method; - args--; - total_args++; + TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - goto error; + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4216 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + stack_pointer += -3; DISPATCH(); } - TARGET(CALL_NO_KW_LIST_APPEND) { - PyObject **args = (stack_pointer - oparg); - PyObject *self = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3051 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg == 1); - PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(method != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(method); - STACK_SHRINK(3); - // CALL + POP_TOP - JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *value; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; DISPATCH(); - #line 4246 "Python/generated_cases.c.h" } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 3071 "Python/bytecodes.c" - assert(kwnames == NULL); - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - args--; - total_args++; - } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4284 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *value; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; - PyObject *res; - #line 3105 "Python/bytecodes.c" - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - args--; - total_args++; - } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - kwnames = NULL; - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4326 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; DISPATCH(); } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; + PyObject *value; PyObject *res; - #line 3137 "Python/bytecodes.c" - assert(kwnames == NULL); - assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - args--; - total_args++; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(TO_BOOL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + /* Skip 2 cache entries */ + // _TO_BOOL + { + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4368 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *owner; + PyObject *value; PyObject *res; - #line 3169 "Python/bytecodes.c" - assert(kwnames == NULL); - int is_meth = method != NULL; - int total_args = oparg; - if (is_meth) { - args--; - total_args++; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _REPLACE_WITH_TRUE + value = owner; + { + Py_DECREF(value); + res = Py_True; } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4409 "Python/generated_cases.c.h" - STACK_SHRINK(oparg); - STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3200 "Python/bytecodes.c" - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4421 "Python/generated_cases.c.h" + TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); } - TARGET(CALL_FUNCTION_EX) { - PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; - PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; - PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; - PyObject *result; - #line 3204 "Python/bytecodes.c" - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - goto error; - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - goto error; - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); - if (err) goto error; - result = PyObject_Call(func, callargs, kwargs); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } + TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortalLoose(value)); + res = Py_False; } else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); + Py_DECREF(value); + res = Py_True; } - #line 4492 "Python/generated_cases.c.h" - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - #line 3266 "Python/bytecodes.c" - assert(PEEK(3 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4499 "Python/generated_cases.c.h" - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; - CHECK_EVAL_BREAKER(); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(MAKE_FUNCTION) { - PyObject *codeobj = stack_pointer[-1]; - PyObject *closure = (oparg & 0x08) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0))] : NULL; - PyObject *annotations = (oparg & 0x04) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0))] : NULL; - PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; - PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; - PyObject *func; - #line 3276 "Python/bytecodes.c" - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - - Py_DECREF(codeobj); - if (func_obj == NULL) { - goto error; - } - - if (oparg & 0x08) { - assert(PyTuple_CheckExact(closure)); - func_obj->func_closure = closure; - } - if (oparg & 0x04) { - assert(PyTuple_CheckExact(annotations)); - func_obj->func_annotations = annotations; - } - if (oparg & 0x02) { - assert(PyDict_CheckExact(kwdefaults)); - func_obj->func_kwdefaults = kwdefaults; - } - if (oparg & 0x01) { - assert(PyTuple_CheckExact(defaults)); - func_obj->func_defaults = defaults; - } - - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; - func = (PyObject *)func_obj; - #line 4543 "Python/generated_cases.c.h" - STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); - stack_pointer[-1] = func; + TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(RETURN_GENERATOR) { - #line 3307 "Python/bytecodes.c" - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - goto error; - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - goto resume_frame; - #line 4571 "Python/generated_cases.c.h" - } - - TARGET(BUILD_SLICE) { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; - PyObject *slice; - #line 3330 "Python/bytecodes.c" - slice = PySlice_New(start, stop, step); - #line 4581 "Python/generated_cases.c.h" - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - #line 3332 "Python/bytecodes.c" - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4587 "Python/generated_cases.c.h" - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(FORMAT_VALUE) { - PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; - PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; - PyObject *result; - #line 3336 "Python/bytecodes.c" - /* Handles f-string value formatting. */ - PyObject *(*conv_fn)(PyObject *); - int which_conversion = oparg & FVC_MASK; - - /* See if any conversion is specified. */ - switch (which_conversion) { - case FVC_NONE: conv_fn = NULL; break; - case FVC_STR: conv_fn = PyObject_Str; break; - case FVC_REPR: conv_fn = PyObject_Repr; break; - case FVC_ASCII: conv_fn = PyObject_ASCII; break; - default: - _PyErr_Format(tstate, PyExc_SystemError, - "unexpected conversion flag %d", - which_conversion); - goto error; + TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortalLoose(value)); + res = Py_False; } - - /* If there's a conversion function, call it and replace - value with that result. Otherwise, just use value, - without conversion. */ - if (conv_fn != NULL) { - result = conv_fn(value); + else { + assert(Py_SIZE(value)); Py_DECREF(value); - if (result == NULL) { - Py_XDECREF(fmt_spec); - if (true) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - } - value = result; + res = Py_True; } + stack_pointer[-1] = res; + DISPATCH(); + } - result = PyObject_Format(value, fmt_spec); + TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Invert(value); Py_DECREF(value); - Py_XDECREF(fmt_spec); - if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4633 "Python/generated_cases.c.h" - STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); - stack_pointer[-1] = result; + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(COPY) { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; - PyObject *top; - #line 3373 "Python/bytecodes.c" - assert(oparg > 0); - top = Py_NewRef(bottom); - #line 4645 "Python/generated_cases.c.h" - STACK_GROW(1); - stack_pointer[-1] = top; + TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_OP) { - PREDICTED(BINARY_OP); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + PyObject *value; PyObject *res; - #line 3378 "Python/bytecodes.c" - #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - res = binary_ops[oparg](lhs, rhs); - #line 4672 "Python/generated_cases.c.h" - Py_DECREF(lhs); - Py_DECREF(rhs); - #line 3393 "Python/bytecodes.c" - if (res == NULL) goto pop_2_error; - #line 4677 "Python/generated_cases.c.h" - STACK_SHRINK(1); + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } - TARGET(SWAP) { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3398 "Python/bytecodes.c" - assert(oparg >= 2); - #line 4689 "Python/generated_cases.c.h" - stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; + TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } - TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3402 "Python/bytecodes.c" - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); - if (next_opcode < 0) goto error; - next_instr--; - if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - #line 4708 "Python/generated_cases.c.h" - } - - TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3416 "Python/bytecodes.c" - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4714 "Python/generated_cases.c.h" + // _UNPACK_SEQUENCE + { + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; DISPATCH(); } - TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3420 "Python/bytecodes.c" - INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4721 "Python/generated_cases.c.h" - CHECK_EVAL_BREAKER(); + TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3425 "Python/bytecodes.c" - PyObject *cond = POP(); - int err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err < 0) goto error; - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - int offset = err*oparg; - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4736 "Python/generated_cases.c.h" + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3436 "Python/bytecodes.c" - PyObject *cond = POP(); - int err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err < 0) goto error; - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - int offset = (1-err)*oparg; - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4750 "Python/generated_cases.c.h" + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject *val1; + PyObject *val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + Py_DECREF(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3447 "Python/bytecodes.c" - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; - int offset; - if (Py_IsNone(value)) { - offset = oparg; + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_func: FOURTH = the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; } else { - Py_DECREF(value); - offset = 0; + Py_DECREF(tb); } - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4767 "Python/generated_cases.c.h" + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3461 "Python/bytecodes.c" - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; - int offset; - if (Py_IsNone(value)) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; - } - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4784 "Python/generated_cases.c.h" + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + PyObject *value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - - TARGET(EXTENDED_ARG) { - #line 3475 "Python/bytecodes.c" - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - #line 4795 "Python/generated_cases.c.h" - } - - TARGET(CACHE) { - #line 3483 "Python/bytecodes.c" - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - #line 4802 "Python/generated_cases.c.h" - } - - TARGET(RESERVED) { - #line 3488 "Python/bytecodes.c" - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); - #line 4809 "Python/generated_cases.c.h" - } +#undef TIER_ONE |